public with sharing class CustomPopupController {

    public boolean showPopup {get;set;}
    
    public CustomPopupController ()
    {
        showPopup = false;
    }
    
    public PageReference openPopup()
    {
        showPopup = true;
        return null;
    }
    
    public PageReference Cancel()
    {
        showPopup = false;
        return null;
    }
    

}

<apex:page controller="CustomPopupController">
<style type="text/css">
    .popupBackground{
        background-color:black;
        opacity: 0.20;
        filter: alpha(opacity = 20);
        position: absolute;
        width: 100%;
        height: 100%;
        top: 0;
        left: 0;
        z-index: 9998;
    }
    .custPopup{
        background-color: white;
        border-width: 2px;
        border-style: solid;
        z-index: 9999;
        left: 50%;
        padding:10px;
        position: absolute;
        width: 500px;
        margin-left: -250px;
        top:100px;
    }

</style>
<apex:form >
 <apex:pageBlock > 

 <apex:commandButton action="{!openPopup}" value="Open Popup" />
 
 <apex:outputPanel id="tstpopup" rendered="{!showPopup}">
                <apex:outputPanel styleClass="popupBackground" layout="block" />
                    <apex:outputPanel styleClass="custPopup" layout="block" >
                        <center>
                              Hello this is Custom pop-Up<BR></BR>
                             <apex:commandButton value="Save"  action="{!Cancel}" />
                             <apex:commandButton value="Cancel" action="{!Cancel}" />
                        </center>
                 </apex:outputPanel>
 </apex:outputPanel>

</apex:pageBlock>
</apex:form>
</apex:page>