Hi,

You can use javascript and action fucntion for the custom search option. So, try the following VF page and controller.

This example page is used to search Candidates from candidate custom object based on their skills.

VF page:

<apex:page standardcontroller="Candidate__c" extensions="ActiveCandidateSearchCon" sidebar="false">
  <apex:form >
            <apex:pageMessages id="errors" />
  <apex:pageBlock title="Find Me A Customer!" mode="edit">
  <table width="100%" border="0">
  <tr>  
    <td width="200" valign="top">
 
      <apex:pageBlock title="Parameters" mode="edit" id="criteria">
      <script type="text/javascript">
      function doSearch() {
        searchServer(
          document.getElementById("skills").value          
          );
      }
      </script>
 
      <apex:actionFunction name="searchServer" action="{!runSearch}" rerender="results,debug,errors">        
          <apex:param name="skills" value="" />        
      <table cellpadding="2" cellspacing="2">     
      <tr>
        <td style="font-weight:bold;">Skills<br/>
        <input type="text" id="skills" onkeyup="doSearch();"/>
        </td>
      </tr>          
      </table>
       </apex:pageBlock>
    </td>

    <td valign="top">
    <apex:pageBlock mode="edit" id="results">
        <apex:pageBlockTable value="{!candidates}" var="cand">
      //Here you can include as much as column you needed to dispaly results for search     

          <apex:column >
                <apex:facet name="header">
                    <apex:commandLink value="Skills" action="{!toggleSort}" rerender="results,debug">
                        <apex:param name="sortField" value="Skills1__c" assignTo="{!sortField}"/>
                    </apex:commandLink>
                </apex:facet>
                <apex:outputField value="{!cand.Skills1__c}"/>
            </apex:column>
        </apex:pageBlockTable>
    </apex:pageBlock>
    </td>
  </tr>
  </table>
  <apex:pageBlock title="Debug - SOQL" id="debug">
      <apex:outputText value="{!debugSoql}" />           
  </apex:pageBlock>  

  </apex:pageBlock>
  </apex:form>
</apex:page>

Controller:

public with sharing class ActiveCandidateSearchCon {
     public Candidate__c can{get;set;}

    //Constructor
    public ActiveCandidateSearchCon(ApexPages.StandardController controller) {
             can = (Candidate__c)controller.getRecord();
    }
  // the soql without the order and limit
  private String soql {get;set;}
  // the collection of Candidates to display
  public List<Candidate__c> candidates{get;set;}
  // the current sort direction. defaults to asc
  public String sortDir {
    get  { if (sortDir == null) {  sortDir = 'asc'; } return sortDir;  }
    set;
  }
  // format the soql for display on the visualforce page
  public String debugSoql {
    get { return soql + ' order by ' + sortField + ' ' + sortDir + ' limit 20'; }
    set;
  }
  // init the controller and display some sample data when the page loads
  public ActiveCandidateSearchCon () {
    soql = 'select id,Skills1__c from Candidate__c where id != null ';
    runQuery();
  }
  // toggles the sorting of query from asc<-->desc
  public void toggleSort() {
    // simply toggle the direction
    sortDir = sortDir.equals('asc') ? 'desc' : 'asc';
    // run the query again
    runQuery();
  }
 
  // runs the actual query
  public void runQuery() {
    try {
    if(soql!= null){
      candidates= Database.query(soql + ' order by ' + sortField + ' ' + sortDir + ' limit 20');
      }
    } catch (Exception e) {
      ApexPages.addMessage(new ApexPages.Message(ApexPages.Severity.ERROR, 'Ooops!'));
    }
  }
 
  // runs the search with parameters passed via Javascript
  public PageReference runSearch() {
       String skills = Apexpages.currentPage().getParameters().get('skills') ;
       soql = 'select id, name, Skills1__c, Status__c from Candidate__c where id !=null'; 
    if (!skills.equals(''))
      soql += ' and Skills1__c includes (\''+skills+'\')';  
    // run the query again
    runQuery();
    return null;
  }
}

Hope this will help you...!

Don't forget to give Kudos and mark this answer as a Solution if this works out.