To get the filtered lookup functionality in CRM 2011 we can make use of the
addCustomView Method of Xrm.Page.ui control.

Something similar to this
Here we have a custom entity name Case which has 1-n relationship with Contact Entity.
Now we want to show only those contact records in lookup which are associated with a particular case record.
We can add the code to the onload of the Entity.
// View ID --> Generate and Assign a new guid.
var viewId = "{C0F1DD64-1BF3-450D-BCDE-DF4732DE1606}";
// Set the entity name
var entityName = "contact";
// Give a meaningful name to the custom view
var viewDisplayName = "Case Member View";
// Create the Advanced find query and download the fetch xml
var fetchXml = "<fetch version='1.0' output-format='xml-platform' mapping='logical' distinct='false'>" +
"<entity name='contact'>" +
"<attribute name='fullname' />" +
" <attribute name='contactid' />" +
"<order attribute='fullname' descending='false' />" +
"<filter type='and'>" +
"<condition attribute='new_caseid' operator='eq' uiname='new case' uitype='new_case' value="+caseID+"/>" +
"</filter>" +
"</entity>" +
"</fetch>";
// specify the layout for the results and which field to display
var layoutXml = "<grid name='resultset' " +
"object='1' " +
"jump='name' " +
"select='1' " +
"icon='1' " +
"preview='1'>" +
"<row name='result' " +
"id='contactid'>" +
"<cell name='fullname' " +
"width='100' />" +
"</row>" +
"</grid>";
// specify the schemaname of the lookup control
var lookupControl = Xrm.Page.ui.controls.get('new_clientid');
// set the parameters
lookupControl.addCustomView(viewId, entityName, viewDisplayName, fetchXml, layoutXml, true);
