Sample code for adding a custom button on a custom tab for SubGrid of an entity in ribbon (CRM 2011)

Just replace the schema name of your entity.

 <RibbonDiffXml>
        <CustomActions>
          <CustomAction Id="Sample.SubGrid.entitySchemaName.CustomTab.CustomAction" Location="Mscrm.SubGrid.entitySchemaName.ContextualTabs._children" Sequence="40">
            <CommandUIDefinition>
              <Tab Id="Sample.SubGrid.entitySchemaName.CustomTab" Command="Sample.SubGrid.entitySchemaName.CustomTab" Title="Custom Tab" Description="Custom Tab Description" Sequence="500">
                <Scaling Id="Sample.SubGrid.entitySchemaName.CustomTab.Scaling">
                  <MaxSize Id="Sample.SubGrid.entitySchemaName.CustomTab.FirstGroup.MaxSize" GroupId="Sample.SubGrid.entitySchemaName.CustomTab.FirstGroup" Sequence="10" Size="LargeMedium" />
                </Scaling>
                <Groups Id="Sample.SubGrid.entitySchemaName.CustomTab.Groups">
                  <Group Id="Sample.SubGrid.entitySchemaName.CustomTab.FirstGroup" Command="Sample.SubGrid.entitySchemaName.FirstGroup" Sequence="10" Title="Custom First Group" Template="Mscrm.Templates.3.3">
                    <Controls Id="Sample.SubGrid.entitySchemaName.CustomTab.FirstGroup.Controls">
                      <Button Id="Sample.SubGrid.entitySchemaName.CustomTab.FirstGroup.FirstButton" ToolTipTitle="ButtonToolTipTitle" ToolTipDescription="ToolTipDescription" Command="Sample.SubGrid.entitySchemaName.FirstButton" Sequence="10" LabelText="LabelText" Alt="LabelText" Image16by16="/_imgs/ribbon/AddEmail_16.png" Image32by32="/_imgs/ribbon/Email_32.png" TemplateAlias="o1" />
                    </Controls>
                  </Group>
                </Groups>
              </Tab>
            </CommandUIDefinition>
          </CustomAction>
        </CustomActions>
        <Templates>
          <RibbonTemplates Id="Mscrm.Templates"></RibbonTemplates>
        </Templates>
        <CommandDefinitions>
          <CommandDefinition Id="Sample.SubGrid.entitySchemaName.FirstGroup">
            <EnableRules>
              <EnableRule Id="Mscrm.Enabled " />
            </EnableRules>
            <DisplayRules>
            </DisplayRules>
            <Actions />
          </CommandDefinition>
          <CommandDefinition Id="Sample.SubGrid.entitySchemaName.CustomTab">
            <EnableRules>
              <EnableRule Id="Mscrm.Enabled " />
            </EnableRules>
            <DisplayRules>
            </DisplayRules>
            <Actions />
          </CommandDefinition>
          <CommandDefinition Id="Sample.SubGrid.entitySchemaName.FirstButton">
            <EnableRules />
            <DisplayRules />
            <Actions>
              <JavaScriptFunction Library="$webresource:new_test" FunctionName="showvalue">
                <StringParameter Value="Guid(s) Selected" > </StringParameter>
                <CrmParameter Value="SelectedControlSelectedItemIds"></CrmParameter>
              </JavaScriptFunction>
            </Actions>
          </CommandDefinition>
        </CommandDefinitions>
        <RuleDefinitions>
          <TabDisplayRules>
            <TabDisplayRule TabCommand="Sample.SubGrid.entitySchemaName.CustomTab">
              <EntityRule EntityName="entitySchemaName" Context="SubGridStandard" />
              <EntityRule EntityName="entitySchemaName" Context="SubGridAssociated" />
            </TabDisplayRule>
            <TabDisplayRule TabCommand="Sample.Form.entitySchemaName.CustomTab">
              <EntityRule EntityName="entitySchemaName" Context="Form" AppliesTo="PrimaryEntity" />
            </TabDisplayRule>
            <TabDisplayRule TabCommand="Sample.Grid.entitySchemaName.CustomTab">
              <EntityRule EntityName="entitySchemaName" Context="HomePageGrid" />
            </TabDisplayRule>
          </TabDisplayRules>
          <DisplayRules />
          <EnableRules />
        </RuleDefinitions>
        <LocLabels />
      </RibbonDiffXml>

This is the helpful post

http://dynamicscrm2011.wordpress.com/tag/adding-a-button-to-the-crm-2011-sub-grid-ribbon/

Hope it helps !