Search with Sitecore - Article - 4 - Create Search API

Dear all friends, this is fourth article in this series and following are the list articles for reference:


1) Search with Sitecore - Article-1 - Introduction 
2) Search with Sitecore - Article - 2 - Configure Advance Database Crawler and More
3) Search with Sitecore - Article - 3 - Crawler to Crawl Media File like PDF, Document  
4) Search with Sitecore - Article - 4 - Create Search API
5) Search with Sitecore - Article - 5 - Auto Complete For Search

Today we will create some methods and class which can help us to search our new indexes created so far.
Now without taking your much time let me put straight the full code i have in front of you then talk about it.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Sitecore.Caching;
using Sitecore.Data.Items;
using Sitecore.Data.Managers;
using scSearchContrib.Searcher;
using scSearchContrib.Searcher.Utilities;
using scSearchContrib.Searcher.Parameters;
using System.IO;
using Sitecore.Data;
using Sitecore.Collections;
using Sitecore.Search;
using Portal.EShopServices.Enumerations;
using Sitecore.Links;
using Lucene.Net.Search;
using Portal.AppServices.Constants;

namespace Portal.EShopServices.Search.Searcher
{
    /// <summary>
    /// This class provide the helper method to query the advance Search Crawler 
    /// This class is based on http://sitecorian.github.io/SitecoreSearchContrib/
    /// </summary>
    public static class MySearcher
    {
        #region All Required Properties
        /// <summary>
        /// Required to Set before any function get called in this Class
        /// This will set the index to search
        /// </summary>
        public static string Index { get; set; }

        /// <summary>
        /// This property lets you know the result count after the search function is executed.
        /// </summary>
        public static int ResultCount { get; set; }

        /// <summary>
        /// This property will allow you to sort the result based on some field
        /// Need to set before function is about to execute.
        /// </summary>
        public static string SortFieldName { get; set; }

        /// <summary>
        /// This property will allow you to tell query did you want all version of the items or not.
        /// Need to set before function is about to execute.
        /// </summary>
        public static bool IsShowAllVersions { get; set; }

        /// <summary>
        /// If you have provided the sort property earlier then this property can arrange the order in descending.. as latest to last
        /// Need to set before function is about to execute
        /// </summary>
        public static bool IsReverseTrue { get; set; }

        /// <summary>
        /// For example, a search resulting in 50 items, page size 10 would go like this:
        ///Page 1 (start=0, end=10) => items 1-10 (ok)
        ///Page 2 (start=10, end=20) => items 10-30 (wrong)
        ///Page 3 (start=20, end=30) => items 20-50 (wrong)
        ///Page 4 (start=30, end=40) => items 30-50 (wrong)
        ///Page 5 (start=40, end=50) => items 40-50 (ok)
        ///Here 'start' is StartRange.
        ///Need to set before function is about to execute
        /// </summary>
        public static int StartRange { get; set; }

        /// <summary>
        /// For example, a search resulting in 50 items, page size 10 would go like this:
        ///Page 1 (start=0, end=10) => items 1-10 (ok)
        ///Page 2 (start=10, end=20) => items 10-30 (wrong)
        ///Page 3 (start=20, end=30) => items 20-50 (wrong)
        ///Page 4 (start=30, end=40) => items 30-50 (wrong)
        ///Page 5 (start=40, end=50) => items 40-50 (ok)
        ///Here 'end' is StartRange.
        ///Need to set before function is about to execute
        /// </summary>
        public static int EndRange { get; set; }

        /// <summary>
        /// This property specifies the query condition like AND/OR/NOT clause
        /// Need to set before function is about to execute
        /// </summary>
        public static Sitecore.Search.QueryOccurance queryOccurence { get; set; }

        /// <summary>
        /// This property allows to search in base template
        /// Need to set before function is about to execute
        /// </summary>
        public static bool IsSearchBaseTemplate { get; set; }
        #endregion

        #region Getting Skinny Items - All Raw Search

        /// <summary>
        /// This will do the search by default for current context database and current context language
        /// Make sure to set properties before calling this function
        /// </summary>
        /// <param name="fullTextQuery">text to be searched</param>
        /// <param name="locationFilters">Multiple IDS can be given</param>
        /// <param name="templateFilters">Multiple templateIDS can be given</param>
        /// <param name="relationFilters">Multiple relationIDS can be given</param>
        /// <returns></returns>
        public static List<SkinnyItem> GetItems(string fullTextQuery, string locationFilters, string templateFilters, string relationFilters)
        {
            return GetItems(Sitecore.Context.Database.Name, Sitecore.Context.Language.Name, templateFilters, locationFilters, relationFilters, fullTextQuery, IsSearchBaseTemplate);
        }

        /// <summary>
        /// This will do the search by default for current context database and current context langauge, template filters and relation filters will be send as blank
        /// Make sure to set properties before calling this function
        /// </summary>
        /// <param name="fullTextQuery">text to be searched</param>
        /// <param name="locationFilters">Multiple locationIDS can be given</param>
        /// <returns></returns>
        public static List<SkinnyItem> GetItems(string fullTextQuery, string locationFilters)
        {
            return GetItems(fullTextQuery, locationFilters, "", "");
        }

        /// <summary>
        /// This will do the search by default for current context database and current context langauge, template filters, location filters and relation filters will be send as blank
        /// Make sure to set properties before calling this function
        /// </summary>
        /// <param name="fullTextQuery">text to be searched</param>
        /// <param name="locationFilters">Multiple locationIDS can be given</param>
        /// <returns></returns>
        public static List<SkinnyItem> GetItems(string fullTextQuery)
        {
            return GetItems(fullTextQuery, "");
        }

        public static List<SkinnyItem> GetItems(string databaseName, string language, string templateFilters, string locationFilters, string relationFilters, string fullTextQuery, bool DoSearchBaseTemplate)
        {
            List<SkinnyItem> result = new List<SkinnyItem>();
            var searchParam = new SearchParam();
            searchParam = new SearchParam
            {
                Database = databaseName,
                Language = language,
                TemplateIds = templateFilters,
                LocationIds = locationFilters,
                FullTextQuery = fullTextQuery,
                RelatedIds = relationFilters,
                SearchBaseTemplates = DoSearchBaseTemplate
            };
            int count = 0;
             //Here Portal.AppServices.Constants.UtilitySettings.FileIndexName is name of our earlier created FileIndex
            if (Index.Equals(Portal.AppServices.Constants.UtilitySettings.FileIndexName, StringComparison.InvariantCultureIgnoreCase))
            {
                Sort sort = null;
                if (!string.IsNullOrEmpty(SortFieldName))
                    sort = new Sort(new SortField(SortFieldName.ToLowerInvariant(), SortField.STRING, IsReverseTrue));

                Query qr = searchParam.ProcessQuery(queryOccurence, SearchManager.GetIndex(Portal.AppServices.Constants.UtilitySettings.FileIndexName), true);
                result = RunQuery(qr, IsShowAllVersions, sort, StartRange, EndRange, out count);
            }
            else
            {
                using (var runner = new QueryRunner(Index, true))
                {
                    Query qr = searchParam.ProcessQuery(queryOccurence, SearchManager.GetIndex(Portal.AppServices.Constants.UtilitySettings.FileIndexName), true);
                    result = runner.RunQuery(qr, false, SortFieldName, true, StartRange, EndRange);
                }
            }
            ResultCount = count;
            return result;
        }

        public static List<SkinnyItem> RunQuery(Query query, bool showAllVersions, Sort sorter, int start, int end, out int totalResults)
        {
            var items = new List<SkinnyItem>();
            if (query == null || string.IsNullOrEmpty(query.ToString()))
            {
                totalResults = 0;
                return items;
            }
            using (var context = new IndexSearchContext(SearchManager.GetIndex(Index)))
            {
                SearchHits searchhits;
                if (sorter != null)
                {
                    var hits = context.Searcher.Search(query, sorter);
                    searchhits = new SearchHits(hits);
                }
                else
                {
                    searchhits = context.Search(query);
                }
                if (searchhits == null)
                {
                    totalResults = 0;
                    return null;
                }

                totalResults = searchhits.Length;
                if (end == 0 || end > searchhits.Length)
                {
                    end = totalResults;
                }
                var resultCollection = searchhits.FetchResults(start, end - start);
                GetItemsFromSearchResult(resultCollection, items, showAllVersions);
            }
            return items;
        }

        public static void GetItemsFromSearchResult(IEnumerable<SearchResult> searchResults, List<SkinnyItem> items, bool showAllVersions)
        {
            foreach (var result in searchResults)
            {
                var guid = result.Document.GetField(BuiltinFields.Tags).StringValue();
                if (guid != null && !string.IsNullOrEmpty(guid))
                {
                    var itemId = new ID(guid);
                    var db = Sitecore.Context.Database;
                    var item = db.GetItem(itemId);
                    var itemInfo = new SkinnyItem(item.Uri);
                    foreach (Lucene.Net.Documents.Field field in result.Document.GetFields())
                    {
                        itemInfo.Fields.Add(field.Name(), field.StringValue());
                    }

                    items.Add(itemInfo);
                }
                if (showAllVersions)
                {
                    GetItemsFromSearchResult(result.Subresults, items, true);
                }
            }
        }
       //This function is used in the Auto Search Functionality for the text box        
 public static Lucene.Net.Store.FSDirectory GetAutoUpdateIndexDirectory()
        {
            string autoUpdate = Portal.AppServices.Constants.UtilitySettings.OuterIndexFolder + @"\" + Portal.AppServices.Constants.UtilitySettings.AutoUpdateIndexName;
            try
            {
                if (!System.IO.Directory.Exists(autoUpdate))
                    System.IO.Directory.CreateDirectory(autoUpdate);
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message + System.Environment.NewLine);
                Console.WriteLine(ex.StackTrace);
            }
            Lucene.Net.Store.FSDirectory fsd = null;
            if (System.IO.Directory.Exists(autoUpdate))
            {
                fsd = Lucene.Net.Store.FSDirectory.Open(new System.IO.DirectoryInfo(autoUpdate));
            }
            return fsd;
        }
        #endregion

        #region Header Search Functions

        /// <summary>
        /// Handle non-pages results to remove them from search results (prevents error pages)
        /// </summary>
        /// <param name="_items">List of Search Results</param>
        /// <returns>List of Filtered Search Results</returns>
        public static List<WrapUpItem> HandleNonPagesItem(List<Item> _items)
        {
            List<WrapUpItem> processList = new List<WrapUpItem>();
            foreach (Item itm in _items)
            {
                if (DoesSitecoreItemHavePeresentation(itm))
                {
                    WrapUpItem _wui = new WrapUpItem();
                    _wui.ItemType = WrapUpItem.type.Default;
                    _wui.Item = itm;
                    processList.Add(_wui);
                }
            }
            return processList;
        }

        public static List<WrapUpItem> HandleMediaItems(List<Item> _items)
        {
            List<WrapUpItem> processList = new List<WrapUpItem>();
            foreach (MediaItem itm in _items)
            {
                WrapUpItem _wui = null;
                try
                {
                    if (Sitecore.Resources.Media.MediaManager.HasMediaContent(itm))
                    {
                        _wui = new WrapUpItem();
                        _wui.MediaUrl = Sitecore.Resources.Media.MediaManager.GetMediaUrl(itm);
                        _wui.Item = itm;
                        if (itm.MimeType.ToLower().Contains("pdf"))
                        {
                            _wui.MediaType = WrapUpItem.mediatype.PDF;
                        }
                        else if (itm.MimeType.ToLower().Contains("word"))
                        {
                            _wui.MediaType = WrapUpItem.mediatype.Word;
                        }
                        else if (itm.MimeType.ToLower().Contains("text"))
                        {
                            _wui.MediaType = WrapUpItem.mediatype.Text;
                        }
                        else if (itm.MimeType.ToLower().Contains("html"))
                        {
                            _wui.MediaType = WrapUpItem.mediatype.HTML;
                        }
                        _wui.ItemType = WrapUpItem.type.Media;
                    }
                }
                catch { }
                finally { if (_wui != null) { processList.Add(_wui); } }
            }
            return processList;
        }

        public static List<WrapUpItem> HandleProductItems(List<Item> _items)
        {
            List<WrapUpItem> processList = new List<WrapUpItem>();
            foreach (Item itm in _items)
            {
                WrapUpItem _wui = null;
                bool isActiveProduct = false;
                try
                {
                    if (isProductItem(itm))
                    {
                        _wui = new WrapUpItem();
                        _wui.ItemType = WrapUpItem.type.Product;
                        _wui.Item = itm;
                       //Write your logic to assign further properties of the WrapUpItem class
                    }
                }
                catch { }
                finally { if (_wui != null && isActiveProduct) { processList.Add(_wui); } }
            }
            return processList;
        }

        public static bool isProductItem(Item itm)
        {
            bool result = false;
            // Write your own logic to distinguish specific Item like product 
            return result;
        }

        public static bool DoesSitecoreItemHavePeresentation(Item item)
        {
            return item.Fields[Sitecore.FieldIDs.LayoutField] != null
                   && item.Fields[Sitecore.FieldIDs.LayoutField].Value
                   != String.Empty;
        }

        /// <summary>
        /// Check if an item URL contains "-w-", which is an invalid link
        /// </summary>
        /// <param name="_item"></param>
        /// <returns>true if contains "-w-"; otherwise, false</returns>
        private static bool CheckForParameters(Item _item)
        {
            string itemUrl = LinkManager.GetItemUrl(_item);
            if (itemUrl.Contains("-w-"))
                return true;
            else
                return false;
        }
        #endregion
    }
    /// <summary>
    /// This Class Provide an Wrap Class which has additional properties required during search. 
    /// </summary>
    public class WrapUpItem
    {
        public enum type
        {
            Product = 0,
            Media = 1,
            Default = 2
        }

        public enum mediatype
        {
            PDF = 0,
            Word = 1,
            Text = 2,
            HTML = 3
        }
        public Item Item { get; set; }
        public type ItemType { get; set; }
        public string MediaUrl { get; set; }
        public string ProductUrl { get; set; }
        public mediatype MediaType { get; set; }
    }
}

First few facts about above code I have three main index and separate index which doesn't build from the Sitecore data.

a) SeacrhItems (This index is for routine content items and its root is pointing to the Home Node which includes it all child also).
b) Product (This index is for specific items which are product items and located under the product catalog obviously out of Home Node).
c) Documents (This index is specifically for the media items stored under the media library path).
d) AutoUpdate (This index is not based on the Sitecore Items and not used for routine search queries, but stored in the same location where other Sitecore Indexes are stored, we will discuss about this index in later post).

Similarly you may have as many index you want targeted to your specific set of contents. I created above class keeping multiple index in mind so that i can query whatever index i like and club them as collection and show them on front-end.
Why i need to club the collection ?
--Well it depend upon requirement, in our project we have just one global textbox sitting on Header where user can type any query and then result should be shown on relevant tabs on an page. So first tab results in collection of all results find through all indexes except fourth one. Here results are arranged in the precedence given to Products >> Normal Search Items >> Files. Well other tabs have particular location targeted like business section, FAQ section etc., under Home node (Using above API we can search based on the location from our SearchItems Index).

Now To quickly explain what we have done above, is first we have an class which provide us properties, flag variables and methods which assist us to make search and differentiate between items. Items could be media files, normal content item or could be some specific business item like product for which you may target in search. Above class also provide some routine functions like knowing whether an Item is an brows-able item or not.
From purpose of making binding easy at the front-end side in .net user control we have created another class called 'WrapUpItem' which as name predict is just a wrapper class which contain additional properties relevant to an searched Item. These properties can be more or less depend upon your business context / requirements. I am not suggesting that above is the best approach of doing it, i am just saying that it is one of the approach available while you do search in Sitecore.
Further below are the initiator functions(How to use above class) for each of the index from which you want to make search, these function can sit within .NET User control or wherever you like.

//This Property store the searched keyword in session you may store wherever you like
public string SearchText
        {
            get
            {
                return ValidationHelper.ValidateToString(Session[Portal.EShopServices.Constants.Sessions.SearchText], "");
            }
            set
            {
                Session[Portal.EShopServices.Constants.Sessions.SearchText] = value;
            }
        }

protected List<Portal.EShopServices.Search.Searcher.WrapUpItem> getProductSearch()
        {
            List<Portal.EShopServices.Search.Searcher.WrapUpItem> liItems = null;
            List<Item> _liSitecoreItems = null;
            List<SkinnyItem> ski = null;
            Portal.EShopServices.Search.Searcher.Searcher.Index = Portal.AppServices.Constants.UtilitySettings.ProductIndexName;
            Portal.EShopServices.Search.Searcher.Searcher.queryOccurence = Sitecore.Search.QueryOccurance.Must;
            //We can concatenate the GUID of all templates required using | sign and get from some constant value
            string templateIDs = Portal.EShopServices.Constants.TemplateID.AllProducts;
           //We can store the location of products stored under product catalog in app settings etc.,
            string locationId = Sitecore.Context.Database.GetItem(Portal.AppServices.Constants.UtilitySettings.eShop).ID.ToString();
            // We are using Advance Database crawler functions need to get them referenced
            ski = Portal.EShopServices.Search.Searcher.MySearcher.GetItems(SearchText, locationId, templateIDs, "");
            //Get Item List collection from the skinny collection
            if (ski != null && ski.Count > 0)
            {
                _liSitecoreItems = SearchHelper.GetItemListFromInformationCollection(ski);
            }
            //Clean the non product Items
            if (_liSitecoreItems != null && _liSitecoreItems.Count > 0)
            {
                liItems = Portal.EShopServices.Search.Searcher.MySearcher.HandleProductItems(_liSitecoreItems);
            }
            return liItems;
        }

        protected List<Portal.EShopServices.Search.Searcher.WrapUpItem> getMySearchItem(string locationId)
        {
            List<Portal.EShopServices.Search.Searcher.WrapUpItem> liItems = null;
            List<Item> _liSitecoreItems = null;
            List<SkinnyItem> ski = null;
            Portal.EShopServices.Search.Searcher.MySearcher.Index = Portal.AppServices.Constants.UtilitySettings.ContentIndexName;
            Portal.EShopServices.Search.Searcher.MySearcher.queryOccurence = Sitecore.Search.QueryOccurance.Must;
            if (locationId.Length > 0)
            {
                ski = Portal.EShopServices.Search.Searcher.Searcher.GetItems(SearchText, locationId);
            }
            else
            {
                ski = Portal.EShopServices.Search.Searcher.Searcher.GetItems(SearchText);
            }
            //Get Item List collection from the skinny collection
            if (ski != null && ski.Count > 0)
            {
                _liSitecoreItems = SearchHelper.GetItemListFromInformationCollection(ski);
            }
            //Clean the non-browsable items
            if (_liSitecoreItems != null && _liSitecoreItems.Count > 0)
            {
                liItems = Portal.EShopServices.Search.Searcher.MySearcher.HandleNonPagesItem(_liSitecoreItems);
            }
            return liItems;
        }

        protected List<Portal.EShopServices.Search.Searcher.WrapUpItem> getFileSearch()
        {

            List<Portal.EShopServices.Search.Searcher.WrapUpItem> liItems = null;
            List<Item> _liSitecoreItems = null;
            List<SkinnyItem> ski = null;
            Portal.EShopServices.Search.Searcher.MySearcher.Index = Portal.AppServices.Constants.UtilitySettings.FileIndexName;
            Portal.EShopServices.Search.Searcher.MySearcher.queryOccurence = Sitecore.Search.QueryOccurance.Must;
            ski = Portal.EShopServices.Search.Searcher.MySearcher.GetItems(SearchText);
            //Get Item List collection from the skinny collection
            if (ski != null && ski.Count > 0)
            {
                _liSitecoreItems = SearchHelper.GetItemListFromInformationCollection(ski);
            }
            //Clean the non-media items
            if (_liSitecoreItems != null && _liSitecoreItems.Count > 0)
            {
                liItems = Portal.EShopServices.Search.Searcher.MySearcher.HandleMediaItems(_liSitecoreItems);
            }
            return liItems;
        }

I hope above article is able to complete the picture of how to search from the custom indexes created using advance database crawler. ADC(advance database crawler) itself gives few function from which you can directly search but i prefer to create my own single class which can use ADC and become centralized approach for querying any index.

In next article we will see how to provide the auto update feature in the text box where keyword should come real from our indexed data, that would be final article in this series. Please let me know feedback so far in comments.

Search with Sitecore - Article - 3 - Crawler to Crawl Media File like PDF, Document

Dear all friends, this is third article in this series and following are the earlier articles for reference :

1) Search with Sitecore - Article-1 - Introduction 
2) Search with Sitecore - Article - 2 - Configure Advance Database Crawler and More
3) Search with Sitecore - Article - 3 - Crawler to Crawl Media File like PDF, Document  
4) Search with Sitecore - Article - 4 - Create Search API
5) Search with Sitecore - Article - 5 - Auto Complete For Search

For the same series, today we will see how i write my own custom crawler and configured it to able to crawl the PDF Files, Word Documents files etc.,

There are two parts where we need write lines in our Sitecore Solution, first is to create a class which has the capability to crawl the Sitecore Media Items. Another is to create a new Index lets name it as 'Documents' and configure this class into that index so that when you build the Indexes next time using Sitecore Index Building Wizard you can see your newly created Index in list in wizard and able to build it.
Following is the typical syntax of such class, you may name your class anything.

using Lucene.Net.Documents;
using Sitecore;
using Sitecore.Search;
using Sitecore.Search.Crawlers;
using System.Diagnostics;
using System.IO;
using System.Text;
using iTextSharp.text.pdf;
using iTextSharp.text.pdf.parser;
using Sitecore.Data.Items;
using System;
using Portal.AppServices.Utilities;
using System.Xml;
using DocumentFormat;
using System.Windows;

using DocumentFormat.OpenXml.Packaging;
public class FileCrawler : BaseCrawler, ICrawler
    {
        public string Root { get; set; }
        public string Database { get; set; }
        public float Boost { get; set; }
        long _totalProcessedSize;
        int _fileCount;
        int _successCount;
        int _failureCount;
        void ICrawler.Add(IndexUpdateContext context)
        {
        }
   }

In above class as you may note there are few default properties which specify key values like default Database, Root and Boost, item which set automatically based on the configuration value you specifiy. Rest other properties are use for run-time value which helps in logging and knowing the number of result. 

First let's assume that our class is ready, then next probable step will be to configure the new Index. You may configure the new Index in the Web.Config file itself or you may choose to create new index file under folder path "~\App_Config\Include\Shared\" with any name you like for example "SearchSettings.config". This config file will be automatically read by the Sitecore. The syntax of file should be as followed.

<configuration xmlns:x="http://www.sitecore.net/xmlconfig/">
  <sitecore>
     <search>

        <configuration>
           <indexes>
              <index id="Documents" type="Sitecore.Search.Index, Sitecore.Kernel">
              <param desc="name">$(id)</param>
              <param desc="folder">Documents</param>
              <Analyzer ref="search/analyzer" />
              <locations hint="list:AddCrawler">
                 <web type="YourNameSpace.FileCrawler,YourAssemblyName">
                    <Database>web</Database>
                    <Root>/sitecore/media library</Root>
                    <Tags>documents</Tags>
                    <Boost>4.0</Boost>
                 </web>
              </locations>
            </index>
           </indexes>
      </configuration>
    </search>
  </sitecore>
</configuration>

If you have configured the above index successfully in config file and your syntax class is in correct place then you may build your solution and check the Sitecore Index Wizard to see does your newly created index appeared. Put your Visual Studio Solution in debug mode and put the debug pointer to "void ICrawler.Add(IndexUpdateContext context" and finally run the wizard, you will be able to see that your debug point is hitting and you may get the Item object inside the 'context' parameter.

How that happens ?
-- As you notice in the configuration above, we tell sitecore to read the items for index building from 'web' Database and use the '/sitecore/media library', rest all settings you may look later and each has its own significance which can customize the Index building approach. So one thing is clear we are reading media files within Sitecore Media Library. You may have other requirement where you may need to parse file stored in file system or some relational database. In such case you need to modify the above configuration file and still write the same way Crawler class.

So till now we have done only our preparatory work which give us code place where we can write our appropriate code and Sitecore is able to call our code. That's great !! because now we know where to write our code and once you know that you can do anything.

First basic, Lucene understands the textual data and we need to feed that for each file into each Lucene Document  for each separate media file with supported attributes we may have. But we have files like PDF and Word Documents which are not simple text files. So we need the capability to parse them, to do so we have rely on third parties and there are various available in market. I will not get into debate what is best among them as it depends upon project cost and performance requirements etc.,

So for parsing PDF files i am using "iText PDF" and to read Word Document files "Open XML SDK 2.0 for Microsoft Office". I know few limitation with above two third parties which are PDF files with images can't be parsed and Office Document which doesn't follow Open XML specification will not be able to parse (Prior to MS Office 2007).

Following is the full code for the class, please remember to add reference to third party and lucene .dll files within your library.

using Lucene.Net.Documents;
using Sitecore;
using Sitecore.Search;
using Sitecore.Search.Crawlers;
using System.Diagnostics;
using System.IO;
using System.Text;
using iTextSharp.text.pdf;
using iTextSharp.text.pdf.parser;
using Sitecore.Data.Items;
using System;
using System.Xml;
using DocumentFormat;
using System.Windows;
using DocumentFormat.OpenXml.Packaging;

namespace YourNameSpace.Crawler
{
    public class FileCrawler : BaseCrawler, ICrawler
    {
        public string Root { get; set; }
        public string Database { get; set; }
        public float Boost { get; set; }
        long _totalProcessedSize;
        int _fileCount;
        int _successCount;
        int _failureCount;

        void ICrawler.Add(IndexUpdateContext context)
        {
            _fileCount = 0;
            _totalProcessedSize = 0;
            _successCount = 0;
            _failureCount = 0;
            //Stopwatch watch = Stopwatch.StartNew();
            ParseMediaLibraryFiles(context);
            //watch.Stop();
            //Log.Info(string.Format("Finished parsing files -- Total files:{0}(Errors:{1}-Success:{2}) -- {3}m:{4}s:{5}ms -- Total bytes {6}", _fileCount, _failureCount, _successCount, watch.Elapsed.Minutes, watch.Elapsed.Seconds,
            //watch.Elapsed.Milliseconds, _totalProcessedSize), this);
        }

        private void ParseMediaLibraryFiles(IndexUpdateContext context)
        {
            if (Root != null && Root.Length > 0)
            {
                Sitecore.Data.Database db = Sitecore.Data.Database.GetDatabase(Database);
                if (db != null)
                {
                    this.AddRecursive(db.GetItem(Root), context);
                }
            }
        }

        private void AddRecursive(Item itm, IndexUpdateContext context)
        {
            if (itm != null && itm.HasChildren)
            {
                foreach (Item singleItm in itm.Children.InnerChildren)
                {
                    //Specify the Media Folder Template ID '{FE5DD826-48C6-436D-B87A-7C4210C7413B}'
                    if (singleItm.Template.ID.ToString().Equals("{FE5DD826-48C6-436D-B87A-7C4210C7413B}", StringComparison.InvariantCultureIgnoreCase))
                    {
                        this.AddRecursive(singleItm, context);
                    }
                    else
                    {
                        var document = new Document();
                        document = AddFileContent(document, singleItm, singleItm.ID.ToString());
                        if (document != null)
                            context.AddDocument(document);
                    }
                }
            }
        }

        void ICrawler.Initialize(Index index)
        {

            //Log.Info("File Crawler Init", this);

        }

        bool IsIndexableFile(MediaItem media)
        {
            bool result = false;
            if (media != null && (media.Extension.Contains("pdf") || media.Extension.Contains("xml") || media.Extension.Contains("txt") || media.Extension.Contains("docx")))
            {
                result = true;
            }
            return result;
        }

        private Document AddFileContent(Document document, MediaItem media, string itemID)
        {
            _fileCount++;
            try
            {
                if (IsIndexableFile(media) && document != null)
                {
                    _totalProcessedSize += media.Size;
                    if (media.GetMediaStream().CanRead)
                    {
                        if (media.Extension.Equals("pdf"))
                        {
                            document.Add(this.CreateTextField(BuiltinFields.Content, this.ParsePDF(media.GetMediaStream(), media.Name)));
                        }
                        else if (media.Extension.Equals("xml"))
                        {
                            this.AddXmlContent(document, media.GetMediaStream());
                        }
                        else if (media.Extension.Equals("txt"))
                        {
                            this.AddTextContent(document, media.GetMediaStream());
                        }
                        else if (media.Extension.Equals("docx"))
                        {
                            document.Add(this.CreateTextField(BuiltinFields.Content, this.AddWordContent(media.GetMediaStream())));
                        }

                        document.Add(this.CreateTextField(BuiltinFields.Name, ValidationHelper.ValidateToString(media.Name, "")));
                        document.Add(this.CreateDataField(BuiltinFields.Icon, ValidationHelper.ValidateToString(media.Icon, "")));
                        document.Add(this.CreateTextField(BuiltinFields.Tags, ValidationHelper.ValidateToString(media.Alt, "")));
                        document.Add(this.CreateValueField(BuiltinFields.Template, ValidationHelper.ValidateToString(media.InnerItem.Template.ID, "")));
                        document.Add(this.CreateValueField(BuiltinFields.TemplateName, ValidationHelper.ValidateToString(media.InnerItem.TemplateName, "")));
                        document.Add(this.CreateDataField(BuiltinFields.Tags, itemID, 5.0f));
                        document.Add(this.CreateTextField(BuiltinFields.Database, this.Database));
                        document.Add(this.CreateTextField(BuiltinFields.Language, Sitecore.Context.Language.ToString()));
                        _successCount++;
                    }
                }
            }
            catch
            {
                _failureCount++;
                document = null;
            }
            return document;
        }

        protected string AddWordContent(Stream mediaStream)
        {
            const string wordmlNamespace = "http://schemas.openxmlformats.org/wordprocessingml/2006/main";
            StringBuilder textBuilder = new StringBuilder();
            try
            {
                using (WordprocessingDocument wdDoc = WordprocessingDocument.Open(mediaStream, false))
                {
                    // Manage namespaces to perform XPath queries.  
                    NameTable nt = new NameTable();
                    XmlNamespaceManager nsManager = new XmlNamespaceManager(nt);
                    nsManager.AddNamespace("w", wordmlNamespace);

                    // Get the document part from the package.  
                    // Load the XML in the document part into an XmlDocument instance.  
                    XmlDocument xdoc = new XmlDocument(nt);
                    xdoc.Load(wdDoc.MainDocumentPart.GetStream());

                    XmlNodeList paragraphNodes = xdoc.SelectNodes("//w:p", nsManager);
                    if (paragraphNodes != null)
                    {
                        foreach (XmlNode paragraphNode in paragraphNodes)
                        {
                            XmlNodeList textNodes = paragraphNode.SelectNodes(".//w:t", nsManager);
                            if (textNodes != null)
                            {
                                foreach (XmlNode textNode in textNodes)
                                    textBuilder.Append(textNode.InnerText);
                            }
                            textBuilder.Append(Environment.NewLine);
                        }
                    }
                }
            }
            catch (Exception)
            {
                _failureCount++;
            }
            finally
            {
                mediaStream.Close();
            }
            return textBuilder.ToString();
        }

        protected void AddTextContent(Document document, Stream mediaStream)
        {
            try
            {
                using (var reader = new StreamReader(mediaStream, Encoding.UTF8))
                {
                    document.Add(this.CreateTextField(BuiltinFields.Content, reader.ReadToEnd()));
                }
            }
            catch
            {
                _failureCount++;
            }
            finally
            {
                if (mediaStream != null)
                    mediaStream.Close();
            }
        }

        protected void AddXmlContent(Document document, Stream mediaStream)
        {
            XmlTextReader xreader = null;
            try
            {
                using (var reader = new StreamReader(mediaStream))
                {
                    xreader = new XmlTextReader(reader);
                    while (xreader.Read())
                    {
                        if (xreader.NodeType == XmlNodeType.Text ||
                        xreader.NodeType == XmlNodeType.Attribute ||
                        xreader.NodeType == XmlNodeType.CDATA ||
                        xreader.NodeType == XmlNodeType.Comment)
                        {
                            float boost = 1.0f;
                            if (xreader.Value.IndexOf("TODO",
                            StringComparison.InvariantCultureIgnoreCase) >= 0)
                            {
                                boost = 5.0f;
                            }
                            document.Add(this.CreateTextField(BuiltinFields.Content, xreader.Value, boost));
                        }
                    }
                }
            }
            catch (Exception)
            {
                _failureCount++;
            }
            finally
            {
                if (xreader != null)
                    xreader.Close();

                if (mediaStream != null)
                    mediaStream.Close();
            }
        }

        protected string ParsePDF(Stream mediaStream, string fileName)
        {
            var returnText = new StringBuilder();
            Stream stream = mediaStream;
            PdfReader pdfReader = null;
            try
            {
                if (stream != null)
                {
                    pdfReader = new PdfReader(new PdfReader(stream));
                    //loop through pages
                    for (int page = 1; page <= pdfReader.NumberOfPages; page++)
                    {
                        returnText.Append(PdfTextExtractor.GetTextFromPage(pdfReader, page));
                        pdfReader.Close();
                    }
                }
            }
            catch (Exception)
            {
                _failureCount++;
            }
            finally
            {
                if (pdfReader != null)
                    pdfReader.Close();
                if (stream != null)
                    stream.Close();
            }
            return returnText.ToString();
        }
    }
}

Once you successfully build the project then you can just trigger the Index building wizard and once that wizard is successful then you may see your new folder named 'Documents' under Index directory.

Figure 3.1

Summary: In this article we have solved the issue to have indexes out of the Sitecore Media Files which are PDF and Word Documents. This make us one step closer to our end goal of having enterprise search. Please let me know if any further queries you may have in comments.
In next article I will write about how to use Sitecore and lucene API to able to search before that will create few more indexes which are targeted toward certain nodes in Sitecore like product catalog.