Provided by: libkinosearch1-perl_1.00-1build3_amd64 bug

NAME

       KinoSearch1::Docs::Tutorial - sample indexing and search applications

DESCRIPTION

       The following sample code for invindexer.plx and search.cgi can be used to create a simple
       search engine. It requires the html presentation of the US Constitution included in the
       distribution for KinoSearch1, under "t/us_constitution".

       Note that a proper indexer for html documents would not rely on quick-n-dirty regular
       expressions for stripping tags, as this one does for the sake of brevity -- it would use a
       dedicated parsing module such as HTML::Parser.

   invindexer.plx
           #!/usr/bin/perl
           use strict;
           use warnings;

           use File::Spec;
           use KinoSearch1::InvIndexer;
           use KinoSearch1::Analysis::PolyAnalyzer;

           ### In order for invindexer.plx to work correctly, you must modify
           ### $source_dir, $path_to_invindex, and possibly $base_url.
           ###
           ### $source_dir must lead to the directory containing the US
           ### Constitution html files.
           ###
           ### $path_to_invindex is the future location of the invindex.
           ###
           ### $base_url should reflect the location of the us_constitution directory
           ### when accessed via a web browser.
           my $source_dir       = '';
           my $path_to_invindex = '';
           my $base_url         = '/us_constitution';

           opendir( my $source_dh, $source_dir )
               or die "Couldn't opendir '$source_dir': $!";
           my @filenames = grep {/\.html/} readdir $source_dh;
           closedir $source_dh or die "Couldn't closedir '$source_dir': $!";

           ### STEP 1: Choose an Analyzer.
           my $analyzer = KinoSearch1::Analysis::PolyAnalyzer->new(
               language => 'en',
           );

           ### STEP 2: Create a InvIndexer object.
           my $invindexer = KinoSearch1::InvIndexer->new(
               analyzer => $analyzer,
               invindex => $path_to_invindex,
               create   => 1,
           );

           ### STEP 3: Define fields.
           $invindexer->spec_field( name => 'title' );
           $invindexer->spec_field(
               name       => 'bodytext',
               vectorized => 1,
           );
           $invindexer->spec_field(
               name    => 'url',
               indexed => 0,
           );

           foreach my $filename (@filenames) {
               next if $filename eq 'index.html';
               my $filepath = File::Spec->catfile( $source_dir, $filename );
               open( my $fh, '<', $filepath )
                   or die "couldn't open file '$filepath': $!";
               my $content = do { local $/; <$fh> };

               ### STEP 4: Start a new document.
               my $doc = $invindexer->new_doc;

               $content =~ m#<title>(.*?)</title>#s
                   or die "couldn't isolate title in '$filepath'";
               my $title = $1;
               $content =~ m#<div id="bodytext">(.*?)</div><!--bodytext-->#s
                   or die "couldn't isolate bodytext in '$filepath'";
               my $bodytext = $1;
               $bodytext =~ s/<.*?>/ /gsm;    # quick and dirty tag stripping

               ### STEP 5: Set the value for each field.
               $doc->set_value( url      => "$base_url/$filename" );
               $doc->set_value( title    => $title );
               $doc->set_value( bodytext => $bodytext );

               ### STEP 6 Add the document to the invindex.
               $invindexer->add_doc($doc);

               ### STEP 7 Repeat steps 3-5 for each document in the collection.
           }

           ### STEP 8 Finalize the invindex.
           $invindexer->finish;

   search.cgi
           #!/usr/bin/perl -T
           use strict;
           use warnings;

           use CGI;
           use List::Util qw( max min );
           use POSIX qw( ceil );
           use KinoSearch1::Searcher;
           use KinoSearch1::Analysis::PolyAnalyzer;
           use KinoSearch1::Highlight::Highlighter;

           my $cgi           = CGI->new;
           my $q             = $cgi->param('q');
           my $offset        = $cgi->param('offset');
           my $hits_per_page = 10;
           $q      = '' unless defined $q;
           $offset = 0  unless defined $offset;

           ### In order for search.cgi to work, $path_to_invindex must be modified so
           ### that it points to the invindex created by invindexer.plx, and
           ### $base_url may have to change to reflect where a web-browser should
           ### look for the us_constitution directory.
           my $path_to_invindex = '';
           my $base_url         = '/us_constitution';

           ### STEP 1: Specify the same Analyzer used to create the invindex.
           my $analyzer = KinoSearch1::Analysis::PolyAnalyzer->new(
               language => 'en',
           );

           ### STEP 2: Create a Searcher object.
           my $searcher = KinoSearch1::Searcher->new(
               invindex => $path_to_invindex,
               analyzer => $analyzer,
           );

           ### STEP 3: Feed a query to the Search object.
           my $hits = $searcher->search($q);

           ### STEP 4: Arrange for highlighted excerpts to be created.
           my $highlighter = KinoSearch1::Highlight::Highlighter->new(
               excerpt_field => 'bodytext' );
           $hits->create_excerpts( highlighter => $highlighter );

           ### STEP 5: Process the search.
           $hits->seek( $offset, $hits_per_page );

           ### STEP 6: Format the results however you like.

           # create result list
           my $report = '';
           while ( my $hit = $hits->fetch_hit_hashref ) {
               my $score = sprintf( "%0.3f", $hit->{score} );
               $report .= qq|
                   <p>
                       <a href="$hit->{url}"><strong>$hit->{title}</strong></a>
                       <em>$score</em>
                       <br>
                       $hit->{excerpt}
                       <br>
                       <span class="excerptURL">$hit->{url}</span>
                   </p>
                   |;
           }

           $q = CGI::escapeHTML($q);

           # display info about the number of hits, paging links
           my $total_hits = $hits->total_hits;
           my $num_hits_info;
           if ( !length $q ) {
               # no query, no display
               $num_hits_info = '';
           }
           elsif ( $total_hits == 0 ) {
               # alert the user that their search failed
               $num_hits_info = qq|<p>No matches for <strong>$q</strong></p>|;
           }
           else {
               # calculate the nums for the first and last hit to display
               my $last_result = min( ( $offset + $hits_per_page ), $total_hits );
               my $first_result = min( ( $offset + 1 ), $last_result );

               # display the result nums, start paging info
               $num_hits_info = qq|
                   <p>
                       Results <strong>$first_result-$last_result</strong>
                       of <strong>$total_hits</strong> for <strong>$q</strong>.
                   </p>
                   <p>
                       Results Page:
                   |;

               # calculate first and last hits pages to display / link to
               my $current_page = int( $first_result / $hits_per_page ) + 1;
               my $last_page    = ceil( $total_hits / $hits_per_page );
               my $first_page   = max( 1, ( $current_page - 9 ) );
               $last_page = min( $last_page, ( $current_page + 10 ) );

               # create a url for use in paging links
               my $href = $cgi->url( -relative => 1 ) . "?" . $cgi->query_string;
               $href .= ";offset=0" unless $href =~ /offset=/;

               # generate the "Prev" link;
               if ( $current_page > 1 ) {
                   my $new_offset = ( $current_page - 2 ) * $hits_per_page;
                   $href =~ s/(?<=offset=)\d+/$new_offset/;
                   $num_hits_info .= qq|<a href="$href">&lt;= Prev</a>\n|;
               }

               # generate paging links
               for my $page_num ( $first_page .. $last_page ) {
                   if ( $page_num == $current_page ) {
                       $num_hits_info .= qq|$page_num \n|;
                   }
                   else {
                       my $new_offset = ( $page_num - 1 ) * $hits_per_page;
                       $href =~ s/(?<=offset=)\d+/$new_offset/;
                       $num_hits_info .= qq|<a href="$href">$page_num</a>\n|;
                   }
               }

               # generate the "Next" link
               if ( $current_page != $last_page ) {
                   my $new_offset = $current_page * $hits_per_page;
                   $href =~ s/(?<=offset=)\d+/$new_offset/;
                   $num_hits_info .= qq|<a href="$href">Next =&gt;</a>\n|;
               }

               # finish paging links
               $num_hits_info .= "</p>\n";
           }

           # blast it all out
           print "Content-type: text/html\n\n";
           print <<END_HTML;
           <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
               "http://www.w3.org/TR/html4/loose.dtd">
           <html>
           <head>
               <meta http-equiv="Content-type"
                   content="text/html;charset=ISO-8859-1">
               <link rel="stylesheet" type="text/css" href="$base_url/uscon.css">
               <title>KinoSearch: $q</title>
           </head>

           <body>

               <div id="navigation">
                   <form id="usconSearch" action="">
                       <strong>
                       Search the <a href="$base_url/index.html">US Constitution</a>:
                       </strong>
                       <input type="text" name="q" id="q" value="$q">
                       <input type="submit" value="=&gt;">
                       <input type="hidden" name="offset" value="0">
                   </form>
               </div><!--navigation-->

               <div id="bodytext">

               $report

               $num_hits_info

               <p style="font-size: smaller; color: #666">
                   <em>Powered by
                       <a href="http://www.rectangular.com/kinosearch/">
                           KinoSearch
                       </a>
                   </em>
               </p>
               </div><!--bodytext-->

           </body>

           </html>
           END_HTML

COPYRIGHT

       Copyright 2005-2010 Marvin Humphrey

LICENSE, DISCLAIMER, BUGS, etc.

       See KinoSearch1 version 1.00.