[Koha-cvs] koha/misc/migration_tools rebuild_zebra.pl [rel_3_0]

[paul poulain]

CVSROOT:        /sources/koha
Module name:    koha
Branch:         rel_3_0
Changes by:     paul poulain <tipaul>   06/11/28 17:49:22

Modified files:
        misc/migration_tools: rebuild_zebra.pl 

Log message:
        adding zebra authorities database creation to this script.
        
        WARNING : it does not work yet. biblios worked in previous version, but 
could be harmed by this commit.
        
        I'll update ASAP

CVSWeb URLs:
http://cvs.savannah.gnu.org/viewcvs/koha/misc/migration_tools/rebuild_zebra.pl?cvsroot=koha&only_with_tag=rel_3_0&r1=1.4.2.3&r2=1.4.2.4

Patches:
Index: rebuild_zebra.pl
===================================================================
RCS file: /sources/koha/koha/misc/migration_tools/rebuild_zebra.pl,v
retrieving revision 1.4.2.3
retrieving revision 1.4.2.4
diff -u -b -r1.4.2.3 -r1.4.2.4
--- rebuild_zebra.pl    28 Nov 2006 15:15:35 -0000      1.4.2.3
+++ rebuild_zebra.pl    28 Nov 2006 17:49:22 -0000      1.4.2.4
@@ -3,6 +3,7 @@
 use C4::Context;
 use Getopt::Long;
 use C4::Biblio;
+use C4::AuthoritiesMarc;
 
 use strict;
 # 
@@ -13,20 +14,6 @@
 $|=1; # flushes output
 
 
-my $zebradir = C4::Context->zebraconfig('biblioserver')->{directory};
-my $kohadir = C4::Context->config('intranetdir');
-my $dbh = C4::Context->dbh;
-my ($biblionumbertagfield,$biblionumbertagsubfield) = 
&MARCfind_marc_from_kohafield($dbh,"biblio.biblionumber","");
-my ($biblioitemnumbertagfield,$biblioitemnumbertagsubfield) = 
&MARCfind_marc_from_kohafield($dbh,"biblioitems.biblioitemnumber","");
-
-print "some informations\n";
-print "=================\n";
-print "Zebra directory =>$zebradir\n";
-print "Koha directory =>$kohadir\n";
-print "BIBLIONUMBER in : $biblionumbertagfield\$$biblionumbertagsubfield\n";
-print "BIBLIOITEMNUMBER in : 
$biblioitemnumbertagfield\$$biblioitemnumbertagsubfield\n";
-print "=================\n";
-
 my $directory;
 my $skip_export;
 my $keep_export;
@@ -38,90 +25,23 @@
 
 $directory = "export" unless $directory;
 
-#
-# creating directory structure
-#
-my $created_dir_or_file = 0;
-print "====================\n";
-print "checking directories & files\n";
-print "====================\n";
-unless (-d "$zebradir") {
-    system("mkdir -p $zebradir");
-    print "Info: created $zebradir\n";
-    $created_dir_or_file++;
-}
-unless (-d "$zebradir/lock") {
-    mkdir "$zebradir/lock";
-    print "Info: created $zebradir/lock\n";
-    $created_dir_or_file++;
-}
-unless (-d "$zebradir/register") {
-    mkdir "$zebradir/register";
-    print "Info: created $zebradir/register\n";
-    $created_dir_or_file++;
-}
-unless (-d "$zebradir/shadow") {
-    mkdir "$zebradir/shadow";
-    print "Info: created $zebradir/shadow\n";
-    $created_dir_or_file++;
-}
-unless (-d "$zebradir/tab") {
-    mkdir "$zebradir/tab";
-    print "Info: created $zebradir/tab\n";
-    $created_dir_or_file++;
-}
-unless (-d "$zebradir/key") {
-    mkdir "$zebradir/key";
-    print "Info: created $zebradir/key\n";
-    $created_dir_or_file++;
-}
 
-unless (-d "$zebradir/etc") {
-    mkdir "$zebradir/etc";
-    print "Info: created $zebradir/etc\n";
-    $created_dir_or_file++;
-}
+my $biblioserverdir = C4::Context->zebraconfig('biblioserver')->{directory};
+my $authorityserverdir = 
C4::Context->zebraconfig('authorityserver')->{directory};
 
-#
-# copying mandatory files.
-#
-# the record model, depending on marc flavour
-unless (-f "$zebradir/tab/record.abs") {
-    if (C4::Context->preference("marcflavour") eq "UNIMARC") {
-        system("cp -f $kohadir/misc/zebra/record_biblios_unimarc.abs 
$zebradir/tab/record.abs");
-        print "Info: copied record.abs for UNIMARC\n";
-    } else {
-        system("cp -f $kohadir/misc/zebra/record_biblios_usmarc.abs 
$zebradir/tab/record.abs");
-        print "Info: copied record.abs for USMARC\n";
-    }
-    $created_dir_or_file++;
-}
-unless (-f "$zebradir/tab/sort-string-utf_french.chr") {
-    system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr 
$zebradir/tab/sort-string-utf.chr");
-    print "Info: copied sort-string-utf.chr\n";
-    $created_dir_or_file++;
-}
-unless (-f "$zebradir/tab/word-phrase-utf.chr") {
-    system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr 
$zebradir/tab/word-phrase-utf.chr");
-    print "Info: copied word-phase-utf.chr\n";
-    $created_dir_or_file++;
-}
-unless (-f "$zebradir/tab/bib1.att") {
-    system("cp -f $kohadir/misc/zebra/bib1_biblios.att 
$zebradir/tab/bib1.att");
-    print "Info: copied bib1.att\n";
-    $created_dir_or_file++;
-}
-unless (-f "$zebradir/etc/ccl.properties") {
-    system("cp -f $kohadir/misc/zebra/ccl.properties 
$zebradir/etc/ccl.properties");
-    print "Info: copied ccl.properties\n";
-    $created_dir_or_file++;
-}
-unless (-f "$zebradir/etc/pqf.properties") {
-    system("cp -f $kohadir/misc/zebra/pqf.properties 
$zebradir/etc/pqf.properties");
-    print "Info: copied pqf.properties\n";
-    $created_dir_or_file++;
-}
+my $kohadir = C4::Context->config('intranetdir');
+my $dbh = C4::Context->dbh;
+my ($biblionumbertagfield,$biblionumbertagsubfield) = 
&MARCfind_marc_from_kohafield($dbh,"biblio.biblionumber","");
+my ($biblioitemnumbertagfield,$biblioitemnumbertagsubfield) = 
&MARCfind_marc_from_kohafield($dbh,"biblioitems.biblioitemnumber","");
 
+print "some informations\n";
+print "=================\n";
+print "Zebra biblio directory =>$biblioserverdir\n";
+print "Zebra authorities directory =>$authorityserverdir\n";
+print "Koha directory =>$kohadir\n";
+print "BIBLIONUMBER in : $biblionumbertagfield\$$biblionumbertagsubfield\n";
+print "BIBLIOITEMNUMBER in : 
$biblioitemnumbertagfield\$$biblioitemnumbertagsubfield\n";
+print "=================\n";
 #
 # creating zebra-biblios.cfg depending on system
 #
@@ -171,7 +91,7 @@
     exit;
 }
 $modulesdir =~ s/\/modules\/.*//;
-warn "Info: zebra modules dir : $modulesdir";
+print "Info: zebra modules dir : $modulesdir\n";
 
 # getting tab directory
 my $tabdir;
@@ -195,14 +115,275 @@
     exit;
 }
 $tabdir =~ s/\/tab\/.*//;
-warn "Info: tab dir : $tabdir";
+print "Info: tab dir : $tabdir\n";
+
+#
+# AUTHORITIES creating directory structure
+#
+my $created_dir_or_file = 0;
+print "====================\n";
+print "checking directories & files\n";
+print "====================\n";
+unless (-d "$authorityserverdir") {
+    system("mkdir -p $authorityserverdir");
+    print "Info: created $authorityserverdir\n";
+    $created_dir_or_file++;
+}
+unless (-d "$authorityserverdir/lock") {
+    mkdir "$authorityserverdir/lock";
+    print "Info: created $authorityserverdir/lock\n";
+    $created_dir_or_file++;
+}
+unless (-d "$authorityserverdir/register") {
+    mkdir "$authorityserverdir/register";
+    print "Info: created $authorityserverdir/register\n";
+    $created_dir_or_file++;
+}
+unless (-d "$authorityserverdir/shadow") {
+    mkdir "$authorityserverdir/shadow";
+    print "Info: created $authorityserverdir/shadow\n";
+    $created_dir_or_file++;
+}
+unless (-d "$authorityserverdir/tab") {
+    mkdir "$authorityserverdir/tab";
+    print "Info: created $authorityserverdir/tab\n";
+    $created_dir_or_file++;
+}
+unless (-d "$authorityserverdir/key") {
+    mkdir "$authorityserverdir/key";
+    print "Info: created $authorityserverdir/key\n";
+    $created_dir_or_file++;
+}
+
+unless (-d "$authorityserverdir/etc") {
+    mkdir "$authorityserverdir/etc";
+    print "Info: created $authorityserverdir/etc\n";
+    $created_dir_or_file++;
+}
 
+#
+# AUTHORITIES : copying mandatory files
+#
+# the record model, depending on marc flavour
+unless (-f "$authorityserverdir/tab/record.abs") {
+    if (C4::Context->preference("marcflavour") eq "UNIMARC") {
+        system("cp -f $kohadir/misc/zebra/record_authorities_unimarc.abs 
$authorityserverdir/tab/record.abs");
+        print "Info: copied record.abs for UNIMARC\n";
+    } else {
+        system("cp -f $kohadir/misc/zebra/record_authorities_usmarc.abs 
$authorityserverdir/tab/record.abs");
+        print "Info: copied record.abs for USMARC\n";
+    }
+    $created_dir_or_file++;
+}
+unless (-f "$authorityserverdir/tab/sort-string-utf_french.chr") {
+    system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr 
$authorityserverdir/tab/sort-string-utf.chr");
+    print "Info: copied sort-string-utf.chr\n";
+    $created_dir_or_file++;
+}
+unless (-f "$authorityserverdir/tab/word-phrase-utf.chr") {
+    system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr 
$authorityserverdir/tab/word-phrase-utf.chr");
+    print "Info: copied word-phase-utf.chr\n";
+    $created_dir_or_file++;
+}
+unless (-f "$authorityserverdir/tab/bib1.att") {
+    system("cp -f $kohadir/misc/zebra/bib1_authorities.att 
$authorityserverdir/tab/bib1.att");
+    print "Info: copied bib1.att\n";
+    $created_dir_or_file++;
+}
+unless (-f "$authorityserverdir/etc/ccl.properties") {
+    system("cp -f $kohadir/misc/zebra/ccl.properties 
$authorityserverdir/etc/ccl.properties");
+    print "Info: copied ccl.properties\n";
+    $created_dir_or_file++;
+}
+unless (-f "$authorityserverdir/etc/pqf.properties") {
+    system("cp -f $kohadir/misc/zebra/pqf.properties 
$authorityserverdir/etc/pqf.properties");
+    print "Info: copied pqf.properties\n";
+    $created_dir_or_file++;
+}
 
-unless (-f "$zebradir/etc/zebra-biblios.cfg") {
-open ZD,">$zebradir/etc/zebra-biblios.cfg";
+#
+# AUTHORITIES : copying mandatory files
+#
+unless (-f "$authorityserverdir/etc/zebra-authorities.cfg") {
+open ZD,">$authorityserverdir/etc/zebra-authorities.cfg";
+print ZD "
+# generated by KOHA/misc/migration_tools/rebuild_zebra.pl 
+profilePath:\${srcdir:-.}:$authorityserverdir/tab/:$tabdir/tab/:\${srcdir:-.}/tab/
+
+encoding: UTF-8
+# Files that describe the attribute sets supported.
+attset: bib1.att
+attset: explain.att
+attset:gils.att
+
+modulePath:$modulesdir/modules/
+systag sysno rank
+# Specify record type
+iso2709.recordType:grs.marcxml.record
+recordType:grs.xml
+recordId: (bib1,Local-Number)
+storeKeys:1
+storeData:1
+
+
+# Lock File Area
+lockDir: $authorityserverdir/lock
+perm.anonymous:r
+perm.kohaadmin:rw
+passw.kohalis
+shadow
+register: $authorityserverdir/register:4G
+shadow: $authorityserverdir/shadow:4G
+
+# Temp File area for result sets
+setTmpDir: $authorityserverdir/tmp
+
+# Temp File area for index program
+keyTmpDir: $authorityserverdir/key
+
+# Approx. Memory usage during indexing
+memMax: 40M
+rank:rank-1
+";
+    print "Info: creating zebra-authorities.cfg\n";
+    $created_dir_or_file++;
+}
+
+if ($created_dir_or_file) {
+    print "Info: created : $created_dir_or_file directories & files\n";
+} else {
+    print "Info: file & directories OK\n";
+}
+
+#
+# exporting authorities
+#
+if ($skip_export) {
+    print "====================\n";
+    print "SKIPPING authorities export\n";
+    print "====================\n";
+} else {
+    print "====================\n";
+    print "exporting authorities\n";
+    print "====================\n";
+    mkdir "$directory" unless (-d $directory);
+    mkdir "$directory/authorities" unless (-d "$directory/authorities");
+    open(OUT,">:utf8","$directory/authorities/authorities.iso2709") or die $!;
+    my $dbh=C4::Context->dbh;
+    my $sth;
+    $sth=$dbh->prepare("select authid from auth_header limit 100");
+    $sth->execute();
+    my $i=0;
+    while (my ($authid) = $sth->fetchrow) {
+        my $record = AUTHgetauthority($dbh,$authid);
+        print ".";
+        print "\r$i" unless ($i++ %100);
+        print OUT $record->as_usmarc();
+    }
+    close(OUT);
+}
+
+#
+# and reindexing everything
+#
+print "====================\n";
+print "REINDEXING zebra\n";
+print "====================\n";
+system("zebraidx -g iso2709 -c $authorityserverdir/etc/zebra-authorities.cfg 
-d authorities update $directory/authorities");
+system("zebraidx -g iso2709 -c $authorityserverdir/etc/zebra-authorities.cfg 
-d authorities commit");
+
+
+#################################################################################################################
+#                        BIBLIOS 
+#################################################################################################################
+
+#
+# BIBLIOS : creating directory structure
+#
+unless (-d "$biblioserverdir") {
+    system("mkdir -p $biblioserverdir");
+    print "Info: created $biblioserverdir\n";
+    $created_dir_or_file++;
+}
+unless (-d "$biblioserverdir/lock") {
+    mkdir "$biblioserverdir/lock";
+    print "Info: created $biblioserverdir/lock\n";
+    $created_dir_or_file++;
+}
+unless (-d "$biblioserverdir/register") {
+    mkdir "$biblioserverdir/register";
+    print "Info: created $biblioserverdir/register\n";
+    $created_dir_or_file++;
+}
+unless (-d "$biblioserverdir/shadow") {
+    mkdir "$biblioserverdir/shadow";
+    print "Info: created $biblioserverdir/shadow\n";
+    $created_dir_or_file++;
+}
+unless (-d "$biblioserverdir/tab") {
+    mkdir "$biblioserverdir/tab";
+    print "Info: created $biblioserverdir/tab\n";
+    $created_dir_or_file++;
+}
+unless (-d "$biblioserverdir/key") {
+    mkdir "$biblioserverdir/key";
+    print "Info: created $biblioserverdir/key\n";
+    $created_dir_or_file++;
+}
+unless (-d "$biblioserverdir/etc") {
+    mkdir "$biblioserverdir/etc";
+    print "Info: created $biblioserverdir/etc\n";
+    $created_dir_or_file++;
+}
+
+#
+# BIBLIOS : copying mandatory files
+#
+# the record model, depending on marc flavour
+unless (-f "$biblioserverdir/tab/record.abs") {
+    if (C4::Context->preference("marcflavour") eq "UNIMARC") {
+        system("cp -f $kohadir/misc/zebra/record_biblios_unimarc.abs 
$biblioserverdir/tab/record.abs");
+        print "Info: copied record.abs for UNIMARC\n";
+    } else {
+        system("cp -f $kohadir/misc/zebra/record_biblios_usmarc.abs 
$biblioserverdir/tab/record.abs");
+        print "Info: copied record.abs for USMARC\n";
+    }
+    $created_dir_or_file++;
+}
+unless (-f "$biblioserverdir/tab/sort-string-utf_french.chr") {
+    system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr 
$biblioserverdir/tab/sort-string-utf.chr");
+    print "Info: copied sort-string-utf.chr\n";
+    $created_dir_or_file++;
+}
+unless (-f "$biblioserverdir/tab/word-phrase-utf.chr") {
+    system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr 
$biblioserverdir/tab/word-phrase-utf.chr");
+    print "Info: copied word-phase-utf.chr\n";
+    $created_dir_or_file++;
+}
+unless (-f "$biblioserverdir/tab/bib1.att") {
+    system("cp -f $kohadir/misc/zebra/bib1_biblios.att 
$biblioserverdir/tab/bib1.att");
+    print "Info: copied bib1.att\n";
+    $created_dir_or_file++;
+}
+unless (-f "$biblioserverdir/etc/ccl.properties") {
+    system("cp -f $kohadir/misc/zebra/ccl.properties 
$biblioserverdir/etc/ccl.properties");
+    print "Info: copied ccl.properties\n";
+    $created_dir_or_file++;
+}
+unless (-f "$biblioserverdir/etc/pqf.properties") {
+    system("cp -f $kohadir/misc/zebra/pqf.properties 
$biblioserverdir/etc/pqf.properties");
+    print "Info: copied pqf.properties\n";
+    $created_dir_or_file++;
+}
+
+#
+# BIBLIOS : copying mandatory files
+#
+unless (-f "$biblioserverdir/etc/zebra-biblios.cfg") {
+open ZD,">$biblioserverdir/etc/zebra-biblios.cfg";
 print ZD "
 # generated by KOHA/misc/migrtion_tools/rebuild_zebra.pl 
-profilePath:\${srcdir:-.}:$zebradir/tab/:$tabdir/tab/:\${srcdir:-.}/tab/
+profilePath:\${srcdir:-.}:$biblioserverdir/tab/:$tabdir/tab/:\${srcdir:-.}/tab/
 
 encoding: UTF-8
 # Files that describe the attribute sets supported.
@@ -221,19 +402,19 @@
 
 
 # Lock File Area
-lockDir: $zebradir/lock
+lockDir: $biblioserverdir/lock
 perm.anonymous:r
 perm.kohaadmin:rw
 passw.kohalis
 shadow
-register: $zebradir/register:4G
-shadow: $zebradir/shadow:4G
+register: $biblioserverdir/register:4G
+shadow: $biblioserverdir/shadow:4G
 
 # Temp File area for result sets
-setTmpDir: $zebradir/tmp
+setTmpDir: $biblioserverdir/tmp
 
 # Temp File area for index program
-keyTmpDir: $zebradir/key
+keyTmpDir: $biblioserverdir/key
 
 # Approx. Memory usage during indexing
 memMax: 40M
@@ -265,7 +446,7 @@
     open(OUT,">:utf8","$directory/export") or die $!;
     my $dbh=C4::Context->dbh;
     my $sth;
-    $sth=$dbh->prepare("select biblionumber from biblioitems where 
biblionumber <200 order by biblionumber");
+    $sth=$dbh->prepare("select biblionumber from biblioitems order by 
biblionumber limit 100");
     $sth->execute();
     my $i=0;
     while (my ($biblionumber) = $sth->fetchrow) {
@@ -336,8 +517,8 @@
 print "====================\n";
 print "REINDEXING zebra\n";
 print "====================\n";
-system("zebraidx -g iso2709 -c $zebradir/etc/zebra-biblios.cfg -d biblios 
update $directory");
-system("zebraidx -g iso2709 -c $zebradir/etc/zebra-biblios.cfg -d biblios 
commit");
+system("zebraidx -g iso2709 -c $biblioserverdir/etc/zebra-biblios.cfg -d 
biblios update $directory");
+system("zebraidx -g iso2709 -c $biblioserverdir/etc/zebra-biblios.cfg -d 
biblios commit");
 
 print "====================\n";
 print "CLEANING\n";