Lines 1-5
Link Here
|
1 |
/******************************************************************************* |
1 |
/******************************************************************************* |
2 |
* Copyright (c) 2000, 2009 IBM Corporation and others. |
2 |
* Copyright (c) 2000, 2010 IBM Corporation and others. |
3 |
* All rights reserved. This program and the accompanying materials |
3 |
* All rights reserved. This program and the accompanying materials |
4 |
* are made available under the terms of the Eclipse Public License v1.0 |
4 |
* are made available under the terms of the Eclipse Public License v1.0 |
5 |
* which accompanies this distribution, and is available at |
5 |
* which accompanies this distribution, and is available at |
Lines 46-52
Link Here
|
46 |
private int streamEnd; // used when writing data from the streamBuffer to the file |
46 |
private int streamEnd; // used when writing data from the streamBuffer to the file |
47 |
char separator = Index.DEFAULT_SEPARATOR; |
47 |
char separator = Index.DEFAULT_SEPARATOR; |
48 |
|
48 |
|
49 |
public static final String SIGNATURE= "INDEX VERSION 1.126"; //$NON-NLS-1$ |
49 |
public static final String SIGNATURE= "INDEX VERSION 1.127"; //$NON-NLS-1$ |
50 |
private static final char[] SIGNATURE_CHARS = SIGNATURE.toCharArray(); |
50 |
private static final char[] SIGNATURE_CHARS = SIGNATURE.toCharArray(); |
51 |
public static boolean DEBUG = false; |
51 |
public static boolean DEBUG = false; |
52 |
|
52 |
|
Lines 132-138
Link Here
|
132 |
} |
132 |
} |
133 |
return results; |
133 |
return results; |
134 |
} |
134 |
} |
135 |
private HashtableOfObject addQueryResult(HashtableOfObject results, char[] word, HashtableOfObject wordsToDocNumbers, MemoryIndex memoryIndex) throws IOException { |
135 |
private HashtableOfObject addQueryResult(HashtableOfObject results, char[] word, Object wordsToDocNumbers, MemoryIndex memoryIndex) throws IOException { |
136 |
// must skip over documents which have been added/changed/deleted in the memory index |
136 |
// must skip over documents which have been added/changed/deleted in the memory index |
137 |
if (results == null) |
137 |
if (results == null) |
138 |
results = new HashtableOfObject(13); |
138 |
results = new HashtableOfObject(13); |
Lines 141-151
Link Here
|
141 |
if (result == null) |
141 |
if (result == null) |
142 |
results.put(word, new EntryResult(word, wordsToDocNumbers)); |
142 |
results.put(word, new EntryResult(word, wordsToDocNumbers)); |
143 |
else |
143 |
else |
144 |
result.addDocumentTable(wordsToDocNumbers); |
144 |
result.addDocument(wordsToDocNumbers); |
145 |
} else { |
145 |
} else { |
146 |
SimpleLookupTable docsToRefs = memoryIndex.docsToReferences; |
146 |
SimpleLookupTable docsToRefs = memoryIndex.docsToReferences; |
147 |
if (result == null) result = new EntryResult(word, null); |
147 |
if (result == null) result = new EntryResult(word, null); |
148 |
int[] docNumbers = readDocumentNumbers(wordsToDocNumbers.get(word)); |
148 |
int[] docNumbers = readDocumentNumbers(wordsToDocNumbers); |
149 |
for (int i = 0, l = docNumbers.length; i < l; i++) { |
149 |
for (int i = 0, l = docNumbers.length; i < l; i++) { |
150 |
String docName = readDocumentName(docNumbers[i]); |
150 |
String docName = readDocumentName(docNumbers[i]); |
151 |
if (!docsToRefs.containsKey(docName)) |
151 |
if (!docsToRefs.containsKey(docName)) |
Lines 170-176
Link Here
|
170 |
results = new HashtableOfObject(wordsToDocNumbers.elementSize); |
170 |
results = new HashtableOfObject(wordsToDocNumbers.elementSize); |
171 |
for (int j = 0, m = words.length; j < m; j++) |
171 |
for (int j = 0, m = words.length; j < m; j++) |
172 |
if (words[j] != null) |
172 |
if (words[j] != null) |
173 |
results = addQueryResult(results, words[j], wordsToDocNumbers, memoryIndex); |
173 |
results = addQueryResult(results, words[j], wordsToDocNumbers.valueTable[j], memoryIndex); |
174 |
} |
174 |
} |
175 |
} |
175 |
} |
176 |
if (results != null && this.cachedChunks == null) |
176 |
if (results != null && this.cachedChunks == null) |
Lines 180-187
Link Here
|
180 |
case SearchPattern.R_EXACT_MATCH | SearchPattern.R_CASE_SENSITIVE: |
180 |
case SearchPattern.R_EXACT_MATCH | SearchPattern.R_CASE_SENSITIVE: |
181 |
for (int i = 0, l = categories.length; i < l; i++) { |
181 |
for (int i = 0, l = categories.length; i < l; i++) { |
182 |
HashtableOfObject wordsToDocNumbers = readCategoryTable(categories[i], false); |
182 |
HashtableOfObject wordsToDocNumbers = readCategoryTable(categories[i], false); |
183 |
if (wordsToDocNumbers != null && wordsToDocNumbers.containsKey(key)) |
183 |
Object value; |
184 |
results = addQueryResult(results, key, wordsToDocNumbers, memoryIndex); |
184 |
if (wordsToDocNumbers != null && ((value = wordsToDocNumbers.get(key)) != null)) |
|
|
185 |
results = addQueryResult(results, key, value, memoryIndex); |
185 |
} |
186 |
} |
186 |
break; |
187 |
break; |
187 |
case SearchPattern.R_PREFIX_MATCH | SearchPattern.R_CASE_SENSITIVE: |
188 |
case SearchPattern.R_PREFIX_MATCH | SearchPattern.R_CASE_SENSITIVE: |
Lines 192-198
Link Here
|
192 |
for (int j = 0, m = words.length; j < m; j++) { |
193 |
for (int j = 0, m = words.length; j < m; j++) { |
193 |
char[] word = words[j]; |
194 |
char[] word = words[j]; |
194 |
if (word != null && key[0] == word[0] && CharOperation.prefixEquals(key, word)) |
195 |
if (word != null && key[0] == word[0] && CharOperation.prefixEquals(key, word)) |
195 |
results = addQueryResult(results, word, wordsToDocNumbers, memoryIndex); |
196 |
results = addQueryResult(results, word, wordsToDocNumbers.valueTable[j], memoryIndex); |
196 |
} |
197 |
} |
197 |
} |
198 |
} |
198 |
} |
199 |
} |
Lines 205-211
Link Here
|
205 |
for (int j = 0, m = words.length; j < m; j++) { |
206 |
for (int j = 0, m = words.length; j < m; j++) { |
206 |
char[] word = words[j]; |
207 |
char[] word = words[j]; |
207 |
if (word != null && Index.isMatch(key, word, matchRule)) |
208 |
if (word != null && Index.isMatch(key, word, matchRule)) |
208 |
results = addQueryResult(results, word, wordsToDocNumbers, memoryIndex); |
209 |
results = addQueryResult(results, word, wordsToDocNumbers.valueTable[j], memoryIndex); |
209 |
} |
210 |
} |
210 |
} |
211 |
} |
211 |
} |
212 |
} |
Lines 613-618
Link Here
|
613 |
this.bufferIndex = 0; |
614 |
this.bufferIndex = 0; |
614 |
this.bufferEnd = stream.read(this.streamBuffer, 0, this.streamBuffer.length); |
615 |
this.bufferEnd = stream.read(this.streamBuffer, 0, this.streamBuffer.length); |
615 |
int size = readStreamInt(stream); |
616 |
int size = readStreamInt(stream); |
|
|
617 |
int hashtablesize = readStreamInt(stream); |
616 |
try { |
618 |
try { |
617 |
if (size < 0) { // DEBUG |
619 |
if (size < 0) { // DEBUG |
618 |
System.err.println("-------------------- DEBUG --------------------"); //$NON-NLS-1$ |
620 |
System.err.println("-------------------- DEBUG --------------------"); //$NON-NLS-1$ |
Lines 621-627
Link Here
|
621 |
System.err.println("size = "+size); //$NON-NLS-1$ |
623 |
System.err.println("size = "+size); //$NON-NLS-1$ |
622 |
System.err.println("-------------------- END --------------------"); //$NON-NLS-1$ |
624 |
System.err.println("-------------------- END --------------------"); //$NON-NLS-1$ |
623 |
} |
625 |
} |
624 |
categoryTable = new HashtableOfObject(size); |
626 |
categoryTable = new HashtableOfObject(size, hashtablesize); |
625 |
} catch (OutOfMemoryError oom) { |
627 |
} catch (OutOfMemoryError oom) { |
626 |
// DEBUG |
628 |
// DEBUG |
627 |
oom.printStackTrace(); |
629 |
oom.printStackTrace(); |
Lines 635-649
Link Here
|
635 |
int largeArraySize = 256; |
637 |
int largeArraySize = 256; |
636 |
for (int i = 0; i < size; i++) { |
638 |
for (int i = 0; i < size; i++) { |
637 |
char[] word = readStreamChars(stream); |
639 |
char[] word = readStreamChars(stream); |
|
|
640 |
if (word.length == 0) { |
641 |
// would have stored the number of empty slots than the real name |
642 |
int num = readStreamInt(stream); |
643 |
categoryTable.putEmptySlots(num); |
644 |
word = readStreamChars(stream); |
645 |
} |
638 |
int arrayOffset = readStreamInt(stream); |
646 |
int arrayOffset = readStreamInt(stream); |
639 |
// if arrayOffset is: |
647 |
// if arrayOffset is: |
640 |
// <= 0 then the array size == 1 with the value -> -arrayOffset |
648 |
// <= 0 then the array size == 1 with the value -> -arrayOffset |
641 |
// > 1 & < 256 then the size of the array is > 1 & < 256, the document array follows immediately |
649 |
// > 1 & < 256 then the size of the array is > 1 & < 256, the document array follows immediately |
642 |
// 256 if the array size >= 256 followed by another int which is the offset to the array (written prior to the table) |
650 |
// 256 if the array size >= 256 followed by another int which is the offset to the array (written prior to the table) |
643 |
if (arrayOffset <= 0) { |
651 |
if (arrayOffset <= 0) { |
644 |
categoryTable.put(word, new int[] {-arrayOffset}); // store 1 element array by negating documentNumber |
652 |
categoryTable.putInNextSlot(word, new int[] {-arrayOffset}); // store 1 element array by negating documentNumber |
645 |
} else if (arrayOffset < largeArraySize) { |
653 |
} else if (arrayOffset < largeArraySize) { |
646 |
categoryTable.put(word, readStreamDocumentArray(stream, arrayOffset)); // read in-lined array providing size |
654 |
categoryTable.putInNextSlot(word, readStreamDocumentArray(stream, arrayOffset)); // read in-lined array providing size |
647 |
} else { |
655 |
} else { |
648 |
arrayOffset = readStreamInt(stream); // read actual offset |
656 |
arrayOffset = readStreamInt(stream); // read actual offset |
649 |
if (readDocNumbers) { |
657 |
if (readDocNumbers) { |
Lines 653-659
Link Here
|
653 |
firstOffset = arrayOffset; |
661 |
firstOffset = arrayOffset; |
654 |
matchingWords[count++] = word; |
662 |
matchingWords[count++] = word; |
655 |
} |
663 |
} |
656 |
categoryTable.put(word, new Integer(arrayOffset)); // offset to array in the file |
664 |
categoryTable.putInNextSlot(word, new Integer(arrayOffset)); // offset to array in the file |
657 |
} |
665 |
} |
658 |
} |
666 |
} |
659 |
this.categoryTables.put(INTERNED_CATEGORY_NAMES.get(categoryName), categoryTable); |
667 |
this.categoryTables.put(INTERNED_CATEGORY_NAMES.get(categoryName), categoryTable); |
Lines 1060-1069
Link Here
|
1060 |
this.categoryOffsets.put(categoryName, this.streamEnd); // remember the offset to the start of the table |
1068 |
this.categoryOffsets.put(categoryName, this.streamEnd); // remember the offset to the start of the table |
1061 |
this.categoryTables.put(categoryName, null); // flush cached table |
1069 |
this.categoryTables.put(categoryName, null); // flush cached table |
1062 |
writeStreamInt(stream, wordsToDocs.elementSize); |
1070 |
writeStreamInt(stream, wordsToDocs.elementSize); |
|
|
1071 |
writeStreamInt(stream, wordsToDocs.valueTable.length); |
1063 |
char[][] words = wordsToDocs.keyTable; |
1072 |
char[][] words = wordsToDocs.keyTable; |
|
|
1073 |
int emptyCount = 0; |
1064 |
for (int i = 0, l = words.length; i < l; i++) { |
1074 |
for (int i = 0, l = words.length; i < l; i++) { |
1065 |
Object o = values[i]; |
1075 |
Object o = values[i]; |
1066 |
if (o != null) { |
1076 |
if (o != null) { |
|
|
1077 |
if (emptyCount != 0) { |
1078 |
writeStreamEmptySlot(stream, emptyCount); |
1079 |
emptyCount = 0; |
1080 |
} |
1067 |
writeStreamChars(stream, words[i]); |
1081 |
writeStreamChars(stream, words[i]); |
1068 |
if (o instanceof int[]) { |
1082 |
if (o instanceof int[]) { |
1069 |
int[] documentNumbers = (int[]) o; |
1083 |
int[] documentNumbers = (int[]) o; |
Lines 1075-1083
Link Here
|
1075 |
writeStreamInt(stream, largeArraySize); // mark to identify that an offset follows |
1089 |
writeStreamInt(stream, largeArraySize); // mark to identify that an offset follows |
1076 |
writeStreamInt(stream, ((Integer) o).intValue()); // offset in the file of the array of document numbers |
1090 |
writeStreamInt(stream, ((Integer) o).intValue()); // offset in the file of the array of document numbers |
1077 |
} |
1091 |
} |
|
|
1092 |
} else { |
1093 |
emptyCount++; |
1078 |
} |
1094 |
} |
1079 |
} |
1095 |
} |
1080 |
} |
1096 |
} |
|
|
1097 |
|
1081 |
private void writeDocumentNumbers(int[] documentNumbers, FileOutputStream stream) throws IOException { |
1098 |
private void writeDocumentNumbers(int[] documentNumbers, FileOutputStream stream) throws IOException { |
1082 |
// must store length as a positive int to detect in-lined array of 1 element |
1099 |
// must store length as a positive int to detect in-lined array of 1 element |
1083 |
int length = documentNumbers.length; |
1100 |
int length = documentNumbers.length; |
Lines 1259-1264
Link Here
|
1259 |
} |
1276 |
} |
1260 |
this.streamEnd += this.bufferIndex - oldIndex; |
1277 |
this.streamEnd += this.bufferIndex - oldIndex; |
1261 |
} |
1278 |
} |
|
|
1279 |
private void writeStreamEmptySlot(FileOutputStream stream, int num) throws IOException{ |
1280 |
if ((this.bufferIndex + 6) >= BUFFER_WRITE_SIZE) { |
1281 |
stream.write(this.streamBuffer, 0, this.bufferIndex); |
1282 |
this.bufferIndex = 0; |
1283 |
} |
1284 |
|
1285 |
this.streamBuffer[this.bufferIndex++] = 0; |
1286 |
this.streamBuffer[this.bufferIndex++] = 0; |
1287 |
this.streamBuffer[this.bufferIndex++] = (byte) (num >> 24); |
1288 |
this.streamBuffer[this.bufferIndex++] = (byte) (num >> 16); |
1289 |
this.streamBuffer[this.bufferIndex++] = (byte) (num >> 8); |
1290 |
this.streamBuffer[this.bufferIndex++] = (byte) num; |
1291 |
this.streamEnd += 6; |
1292 |
} |
1262 |
private void writeStreamInt(FileOutputStream stream, int val) throws IOException { |
1293 |
private void writeStreamInt(FileOutputStream stream, int val) throws IOException { |
1263 |
if ((this.bufferIndex + 4) >= BUFFER_WRITE_SIZE) { |
1294 |
if ((this.bufferIndex + 4) >= BUFFER_WRITE_SIZE) { |
1264 |
stream.write(this.streamBuffer, 0, this.bufferIndex); |
1295 |
stream.write(this.streamBuffer, 0, this.bufferIndex); |