Attachment #161407 for bug #289057




/*******************************************************************************
 * Copyright (c) 2000, 2010 IBM Corporation and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at

/**
 * Hashtable of {char[] --> Object }
 */
public class HashtableOfObject implements Cloneable {

	// to avoid using Enumerations, walk the individual tables skipping nulls
	public char[] keyTable[];
	public Object valueTable[];

	public int elementSize; // number of elements in the table
	protected int threshold;

	public HashtableOfObject() {
		this(13);


	public boolean containsKey(char[] key) {
		int length = this.keyTable.length,
			index = hashCode(key) % length;
		int keyLength = key.length;
		char[] currentKey;
		while ((currentKey = this.keyTable[index]) != null) {


	public Object get(char[] key) {
		int length = this.keyTable.length,
			index = hashCode(key) % length;
		int keyLength = key.length;
		char[] currentKey;
		while ((currentKey = this.keyTable[index]) != null) {

		return null;
	}

	/*
	 * This method implementation is an inline of the {@link CharOperation#hashCode(char[])} method...
	 */
	protected int hashCode(char[] key) {
		int length = key.length;
		int hash = length == 0 ? 31 : key[0];
		if (length < 8) {
			for (int i = length; --i > 0;)
				hash = (hash * 31) + key[i];
		} else {
			// 8 characters is enough to compute a decent hash code, don't waste time examining every character
			for (int i = length - 1, last = i > 16 ? i - 16 : 0; i > last; i -= 2)
				hash = (hash * 31) + key[i];
		}
		return hash & 0x7FFFFFFF;
	}

	public Object put(char[] key, Object value) {
		int length = this.keyTable.length,
			index = hashCode(key) % length;
		int keyLength = key.length;
		char[] currentKey;
		while ((currentKey = this.keyTable[index]) != null) {

		return value;
	}

	/**
	 * Put a value at the index of the given using the local hash code computation.
	 * <p>
	 * Note that this is an unsafe put as there's no prior verification whether
	 * the given key already exists in the table or not.
	 * </p>
	 * @param key The key of the table entry
	 * @param value The value of the table entry
	 */
	public void putUnsafely(char[] key, Object value) {
		int length = this.keyTable.length,
			index = hashCode(key) % length;
		while (this.keyTable[index] != null) {
			if (++index == length) {
				index = 0;
			}
		}
		this.keyTable[index] = key;
		this.valueTable[index] = value;
	
		// assumes the threshold is never equal to the size of the table
		if (++this.elementSize > this.threshold) {
			rehash();
		}
	}

	public Object removeKey(char[] key) {
		int length = this.keyTable.length,
			index = hashCode(key) % length;
		int keyLength = key.length;
		char[] currentKey;
		while ((currentKey = this.keyTable[index]) != null) {

		return null;
	}

	protected void rehash() {

		HashtableOfObject newHashtable = new HashtableOfObject(this.elementSize * 2);		// double the number of expected elements
		char[] currentKey;
		for (int i = this.keyTable.length; --i >= 0;)
			if ((currentKey = this.keyTable[i]) != null)
				newHashtable.putUnsafely(currentKey, this.valueTable[i]);

		this.keyTable = newHashtable.keyTable;
		this.valueTable = newHashtable.valueTable;




/*******************************************************************************
 * Copyright (c) 2000, 2010 IBM Corporation and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     IBM Corporation - initial API and implementation
 *******************************************************************************/
package org.eclipse.jdt.internal.core.index;

import org.eclipse.jdt.internal.compiler.util.HashtableOfObject;


/**
 * Hashtable of {char[] --> Object } typically used for the category table
 * stored in the {@link DiskIndex disk index}.
 * <p>
 * This is a more specific {@link HashtableOfObject hashtable of objects} where
 * the hash code is computed by using twice more characters than the hash
 * code computation of the {@link HashtableOfObject original hashtable}.
 * </p><p>
 * This is typically important for type declaration category table for which the
 * key postfix might have more than 16 identical characters...
 * </p>
 * @see "https://bugs.eclipse.org/bugs/show_bug.cgi?id=289057"
 */
public final class CategoryTable extends HashtableOfObject {

	public CategoryTable() {
		super();
	}
	public CategoryTable(int size) {
		super(size);
	}
	
	/* (non-Javadoc)
	 * 
	 * The hash code computation is done by using half of the 32 characters last
	 * characters of the given array instead of only half of the 16 last ones for the
	 * super implementation...
	 * 
	 * @see org.eclipse.jdt.internal.compiler.util.HashtableOfObject#hashCode(char[])
	 */
	protected int hashCode(char[] array) {
		int length = array.length;
		int hash = length == 0 ? 31 : array[0];
		if (length < 16) {
			for (int i = length; --i > 0;)
				hash = (hash * 31) + array[i];
		} else {
			// 16 characters is enough to compute a decent hash code, don't waste time examining every character
			for (int i = length - 1, last = i > 32 ? i - 32 : 0; i > last; i -= 2)
				hash = (hash * 31) + array[i];
			
		}
		return hash & 0x7FFFFFFF;
		
	}

	/* (non-Javadoc)
	 * @see org.eclipse.jdt.internal.compiler.util.HashtableOfObject#rehash()
	 */
	protected void rehash() {
		CategoryTable newHashtable = new CategoryTable(this.elementSize * 2);		// double the number of expected elements
		char[] currentKey;
		for (int i = this.keyTable.length; --i >= 0;)
			if ((currentKey = this.keyTable[i]) != null)
				newHashtable.putUnsafely(currentKey, this.valueTable[i]);
	
		this.keyTable = newHashtable.keyTable;
		this.valueTable = newHashtable.valueTable;
		this.threshold = newHashtable.threshold;
	}
}




/*******************************************************************************
 * Copyright (c) 2000, 2010 IBM Corporation and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at

	}
	return results;
}
private HashtableOfObject addQueryResult(HashtableOfObject results, char[] word, CategoryTable wordsToDocNumbers, MemoryIndex memoryIndex) throws IOException {
	// must skip over documents which have been added/changed/deleted in the memory index
	if (results == null)
		results = new HashtableOfObject(13);
	EntryResult result = (EntryResult) results.get(word);
	if (memoryIndex == null) {
		if (result == null)
			results.putUnsafely(word, new EntryResult(word, wordsToDocNumbers));
		else
			result.addDocumentTable(wordsToDocNumbers);
	} else {

	HashtableOfObject results = null; // initialized if needed
	if (key == null) {
		for (int i = 0, l = categories.length; i < l; i++) {
			CategoryTable wordsToDocNumbers = readCategoryTable(categories[i], true); // cache if key is null since its a definite match
			if (wordsToDocNumbers != null) {
				char[][] words = wordsToDocNumbers.keyTable;
				if (results == null)

		switch (matchRule) {
			case SearchPattern.R_EXACT_MATCH | SearchPattern.R_CASE_SENSITIVE:
				for (int i = 0, l = categories.length; i < l; i++) {
					CategoryTable wordsToDocNumbers = readCategoryTable(categories[i], false);
					if (wordsToDocNumbers != null && wordsToDocNumbers.containsKey(key))
						results = addQueryResult(results, key, wordsToDocNumbers, memoryIndex);
				}
				break;
			case SearchPattern.R_PREFIX_MATCH | SearchPattern.R_CASE_SENSITIVE:
				for (int i = 0, l = categories.length; i < l; i++) {
					CategoryTable wordsToDocNumbers = readCategoryTable(categories[i], false);
					if (wordsToDocNumbers != null) {
						char[][] words = wordsToDocNumbers.keyTable;
						for (int j = 0, m = words.length; j < m; j++) {

				break;
			default:
				for (int i = 0, l = categories.length; i < l; i++) {
					CategoryTable wordsToDocNumbers = readCategoryTable(categories[i], false);
					if (wordsToDocNumbers != null) {
						char[][] words = wordsToDocNumbers.keyTable;
						for (int j = 0, m = words.length; j < m; j++) {

		char[] categoryName = categoryNames[i];
		if (categoryName != null) {
			SimpleWordSet wordSet = (SimpleWordSet) wordSets[i];
			CategoryTable wordsToDocs = (CategoryTable) this.categoryTables.get(categoryName);
			if (wordsToDocs == null)
				this.categoryTables.put(categoryName, wordsToDocs = new CategoryTable(wordSet.elementSize));

			char[][] words = wordSet.words;
			for (int j = 0, m = words.length; j < m; j++) {

				if (word != null) {
					Object o = wordsToDocs.get(word);
					if (o == null) {
						wordsToDocs.putUnsafely(word, new int[] {newPosition});
					} else if (o instanceof IntList) {
						((IntList) o).add(newPosition);
					} else {

	this.categoryTables = null;
}
private void mergeCategory(char[] categoryName, DiskIndex onDisk, int[] positions, FileOutputStream stream) throws IOException {
	CategoryTable wordsToDocs = (CategoryTable) this.categoryTables.get(categoryName);
	if (wordsToDocs == null)
		wordsToDocs = new CategoryTable(3);

	CategoryTable oldWordsToDocs = onDisk.readCategoryTable(categoryName, true);
	if (oldWordsToDocs != null) {
		char[][] oldWords = oldWordsToDocs.keyTable;
		Object[] oldArrayOffsets = oldWordsToDocs.valueTable;


				Object o = wordsToDocs.get(oldWord);
				if (o == null) {
					wordsToDocs.putUnsafely(oldWord, mappedNumbers);
				} else {
					IntList list = null;
					if (o instanceof IntList) {

		this.streamBuffer = null;
	}
}
private synchronized CategoryTable readCategoryTable(char[] categoryName, boolean readDocNumbers) throws IOException {
	// result will be null if categoryName is unknown
	int offset = this.categoryOffsets.get(categoryName);
	if (offset == HashtableOfIntValues.NO_VALUE) {

	if (this.categoryTables == null) {
		this.categoryTables = new HashtableOfObject(3);
	} else {
		CategoryTable cachedTable = (CategoryTable) this.categoryTables.get(categoryName);
		if (cachedTable != null) {
			if (readDocNumbers) { // must cache remaining document number arrays
				Object[] arrayOffsets = cachedTable.valueTable;

	}

	FileInputStream stream = new FileInputStream(this.indexFile);
	CategoryTable categoryTable = null;
	char[][] matchingWords = null;
	int count = 0;
	int firstOffset = -1;

				System.err.println("size = "+size); //$NON-NLS-1$
				System.err.println("--------------------   END   --------------------"); //$NON-NLS-1$
			}
			categoryTable = new CategoryTable(size);
		} catch (OutOfMemoryError oom) {
			// DEBUG
			oom.printStackTrace();

			//		> 1 & < 256 then the size of the array is > 1 & < 256, the document array follows immediately
			//		256 if the array size >= 256 followed by another int which is the offset to the array (written prior to the table)
			if (arrayOffset <= 0) {
				categoryTable.putUnsafely(word, new int[] {-arrayOffset}); // store 1 element array by negating documentNumber
			} else if (arrayOffset < largeArraySize) {
				categoryTable.putUnsafely(word, readStreamDocumentArray(stream, arrayOffset)); // read in-lined array providing size
			} else {
				arrayOffset = readStreamInt(stream); // read actual offset
				if (readDocNumbers) {

						firstOffset = arrayOffset;
					matchingWords[count++] = word;
				}
				categoryTable.putUnsafely(word, new Integer(arrayOffset)); // offset to array in the file
			}
		}
		this.categoryTables.put(INTERNED_CATEGORY_NAMES.get(categoryName), categoryTable);

	Object[] tables = this.categoryTables.valueTable;
	for (int i = 0, l = categoryNames.length; i < l; i++)
		if (categoryNames[i] != null)
			writeCategoryTable(categoryNames[i], (CategoryTable) tables[i], stream);
	this.categoryTables = null;
}
private void writeCategoryTable(char[] categoryName, CategoryTable wordsToDocs, FileOutputStream stream) throws IOException {
	// the format of a category table is as follows:
	// any document number arrays with >= 256 elements are written before the table (the offset to each array is remembered)
	// then the number of word->int[] pairs in the table is written

Return to bug 289057

Lines 1-5 Link Here

(-)compiler/org/eclipse/jdt/internal/compiler/util/HashtableOfObject.java (-9 / +52 lines)
1	/*******************************************************************************	1	/*******************************************************************************
2	* Copyright (c) 2000, 2009 IBM Corporation and others.	2	* Copyright (c) 2000, 2010 IBM Corporation and others.
3	* All rights reserved. This program and the accompanying materials	3	* All rights reserved. This program and the accompanying materials
4	* are made available under the terms of the Eclipse Public License v1.0	4	* are made available under the terms of the Eclipse Public License v1.0
5	* which accompanies this distribution, and is available at	5	* which accompanies this distribution, and is available at
Lines 15-28 Link Here
15	/**	15	/**
16	* Hashtable of {char[] --> Object }	16	* Hashtable of {char[] --> Object }
17	*/	17	*/
18	public final class HashtableOfObject implements Cloneable {	18	public class HashtableOfObject implements Cloneable {
19		19
20	// to avoid using Enumerations, walk the individual tables skipping nulls	20	// to avoid using Enumerations, walk the individual tables skipping nulls
21	public char[] keyTable[];	21	public char[] keyTable[];
22	public Object valueTable[];	22	public Object valueTable[];
23		23
24	public int elementSize; // number of elements in the table	24	public int elementSize; // number of elements in the table
25	int threshold;	25	protected int threshold;
26		26
27	public HashtableOfObject() {	27	public HashtableOfObject() {
28	this(13);	28	this(13);
Lines 64-70 Link Here
64		64
65	public boolean containsKey(char[] key) {	65	public boolean containsKey(char[] key) {
66	int length = this.keyTable.length,	66	int length = this.keyTable.length,
67	index = CharOperation.hashCode(key) % length;	67	index = hashCode(key) % length;
68	int keyLength = key.length;	68	int keyLength = key.length;
69	char[] currentKey;	69	char[] currentKey;
70	while ((currentKey = this.keyTable[index]) != null) {	70	while ((currentKey = this.keyTable[index]) != null) {
Lines 79-85 Link Here
79		79
80	public Object get(char[] key) {	80	public Object get(char[] key) {
81	int length = this.keyTable.length,	81	int length = this.keyTable.length,
82	index = CharOperation.hashCode(key) % length;	82	index = hashCode(key) % length;
83	int keyLength = key.length;	83	int keyLength = key.length;
84	char[] currentKey;	84	char[] currentKey;
85	while ((currentKey = this.keyTable[index]) != null) {	85	while ((currentKey = this.keyTable[index]) != null) {
Lines 92-100 Link Here
92	return null;	92	return null;
93	}	93	}
94		94
		95	/*
		96	* This method implementation is an inline of the {@link CharOperation#hashCode(char[])} method...
		97	*/
		98	protected int hashCode(char[] key) {
		99	int length = key.length;
		100	int hash = length == 0 ? 31 : key[0];
		101	if (length < 8) {
		102	for (int i = length; --i > 0;)
		103	hash = (hash * 31) + key[i];
		104	} else {
		105	// 8 characters is enough to compute a decent hash code, don't waste time examining every character
		106	for (int i = length - 1, last = i > 16 ? i - 16 : 0; i > last; i -= 2)
		107	hash = (hash * 31) + key[i];
		108	}
		109	return hash & 0x7FFFFFFF;
		110	}
		111
95	public Object put(char[] key, Object value) {	112	public Object put(char[] key, Object value) {
96	int length = this.keyTable.length,	113	int length = this.keyTable.length,
97	index = CharOperation.hashCode(key) % length;	114	index = hashCode(key) % length;
98	int keyLength = key.length;	115	int keyLength = key.length;
99	char[] currentKey;	116	char[] currentKey;
100	while ((currentKey = this.keyTable[index]) != null) {	117	while ((currentKey = this.keyTable[index]) != null) {
Lines 113-121 Link Here
113	return value;	130	return value;
114	}	131	}
115		132
		133	/**
		134	* Put a value at the index of the given using the local hash code computation.
		135	* <p>
		136	* Note that this is an unsafe put as there's no prior verification whether
		137	* the given key already exists in the table or not.
		138	* </p>
		139	* @param key The key of the table entry
		140	* @param value The value of the table entry
		141	*/
		142	public void putUnsafely(char[] key, Object value) {
		143	int length = this.keyTable.length,
		144	index = hashCode(key) % length;
		145	while (this.keyTable[index] != null) {
		146	if (++index == length) {
		147	index = 0;
		148	}
		149	}
		150	this.keyTable[index] = key;
		151	this.valueTable[index] = value;
		152
		153	// assumes the threshold is never equal to the size of the table
		154	if (++this.elementSize > this.threshold) {
		155	rehash();
		156	}
		157	}
		158
116	public Object removeKey(char[] key) {	159	public Object removeKey(char[] key) {
117	int length = this.keyTable.length,	160	int length = this.keyTable.length,
118	index = CharOperation.hashCode(key) % length;	161	index = hashCode(key) % length;
119	int keyLength = key.length;	162	int keyLength = key.length;
120	char[] currentKey;	163	char[] currentKey;
121	while ((currentKey = this.keyTable[index]) != null) {	164	while ((currentKey = this.keyTable[index]) != null) {
Lines 134-146 Link Here
134	return null;	177	return null;
135	}	178	}
136		179
137	private void rehash() {	180	protected void rehash() {
138		181
139	HashtableOfObject newHashtable = new HashtableOfObject(this.elementSize * 2); // double the number of expected elements	182	HashtableOfObject newHashtable = new HashtableOfObject(this.elementSize * 2); // double the number of expected elements
140	char[] currentKey;	183	char[] currentKey;
141	for (int i = this.keyTable.length; --i >= 0;)	184	for (int i = this.keyTable.length; --i >= 0;)
142	if ((currentKey = this.keyTable[i]) != null)	185	if ((currentKey = this.keyTable[i]) != null)
143	newHashtable.put(currentKey, this.valueTable[i]);	186	newHashtable.putUnsafely(currentKey, this.valueTable[i]);
144		187
145	this.keyTable = newHashtable.keyTable;	188	this.keyTable = newHashtable.keyTable;
146	this.valueTable = newHashtable.valueTable;	189	this.valueTable = newHashtable.valueTable;

Lines 1-5 Link Here

(-)search/org/eclipse/jdt/internal/core/index/DiskIndex.java (-23 / +23 lines)
1	/*******************************************************************************	1	/*******************************************************************************
2	* Copyright (c) 2000, 2009 IBM Corporation and others.	2	* Copyright (c) 2000, 2010 IBM Corporation and others.
3	* All rights reserved. This program and the accompanying materials	3	* All rights reserved. This program and the accompanying materials
4	* are made available under the terms of the Eclipse Public License v1.0	4	* are made available under the terms of the Eclipse Public License v1.0
5	* which accompanies this distribution, and is available at	5	* which accompanies this distribution, and is available at
Lines 132-145 Link Here
132	}	132	}
133	return results;	133	return results;
134	}	134	}
135	private HashtableOfObject addQueryResult(HashtableOfObject results, char[] word, HashtableOfObject wordsToDocNumbers, MemoryIndex memoryIndex) throws IOException {	135	private HashtableOfObject addQueryResult(HashtableOfObject results, char[] word, CategoryTable wordsToDocNumbers, MemoryIndex memoryIndex) throws IOException {
136	// must skip over documents which have been added/changed/deleted in the memory index	136	// must skip over documents which have been added/changed/deleted in the memory index
137	if (results == null)	137	if (results == null)
138	results = new HashtableOfObject(13);	138	results = new HashtableOfObject(13);
139	EntryResult result = (EntryResult) results.get(word);	139	EntryResult result = (EntryResult) results.get(word);
140	if (memoryIndex == null) {	140	if (memoryIndex == null) {
141	if (result == null)	141	if (result == null)
142	results.put(word, new EntryResult(word, wordsToDocNumbers));	142	results.putUnsafely(word, new EntryResult(word, wordsToDocNumbers));
143	else	143	else
144	result.addDocumentTable(wordsToDocNumbers);	144	result.addDocumentTable(wordsToDocNumbers);
145	} else {	145	} else {
Lines 163-169 Link Here
163	HashtableOfObject results = null; // initialized if needed	163	HashtableOfObject results = null; // initialized if needed
164	if (key == null) {	164	if (key == null) {
165	for (int i = 0, l = categories.length; i < l; i++) {	165	for (int i = 0, l = categories.length; i < l; i++) {
166	HashtableOfObject wordsToDocNumbers = readCategoryTable(categories[i], true); // cache if key is null since its a definite match	166	CategoryTable wordsToDocNumbers = readCategoryTable(categories[i], true); // cache if key is null since its a definite match
167	if (wordsToDocNumbers != null) {	167	if (wordsToDocNumbers != null) {
168	char[][] words = wordsToDocNumbers.keyTable;	168	char[][] words = wordsToDocNumbers.keyTable;
169	if (results == null)	169	if (results == null)
Lines 179-192 Link Here
179	switch (matchRule) {	179	switch (matchRule) {
180	case SearchPattern.R_EXACT_MATCH \| SearchPattern.R_CASE_SENSITIVE:	180	case SearchPattern.R_EXACT_MATCH \| SearchPattern.R_CASE_SENSITIVE:
181	for (int i = 0, l = categories.length; i < l; i++) {	181	for (int i = 0, l = categories.length; i < l; i++) {
182	HashtableOfObject wordsToDocNumbers = readCategoryTable(categories[i], false);	182	CategoryTable wordsToDocNumbers = readCategoryTable(categories[i], false);
183	if (wordsToDocNumbers != null && wordsToDocNumbers.containsKey(key))	183	if (wordsToDocNumbers != null && wordsToDocNumbers.containsKey(key))
184	results = addQueryResult(results, key, wordsToDocNumbers, memoryIndex);	184	results = addQueryResult(results, key, wordsToDocNumbers, memoryIndex);
185	}	185	}
186	break;	186	break;
187	case SearchPattern.R_PREFIX_MATCH \| SearchPattern.R_CASE_SENSITIVE:	187	case SearchPattern.R_PREFIX_MATCH \| SearchPattern.R_CASE_SENSITIVE:
188	for (int i = 0, l = categories.length; i < l; i++) {	188	for (int i = 0, l = categories.length; i < l; i++) {
189	HashtableOfObject wordsToDocNumbers = readCategoryTable(categories[i], false);	189	CategoryTable wordsToDocNumbers = readCategoryTable(categories[i], false);
190	if (wordsToDocNumbers != null) {	190	if (wordsToDocNumbers != null) {
191	char[][] words = wordsToDocNumbers.keyTable;	191	char[][] words = wordsToDocNumbers.keyTable;
192	for (int j = 0, m = words.length; j < m; j++) {	192	for (int j = 0, m = words.length; j < m; j++) {
Lines 199-205 Link Here
199	break;	199	break;
200	default:	200	default:
201	for (int i = 0, l = categories.length; i < l; i++) {	201	for (int i = 0, l = categories.length; i < l; i++) {
202	HashtableOfObject wordsToDocNumbers = readCategoryTable(categories[i], false);	202	CategoryTable wordsToDocNumbers = readCategoryTable(categories[i], false);
203	if (wordsToDocNumbers != null) {	203	if (wordsToDocNumbers != null) {
204	char[][] words = wordsToDocNumbers.keyTable;	204	char[][] words = wordsToDocNumbers.keyTable;
205	for (int j = 0, m = words.length; j < m; j++) {	205	for (int j = 0, m = words.length; j < m; j++) {
Lines 336-344 Link Here
336	char[] categoryName = categoryNames[i];	336	char[] categoryName = categoryNames[i];
337	if (categoryName != null) {	337	if (categoryName != null) {
338	SimpleWordSet wordSet = (SimpleWordSet) wordSets[i];	338	SimpleWordSet wordSet = (SimpleWordSet) wordSets[i];
339	HashtableOfObject wordsToDocs = (HashtableOfObject) this.categoryTables.get(categoryName);	339	CategoryTable wordsToDocs = (CategoryTable) this.categoryTables.get(categoryName);
340	if (wordsToDocs == null)	340	if (wordsToDocs == null)
341	this.categoryTables.put(categoryName, wordsToDocs = new HashtableOfObject(wordSet.elementSize));	341	this.categoryTables.put(categoryName, wordsToDocs = new CategoryTable(wordSet.elementSize));
342		342
343	char[][] words = wordSet.words;	343	char[][] words = wordSet.words;
344	for (int j = 0, m = words.length; j < m; j++) {	344	for (int j = 0, m = words.length; j < m; j++) {
Lines 346-352 Link Here
346	if (word != null) {	346	if (word != null) {
347	Object o = wordsToDocs.get(word);	347	Object o = wordsToDocs.get(word);
348	if (o == null) {	348	if (o == null) {
349	wordsToDocs.put(word, new int[] {newPosition});	349	wordsToDocs.putUnsafely(word, new int[] {newPosition});
350	} else if (o instanceof IntList) {	350	} else if (o instanceof IntList) {
351	((IntList) o).add(newPosition);	351	((IntList) o).add(newPosition);
352	} else {	352	} else {
Lines 442-452 Link Here
442	this.categoryTables = null;	442	this.categoryTables = null;
443	}	443	}
444	private void mergeCategory(char[] categoryName, DiskIndex onDisk, int[] positions, FileOutputStream stream) throws IOException {	444	private void mergeCategory(char[] categoryName, DiskIndex onDisk, int[] positions, FileOutputStream stream) throws IOException {
445	HashtableOfObject wordsToDocs = (HashtableOfObject) this.categoryTables.get(categoryName);	445	CategoryTable wordsToDocs = (CategoryTable) this.categoryTables.get(categoryName);
446	if (wordsToDocs == null)	446	if (wordsToDocs == null)
447	wordsToDocs = new HashtableOfObject(3);	447	wordsToDocs = new CategoryTable(3);
448		448
449	HashtableOfObject oldWordsToDocs = onDisk.readCategoryTable(categoryName, true);	449	CategoryTable oldWordsToDocs = onDisk.readCategoryTable(categoryName, true);
450	if (oldWordsToDocs != null) {	450	if (oldWordsToDocs != null) {
451	char[][] oldWords = oldWordsToDocs.keyTable;	451	char[][] oldWords = oldWordsToDocs.keyTable;
452	Object[] oldArrayOffsets = oldWordsToDocs.valueTable;	452	Object[] oldArrayOffsets = oldWordsToDocs.valueTable;
Lines 469-475 Link Here
469		469
470	Object o = wordsToDocs.get(oldWord);	470	Object o = wordsToDocs.get(oldWord);
471	if (o == null) {	471	if (o == null) {
472	wordsToDocs.put(oldWord, mappedNumbers);	472	wordsToDocs.putUnsafely(oldWord, mappedNumbers);
473	} else {	473	} else {
474	IntList list = null;	474	IntList list = null;
475	if (o instanceof IntList) {	475	if (o instanceof IntList) {
Lines 580-586 Link Here
580	this.streamBuffer = null;	580	this.streamBuffer = null;
581	}	581	}
582	}	582	}
583	private synchronized HashtableOfObject readCategoryTable(char[] categoryName, boolean readDocNumbers) throws IOException {	583	private synchronized CategoryTable readCategoryTable(char[] categoryName, boolean readDocNumbers) throws IOException {
584	// result will be null if categoryName is unknown	584	// result will be null if categoryName is unknown
585	int offset = this.categoryOffsets.get(categoryName);	585	int offset = this.categoryOffsets.get(categoryName);
586	if (offset == HashtableOfIntValues.NO_VALUE) {	586	if (offset == HashtableOfIntValues.NO_VALUE) {
Lines 590-596 Link Here
590	if (this.categoryTables == null) {	590	if (this.categoryTables == null) {
591	this.categoryTables = new HashtableOfObject(3);	591	this.categoryTables = new HashtableOfObject(3);
592	} else {	592	} else {
593	HashtableOfObject cachedTable = (HashtableOfObject) this.categoryTables.get(categoryName);	593	CategoryTable cachedTable = (CategoryTable) this.categoryTables.get(categoryName);
594	if (cachedTable != null) {	594	if (cachedTable != null) {
595	if (readDocNumbers) { // must cache remaining document number arrays	595	if (readDocNumbers) { // must cache remaining document number arrays
596	Object[] arrayOffsets = cachedTable.valueTable;	596	Object[] arrayOffsets = cachedTable.valueTable;
Lines 603-609 Link Here
603	}	603	}
604		604
605	FileInputStream stream = new FileInputStream(this.indexFile);	605	FileInputStream stream = new FileInputStream(this.indexFile);
606	HashtableOfObject categoryTable = null;	606	CategoryTable categoryTable = null;
607	char[][] matchingWords = null;	607	char[][] matchingWords = null;
608	int count = 0;	608	int count = 0;
609	int firstOffset = -1;	609	int firstOffset = -1;
Lines 621-627 Link Here
621	System.err.println("size = "+size); //$NON-NLS-1$	621	System.err.println("size = "+size); //$NON-NLS-1$
622	System.err.println("-------------------- END --------------------"); //$NON-NLS-1$	622	System.err.println("-------------------- END --------------------"); //$NON-NLS-1$
623	}	623	}
624	categoryTable = new HashtableOfObject(size);	624	categoryTable = new CategoryTable(size);
625	} catch (OutOfMemoryError oom) {	625	} catch (OutOfMemoryError oom) {
626	// DEBUG	626	// DEBUG
627	oom.printStackTrace();	627	oom.printStackTrace();
Lines 641-649 Link Here
641	// > 1 & < 256 then the size of the array is > 1 & < 256, the document array follows immediately	641	// > 1 & < 256 then the size of the array is > 1 & < 256, the document array follows immediately
642	// 256 if the array size >= 256 followed by another int which is the offset to the array (written prior to the table)	642	// 256 if the array size >= 256 followed by another int which is the offset to the array (written prior to the table)
643	if (arrayOffset <= 0) {	643	if (arrayOffset <= 0) {
644	categoryTable.put(word, new int[] {-arrayOffset}); // store 1 element array by negating documentNumber	644	categoryTable.putUnsafely(word, new int[] {-arrayOffset}); // store 1 element array by negating documentNumber
645	} else if (arrayOffset < largeArraySize) {	645	} else if (arrayOffset < largeArraySize) {
646	categoryTable.put(word, readStreamDocumentArray(stream, arrayOffset)); // read in-lined array providing size	646	categoryTable.putUnsafely(word, readStreamDocumentArray(stream, arrayOffset)); // read in-lined array providing size
647	} else {	647	} else {
648	arrayOffset = readStreamInt(stream); // read actual offset	648	arrayOffset = readStreamInt(stream); // read actual offset
649	if (readDocNumbers) {	649	if (readDocNumbers) {
Lines 653-659 Link Here
653	firstOffset = arrayOffset;	653	firstOffset = arrayOffset;
654	matchingWords[count++] = word;	654	matchingWords[count++] = word;
655	}	655	}
656	categoryTable.put(word, new Integer(arrayOffset)); // offset to array in the file	656	categoryTable.putUnsafely(word, new Integer(arrayOffset)); // offset to array in the file
657	}	657	}
658	}	658	}
659	this.categoryTables.put(INTERNED_CATEGORY_NAMES.get(categoryName), categoryTable);	659	this.categoryTables.put(INTERNED_CATEGORY_NAMES.get(categoryName), categoryTable);
Lines 1030-1039 Link Here
1030	Object[] tables = this.categoryTables.valueTable;	1030	Object[] tables = this.categoryTables.valueTable;
1031	for (int i = 0, l = categoryNames.length; i < l; i++)	1031	for (int i = 0, l = categoryNames.length; i < l; i++)
1032	if (categoryNames[i] != null)	1032	if (categoryNames[i] != null)
1033	writeCategoryTable(categoryNames[i], (HashtableOfObject) tables[i], stream);	1033	writeCategoryTable(categoryNames[i], (CategoryTable) tables[i], stream);
1034	this.categoryTables = null;	1034	this.categoryTables = null;
1035	}	1035	}
1036	private void writeCategoryTable(char[] categoryName, HashtableOfObject wordsToDocs, FileOutputStream stream) throws IOException {	1036	private void writeCategoryTable(char[] categoryName, CategoryTable wordsToDocs, FileOutputStream stream) throws IOException {
1037	// the format of a category table is as follows:	1037	// the format of a category table is as follows:
1038	// any document number arrays with >= 256 elements are written before the table (the offset to each array is remembered)	1038	// any document number arrays with >= 256 elements are written before the table (the offset to each array is remembered)
1039	// then the number of word->int[] pairs in the table is written	1039	// then the number of word->int[] pairs in the table is written