Attachment #199070 for bug #183164

View | Details | Raw Unified | Return to bug 183164 | Differences between

and this patch




package org.eclipse.equinox.bidi.internal.tests;

import org.eclipse.equinox.bidi.STextEngine;
import org.eclipse.equinox.bidi.custom.STextProcessor;
import org.eclipse.equinox.bidi.custom.STextStringProcessor;
import org.eclipse.equinox.bidi.custom.ISTextProcessor;

/**
 * Tests contribution of BiDi processors.

		// check one of the types that we know should be there
		assertTrue(isTypePresent(types, "regex"));

		STextProcessor processor = STextStringProcessor.getProcessor("regex");
		assertNotNull(processor);
	}


		// check the type added by the test bundle
		assertTrue(isTypePresent(types, "test"));

		STextProcessor processor = STextStringProcessor.getProcessor("test");
		assertNotNull(processor);

		processor = STextStringProcessor.getProcessor("badtest");

		String data, lean, full, model;
		data = "ABC.DEF:HOST-COM=HELLO";
		lean = toUT16(data);
		processor = STextStringProcessor.getProcessor("test");
		full = STextEngine.leanToFullText(processor, null, lean, null);
		model = "ABC@.DEF@:HOST@-COM@=HELLO";
		assertEquals("Test 'test' plugin", model, toPseudo(full));
	}





package org.eclipse.equinox.bidi.internal.tests;

import java.util.Locale;
import org.eclipse.equinox.bidi.STextEngine;
import org.eclipse.equinox.bidi.custom.STextProcessor;
import org.eclipse.equinox.bidi.custom.*;

/**
 * Tests all plug-in extensions


public class STextExtensionsTest extends STextTestBase {

	STextProcessor processor;
	STextFeatures f1, f2;
	int[] state = new int[1];
	String type;

	private void doTest1(String label, String data, String result) {
		String full;
		full = STextEngine.leanToFullText(processor, null, toUT16(data), state);
		assertEquals(label + " data = " + data, result, toPseudo(full));
	}

	private void doTest2(String label, String data, String result) {
		String full;
		full = STextEngine.leanToFullText(processor, null, data, state);
		assertEquals(label + " data = " + data, result, toPseudo(full));
	}

	private void doTest3(String label, String data, String result) {
		String full;
		full = STextEngine.leanToFullText(processor, null, toUT16(data), state);
		assertEquals(label + " data = " + data, result, toPseudo(full));
	}

	public void testExtensions() {

		String data;
		processor = STextEngine.PROC_COMMA_DELIMITED;
		state[0] = STextEngine.STATE_INITIAL;
		doTest1("Comma #1", "ab,cd, AB, CD, EFG", "ab,cd, AB@, CD@, EFG");

		processor = STextEngine.PROC_EMAIL;
		state[0] = STextEngine.STATE_INITIAL;
		doTest1("Email #1", "abc.DEF:GHI", "abc.DEF@:GHI");
		doTest1("Email #2", "DEF.GHI \"A.B\":JK ", "DEF@.GHI @\"A.B\"@:JK ");
		doTest1("Email #3", "DEF,GHI (A,B);JK ", "DEF@,GHI @(A,B)@;JK ");
		doTest1("Email #4", "DEF.GHI (A.B :JK ", "DEF@.GHI @(A.B :JK ");
		Locale.setDefault(new Locale("ar"));
		doTest1("Email #5", "#EF.GHI \"A.B\":JK ", "<&#EF.GHI \"A.B\":JK &^");
		doTest1("Email #6", "#EF,GHI (A,B);JK ", "<&#EF,GHI (A,B);JK &^");
		doTest1("Email #7", "#EF.GHI (A.B :JK ", "<&#EF.GHI (A.B :JK &^");
		data = toUT16("peter.pan") + "@" + toUT16("#EF.GHI");
		doTest2("Email #8", data, "<&peter&.pan@#EF.GHI&^");
		Locale.setDefault(new Locale("he"));
		data = toUT16("peter.pan") + "@" + toUT16("DEF.GHI");
		doTest2("Email #9", data, "peter.pan@DEF@.GHI");

		processor = STextEngine.PROC_FILE;
		state[0] = STextEngine.STATE_INITIAL;
		doTest1("File #1", "c:\\A\\B\\FILE.EXT", "c:\\A@\\B@\\FILE@.EXT");

		processor = STextEngine.PROC_JAVA;
		state[0] = STextEngine.STATE_INITIAL;
		doTest1("Java #1", "A = B + C;", "A@ = B@ + C;");
		doTest1("Java #2", "A   = B + C;", "A@   = B@ + C;");

		doTest1("Java #9", "A = //B+C* D;", "A@ = //B+C* D;");
		doTest1("Java #10", "A = //B+C`|D+E;", "A@ = //B+C`|D@+E;");

		processor = STextEngine.PROC_PROPERTY;
		state[0] = STextEngine.STATE_INITIAL;
		doTest1("Property #0", "NAME,VAL1,VAL2", "NAME,VAL1,VAL2");
		doTest1("Property #1", "NAME=VAL1,VAL2", "NAME@=VAL1,VAL2");
		doTest1("Property #2", "NAME=VAL1,VAL2=VAL3", "NAME@=VAL1,VAL2=VAL3");

		processor = STextEngine.PROC_REGEXP;
		type = ISTextTypes.REGEXP;
		state[0] = STextEngine.STATE_INITIAL;
		data = toUT16("ABC(?") + "#" + toUT16("DEF)GHI");
		doTest2("Regex #0.0", data, "A@B@C@(?#DEF)@G@H@I");

		doTest1("Regex #17.6", "aB*123", "aB*@123");
		doTest1("Regex #17.7", "aB*567", "aB*@567");

		Locale.setDefault(new Locale("ar"));
		data = toUT16("#BC(?") + "#" + toUT16("DEF)GHI");
		doTest2("Regex #0.0", data, "<&#BC(?#DEF)GHI&^");
		data = toUT16("#BC(?") + "#" + toUT16("DEF");
		doTest2("Regex #0.1", data, "<&#BC(?#DEF&^");
		doTest1("Regex #0.2", "#HI)JKL", "<&#HI)JKL&^");
		data = toUT16("#BC(?") + "<" + toUT16("DEF") + ">" + toUT16("GHI");
		doTest2("Regex #1", data, "<&#BC(?<DEF>GHI&^");
		doTest1("Regex #2.0", "#BC(?'DEF'GHI", "<&#BC(?'DEF'GHI&^");
		doTest1("Regex #2.1", "#BC(?'DEFGHI", "<&#BC(?'DEFGHI&^");
		data = toUT16("#BC(?(") + "<" + toUT16("DEF") + ">" + toUT16(")GHI");
		doTest2("Regex #3", data, "<&#BC(?(<DEF>)GHI&^");
		doTest1("Regex #4", "#BC(?('DEF')GHI", "<&#BC(?('DEF')GHI&^");
		doTest1("Regex #5", "#BC(?(DEF)GHI", "<&#BC(?(DEF)GHI&^");
		data = toUT16("#BC(?") + "&" + toUT16("DEF)GHI");
		doTest2("Regex #6", data, "<&#BC(?&DEF)GHI&^");
		data = toUT16("#BC(?") + "P<" + toUT16("DEF") + ">" + toUT16("GHI");
		doTest2("Regex #7", data, "<&#BC(?p<DEF>GHI&^");
		data = toUT16("#BC\\k") + "<" + toUT16("DEF") + ">" + toUT16("GHI");
		doTest2("Regex #8", data, "<&#BC\\k<DEF>GHI&^");
		doTest1("Regex #9", "#BC\\k'DEF'GHI", "<&#BC\\k'DEF'GHI&^");
		doTest1("Regex #10", "#BC\\k{DEF}GHI", "<&#BC\\k{DEF}GHI&^");
		data = toUT16("#BC(?") + "P=" + toUT16("DEF)GHI");
		doTest2("Regex #11", data, "<&#BC(?p=DEF)GHI&^");
		doTest1("Regex #12", "#BC\\g{DEF}GHI", "<&#BC\\g{DEF}GHI&^");
		data = toUT16("#BC\\g") + "<" + toUT16("DEF") + ">" + toUT16("GHI");
		doTest2("Regex #13", data, "<&#BC\\g<DEF>GHI&^");
		doTest1("Regex #14", "#BC\\g'DEF'GHI", "<&#BC\\g'DEF'GHI&^");
		data = toUT16("#BC(?(") + "R&" + toUT16("DEF)GHI");
		doTest2("Regex #15", data, "<&#BC(?(r&DEF)GHI&^");
		data = toUT16("#BC") + "\\Q" + toUT16("DEF") + "\\E" + toUT16("GHI");
		doTest2("Regex #16.0", data, "<&#BC\\qDEF\\eGHI&^");
		data = toUT16("#BC") + "\\Q" + toUT16("DEF");
		doTest2("Regex #16.1", data, "<&#BC\\qDEF&^");
		data = toUT16("#HI") + "\\E" + toUT16("JKL");
		doTest2("Regex #16.2", data, "<&#HI\\eJKL&^");
		Locale.setDefault(new Locale("he"));

		processor = STextEngine.PROC_SQL;
		state[0] = STextEngine.STATE_INITIAL;
		doTest1("SQL #0", "abc GHI", "abc GHI");
		doTest1("SQL #1", "abc DEF   GHI", "abc DEF@   GHI");

		doTest1("SQL #12", "ABC\"DEF \"\" G I\" JKL,MN", "ABC@\"DEF \"\" G I\"@ JKL@,MN");
		doTest1("SQL #13", "ABC--DEF GHI`|JKL MN", "ABC@--DEF GHI`|JKL@ MN");

		processor = STextEngine.PROC_SYSTEM_USER;
		state[0] = STextEngine.STATE_INITIAL;
		doTest1("System #1", "HOST(JACK)", "HOST@(JACK)");

		processor = STextEngine.PROC_UNDERSCORE;
		state[0] = STextEngine.STATE_INITIAL;
		doTest1("Underscore #1", "A_B_C_d_e_F_G", "A@_B@_C_d_e_F@_G");

		processor = STextEngine.PROC_URL;
		state[0] = STextEngine.STATE_INITIAL;
		doTest1("URL #1", "WWW.DOMAIN.COM/DIR1/DIR2/dir3/DIR4", "WWW@.DOMAIN@.COM@/DIR1@/DIR2/dir3/DIR4");

		processor = STextEngine.PROC_XPATH;
		state[0] = STextEngine.STATE_INITIAL;
		doTest1("Xpath #1", "abc(DEF)GHI", "abc(DEF@)GHI");
		doTest1("Xpath #2", "DEF.GHI \"A.B\":JK ", "DEF@.GHI@ \"A.B\"@:JK ");
		doTest1("Xpath #3", "DEF!GHI 'A!B'=JK ", "DEF@!GHI@ 'A!B'@=JK ");
		doTest1("Xpath #4", "DEF.GHI 'A.B :JK ", "DEF@.GHI@ 'A.B :JK ");

		processor = STextEngine.PROC_EMAIL;
		processor = STextStringProcessor.getProcessor(type);
		state[0] = STextEngine.STATE_INITIAL;
		assertEquals("<>.:,;@", processor.getSeparators(null, "", null));
		doTest3("DelimsEsc #1", "abc.DEF.GHI", "abc.DEF@.GHI");
		doTest3("DelimsEsc #2", "DEF.GHI (A:B);JK ", "DEF@.GHI @(A:B)@;JK ");
		doTest3("DelimsEsc #3", "DEF.GHI (A:B);JK ", "DEF@.GHI @(A:B)@;JK ");
		doTest3("DelimsEsc #4", "DEF.GHI (A:B\\):C) ;JK ", "DEF@.GHI @(A:B\\):C) @;JK ");
		doTest3("DelimsEsc #5", "DEF.GHI (A\\\\\\):C) ;JK ", "DEF@.GHI @(A\\\\\\):C) @;JK ");
		doTest3("DelimsEsc #6", "DEF.GHI (A\\\\):C ;JK ", "DEF@.GHI @(A\\\\)@:C @;JK ");
		doTest3("DelimsEsc #7", "DEF.GHI (A\\):C ;JK ", "DEF@.GHI @(A\\):C ;JK ");
		doTest3("DelimsEsc #6", "DEF-GHI (A\\\\)*C /JK ", "DEF@-GHI @(A\\\\)@*C @/JK ");
		doTest3("DelimsEsc #7", "DEF-GHI (A\\)*C /JK ", "DEF@-GHI @(A\\)*C /JK ");
	}
}





package org.eclipse.equinox.bidi.internal.tests;

import org.eclipse.equinox.bidi.STextEngine;
import org.eclipse.equinox.bidi.STextEnvironment;
import org.eclipse.equinox.bidi.custom.STextProcessor;

/**
 * Tests fullToLean method

	static final STextEnvironment envLTR = new STextEnvironment(null, false, STextEnvironment.ORIENT_LTR);
	static final STextEnvironment envRTL = new STextEnvironment(null, false, STextEnvironment.ORIENT_RTL);

	STextProcessor processor;

	private void doTest1(String msg, String data, String leanLTR, String fullLTR, int[] l2fMapLTR, int[] f2lMapLTR, String leanRTL, String fullRTL, int[] l2fMapRTL, int[] f2lMapRTL) {
		String text, full, lean, label;
		int[] map;

		text = toUT16(data);
		lean = STextEngine.fullToLeanText(processor, envLTR, text, null);
		assertEquals(msg + "LTR lean", leanLTR, toPseudo(lean));
		full = STextEngine.leanToFullText(processor, envLTR, lean, null);
		assertEquals(msg + "LTR full", fullLTR, toPseudo(full));
		map = STextEngine.leanToFullMap(processor, envLTR, lean, null);
		label = msg + "leanToFullMap() LTR";
		assertEquals(label, array_display(l2fMapLTR), array_display(map));
		map = STextEngine.fullToLeanMap(processor, envLTR, text, null);
		label = msg + "fullToLeanMap() LTR";
		assertEquals(label, array_display(f2lMapLTR), array_display(map));

		lean = STextEngine.fullToLeanText(processor, envRTL, text, null);
		assertEquals(msg + "RTL lean", leanRTL, toPseudo(lean));
		full = STextEngine.leanToFullText(processor, envRTL, lean, null);
		assertEquals(msg + "RTL full", fullRTL, toPseudo(full));
		map = STextEngine.leanToFullMap(processor, envRTL, lean, null);
		label = msg + "leanToFullMap() RTL";
		assertEquals(label, array_display(l2fMapRTL), array_display(map));
		map = STextEngine.fullToLeanMap(processor, envRTL, text, null);
		label = msg + "fullToLeanMap() RTL";
		assertEquals(label, array_display(f2lMapRTL), array_display(map));
	}

		data = "update \"AB_CDE\" set \"COL1\"@='01', \"COL2\"@='02' /* GH IJK";
		text = toUT16(data);
		state[0] = -1;
		lean = STextEngine.fullToLeanText(processor, envLTR, text, state);
		state1 = state[0];
		model = "update \"AB_CDE\" set \"COL1\"='01', \"COL2\"='02' /* GH IJK";
		assertEquals(msg + "LTR lean", model, toPseudo(lean));
		state[0] = -1;
		full = STextEngine.leanToFullText(processor, envLTR, lean, state);
		assertEquals(msg + "LTR full", data, toPseudo(full));
		assertEquals(msg + "state from leanToFullText", state1, state[0]);
		data = "THIS IS A COMMENT LINE";
		text = toUT16(data);
		state[0] = state1;
		lean = STextEngine.fullToLeanText(processor, envLTR, text, state);
		state2 = state[0];
		model = "THIS IS A COMMENT LINE";
		assertEquals(msg + "LTR lean2", model, toPseudo(lean));
		state[0] = state1;
		full = STextEngine.leanToFullText(processor, envLTR, lean, state);
		assertEquals(msg + "LTR full2", data, toPseudo(full));
		assertEquals(msg + "state from leanToFullText2", state2, state[0]);
		data = "SOME MORE */ where \"COL3\"@=123";
		text = toUT16(data);
		state[0] = state2;
		lean = STextEngine.fullToLeanText(processor, envLTR, text, state);
		state3 = state[0];
		model = "SOME MORE */ where \"COL3\"=123";
		assertEquals(msg + "LTR lean3", model, toPseudo(lean));
		state[0] = state2;
		full = STextEngine.leanToFullText(processor, envLTR, lean, state);
		assertEquals(msg + "LTR full3", data, toPseudo(full));
		assertEquals(msg + "state from leanToFullText3", state3, state[0]);
	}

	public void testFullToLean() {

		processor = STextEngine.PROC_COMMA_DELIMITED;
		doTest1("testFullToLean #1 - ", "", "", "", new int[0], new int[0], "", "", new int[0], new int[0]);
		int[] map1 = new int[] {0, 1, 2, 3, 4};
		int[] map2 = new int[] {2, 3, 4, 5, 6};

		doTest1("testFullToLean #37 - ", ">>>@@@@@^^^", "", "", map1, map2, "", "", map1, map2);

		// test fullToLeanText with initial state
		processor = STextEngine.PROC_SQL;
		doTest2("testFullToLean #38 - ");
	}
}





package org.eclipse.equinox.bidi.internal.tests;

import java.util.Locale;
import org.eclipse.equinox.bidi.STextEngine;
import org.eclipse.equinox.bidi.STextEnvironment;
import org.eclipse.equinox.bidi.custom.STextProcessor;

/**
 * Tests RTL arithmetic

	static final STextEnvironment envLTR = new STextEnvironment(null, false, STextEnvironment.ORIENT_LTR);
	static final STextEnvironment envRTL = new STextEnvironment(null, false, STextEnvironment.ORIENT_RTL);

	STextProcessor processor = STextEngine.PROC_RTL_ARITHMETIC;

	private void verifyOneLine(String msg, String data, String resLTR, String resRTL) {
		String lean = toUT16(data);
		String fullLTR = STextEngine.leanToFullText(processor, envLTR, lean, null);
		assertEquals(msg + " LTR - ", resLTR, toPseudo(fullLTR));
		String fullRTL = STextEngine.leanToFullText(processor, envRTL, lean, null);
		assertEquals(msg + " RTL - ", resRTL, toPseudo(fullRTL));
	}

	public void testRTLarithmetic() {
		verifyOneLine("Math #0", "", "", "");
		verifyOneLine("Math #1", "1+ABC", "1+ABC", ">@1+ABC@^");
		verifyOneLine("Math #2", "2+ABC-DEF", "2+ABC@-DEF", ">@2+ABC@-DEF@^");
		verifyOneLine("Math #3", "A+3*BC/DEF", "A@+3*BC@/DEF", ">@A@+3*BC@/DEF@^");
		verifyOneLine("Math #4", "4+ABC/DEF", "4+ABC@/DEF", ">@4+ABC@/DEF@^");
		Locale.setDefault(new Locale("ar"));
		verifyOneLine("Math #5", "5#BC", "<&5#BC&^", "5#BC");
		verifyOneLine("Math #6", "6#BC-DE", "<&6#BC-DE&^", "6#BC-DE");
		verifyOneLine("Math #7", "7#BC+DE", "<&7#BC+DE&^", "7#BC+DE");
		verifyOneLine("Math #8", "8#BC*DE", "<&8#BC*DE&^", "8#BC*DE");
		verifyOneLine("Math #9", "9#BC/DE", "<&9#BC/DE&^", "9#BC/DE");
		verifyOneLine("Math #10", "10ab+cd-ef", "10ab+cd-ef", ">@10ab+cd-ef@^");
		Locale.setDefault(new Locale("he"));
	}
}





package org.eclipse.equinox.bidi.internal.tests;

import org.eclipse.equinox.bidi.STextEngine;
import org.eclipse.equinox.bidi.STextEnvironment;
import org.eclipse.equinox.bidi.custom.STextProcessor;

/**
 * Tests most public methods of BidiComplexEngine


public class STextMethodsTest extends STextTestBase {

	final static int LTR = STextEngine.DIR_LTR;
	final static int RTL = STextEngine.DIR_RTL;
	final static STextEnvironment envLTR = new STextEnvironment(null, false, STextEnvironment.ORIENT_LTR);
	final static STextEnvironment envRTL = new STextEnvironment(null, false, STextEnvironment.ORIENT_RTL);
	final static STextEnvironment envRTLMIR = new STextEnvironment(null, true, STextEnvironment.ORIENT_RTL);

	final static STextEnvironment envCLR = new STextEnvironment(null, false, STextEnvironment.ORIENT_CONTEXTUAL_LTR);
	final static STextEnvironment envCRL = new STextEnvironment(null, false, STextEnvironment.ORIENT_CONTEXTUAL_RTL);
	final static STextEnvironment envERR = new STextEnvironment(null, false, 9999);
	final static byte AL = Character.DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC;
	int dirArabic, dirHebrew;

	class MyComma extends STextProcessor {

		public String getSeparators(STextEnvironment environment, String text, byte[] dirProps) {
			return ","; //$NON-NLS-1$
		}

		public boolean skipProcessing(STextEnvironment environment, String text, byte[] dirProps) {
			byte dirProp = getDirProp(text, dirProps, 0);
			if (dirProp == AL)
				return true;
			return false;
		}

		public int getDirection(STextEnvironment environment, String text, byte[] dirProps) {
			for (int i = 0; i < text.length(); i++) {
				byte dirProp = getDirProp(text, dirProps, i);
				if (dirProp == AL)
					return dirArabic;
			}
			return dirHebrew;
		}
	}

	STextProcessor processor;

	private void doTestTools() {


		String data, lean, full, model;
		int[] state = new int[1];
		state[0] = -1;
		processor = STextEngine.PROC_JAVA;
		data = "A=B+C;/* D=E+F;";
		lean = toUT16(data);
		full = STextEngine.leanToFullText(processor, null, lean, state);
		model = "A@=B@+C@;/* D=E+F;";
		assertEquals("full1", model, toPseudo(full));
		data = "A=B+C; D=E+F;";
		lean = toUT16(data);
		full = STextEngine.leanToFullText(processor, null, lean, state);
		model = "A=B+C; D=E+F;";
		assertEquals("full2", model, toPseudo(full));
		data = "A=B+C;*/ D=E+F;";
		lean = toUT16(data);
		full = STextEngine.leanToFullText(processor, null, lean, state);
		model = "A=B+C;@*/ D@=E@+F;";
		assertEquals("full3", model, toPseudo(full));
	}

		assertEquals("orient #4", STextEnvironment.ORIENT_UNKNOWN, orient);
	}

	private void doTestOrient(String label, String data, String resLTR, String resRTL, String resCon) {
		String full, lean;
		String type = ISTextTypes.COMMA_DELIMITED;

		lean = toUT16(data);
		full = STextEngine.leanToFullText(processor, envLTR, lean, null);
		assertEquals(label + "LTR full", resLTR, toPseudo(full));
		full = STextEngine.leanToFullText(processor, envRTL, lean, null);
		assertEquals("label + RTL full", resRTL, toPseudo(full));
		full = STextEngine.leanToFullText(processor, envCRL, lean, null);
		assertEquals(label + "CON full", resCon, toPseudo(full));
	}

	private void doTestSkipProcessing() {
		processor = new MyComma();
		doTestOrient("Skip #1 ", "BCD,EF", "BCD@,EF", ">@BCD@,EF@^", "@BCD@,EF");
		doTestOrient("Skip #2 ", "#CD,EF", "#CD,EF", ">@#CD,EF@^", "@#CD,EF");
		assertFalse("Ignores Arabic 1", flag);
		flag = f1.getIgnoreHebrew();
		assertFalse("Ignores Hebrew 1", flag);

		f2 = new STextFeatures(f1.getSeparators(), 0, -1, -1, true, true);
		flag = f2.getIgnoreArabic();
		assertTrue("Ignores Arabic 2", flag);
		flag = f2.getIgnoreHebrew();
		assertTrue("Ignores Hebrew 2", flag);
		doTestOrient(f2, "Scripts #1 ", "BCD,EF", "BCD,EF", ">@BCD,EF@^", "@BCD,EF");
		f2 = new STextFeatures(f1.getSeparators(), 0, -1, -1, false, true);
		flag = f2.getIgnoreArabic();
		assertFalse("Ignores Arabic 3", flag);
		flag = f2.getIgnoreHebrew();
		assertTrue("Ignores Hebrew 3", flag);
		doTestOrient(f2, "Scripts #2 ", "d,EF", "d,EF", ">@d,EF@^", "d,EF");
		doTestOrient(f2, "Scripts #3 ", "#,eF", "#,eF", ">@#,eF@^", "@#,eF");
		doTestOrient(f2, "Scripts #4 ", "#,12", "#@,12", ">@#@,12@^", "@#@,12");
		doTestOrient(f2, "Scripts #5 ", "#,##", "#@,##", ">@#@,##@^", "@#@,##");
		doTestOrient(f2, "Scripts #6 ", "#,89", "#@,89", ">@#@,89@^", "@#@,89");
		doTestOrient(f2, "Scripts #7 ", "#,ef", "#,ef", ">@#,ef@^", "@#,ef");
		doTestOrient(f2, "Scripts #8 ", "#,", "#,", ">@#,@^", "@#,");
		doTestOrient(f2, "Scripts #9 ", "9,ef", "9,ef", ">@9,ef@^", "9,ef");
		doTestOrient(f2, "Scripts #10 ", "9,##", "9@,##", ">@9@,##@^", "9@,##");
		doTestOrient(f2, "Scripts #11 ", "7,89", "7@,89", ">@7@,89@^", "7@,89");
		doTestOrient(f2, "Scripts #12 ", "7,EF", "7,EF", ">@7,EF@^", "@7,EF");
		doTestOrient(f2, "Scripts #13 ", "BCD,EF", "BCD,EF", ">@BCD,EF@^", "@BCD,EF");

		f2 = new STextFeatures(f1.getSeparators(), 0, -1, -1, true, false);
		flag = f2.getIgnoreArabic();
		assertTrue("Ignores Arabic 4", flag);
		flag = f2.getIgnoreHebrew();
		assertFalse("Ignores Hebrew 4", flag);
		doTestOrient(f2, "Scripts #14 ", "BCd,EF", "BCd,EF", ">@BCd,EF@^", "@BCd,EF");
		doTestOrient(f2, "Scripts #15 ", "BCD,eF", "BCD,eF", ">@BCD,eF@^", "@BCD,eF");
		doTestOrient(f2, "Scripts #16 ", "BCD,EF", "BCD@,EF", ">@BCD@,EF@^", "@BCD@,EF");
		doTestOrient(f2, "Scripts #17 ", "BCD,12", "BCD@,12", ">@BCD@,12@^", "@BCD@,12");
		doTestOrient(f2, "Scripts #18 ", "BCD,", "BCD,", ">@BCD,@^", "@BCD,");

		f2 = new STextFeatures(f1.getSeparators(), 0, -1, -1, false, false);
		doTestOrient(f2, "Scripts #19 ", "123,45|67", "123,45|67", ">@123,45|67@^", "@123,45|67");
		doTestOrient(f2, "Scripts #20 ", "5,e", "5,e", ">@5,e@^", "5,e");
		doTestOrient(f2, "Scripts #21 ", "5,#", "5@,#", ">@5@,#@^", "5@,#");
		doTestOrient(f2, "Scripts #22 ", "5,6", "5@,6", ">@5@,6@^", "5@,6");
		doTestOrient(f2, "Scripts #23 ", "5,D", "5@,D", ">@5@,D@^", "5@,D");
		doTestOrient(f2, "Scripts #24 ", "5,--", "5,--", ">@5,--@^", "@5,--");
	}

	private void doTestLeanOffsets() {
		String lean, data, label;
		int[] state = new int[1];
		processor = STextEngine.PROC_JAVA;
		int[] offsets;
		int[] model;

		data = "A=B+C;/* D=E+F;";
		lean = toUT16(data);
		state[0] = -1;
		offsets = STextEngine.leanBidiCharOffsets(processor, null, lean, state);
		model = new int[] {1, 3, 5};
		label = "leanBidiCharOffsets() #1 ";
		assertEquals(label, array_display(model), array_display(offsets));
		data = "A=B+C;*/ D=E+F;";
		lean = toUT16(data);
		offsets = STextEngine.leanBidiCharOffsets(processor, null, lean, state);
		model = new int[] {6, 10, 12};
		label = "leanBidiCharOffsets() #2 ";
		assertEquals(label, array_display(model), array_display(offsets));

	private void doTestFullOffsets(String label, String data, int[] resLTR, int[] resRTL, int[] resCon) {
		String full, lean, msg;
		int[] offsets;
		processor = STextEngine.PROC_COMMA_DELIMITED;

		lean = toUT16(data);
		full = STextEngine.leanToFullText(processor, envLTR, lean, null);
		offsets = STextEngine.fullBidiCharOffsets(processor, envLTR, full, null);
		msg = label + "LTR ";
		assertEquals(msg, array_display(resLTR), array_display(offsets));
		full = STextEngine.leanToFullText(processor, envRTL, lean, null);
		offsets = STextEngine.fullBidiCharOffsets(processor, envRTL, full, null);
		msg = label + "RTL ";
		assertEquals(msg, array_display(resRTL), array_display(offsets));
		full = STextEngine.leanToFullText(processor, envCLR, lean, null);
		offsets = STextEngine.fullBidiCharOffsets(processor, envCLR, full, null);
		msg = label + "CON ";
		assertEquals(msg, array_display(resCon), array_display(offsets));
	}


	private void doTestDirection() {
		String data, lean, full, model, msg;
		int dirA, dirH;
		processor = new MyComma();
		dirArabic = RTL;
		dirHebrew = LTR;
		msg = "TestDirection #1";
		dirA = processor.getDirection(null, toUT16("###"), null);
		dirH = processor.getDirection(null, toUT16("ABC"), null);
		assertTrue(msg, dirA == RTL && dirH == LTR);

		dirArabic = RTL;
		dirHebrew = RTL;
		msg = "TestDirection #2";
		dirA = processor.getDirection(null, toUT16("###"), null);
		dirH = processor.getDirection(null, toUT16("ABC"), null);
		assertTrue(msg, dirA == RTL && dirH == RTL);

		dirArabic = dirHebrew = LTR;
		processor = STextStringProcessor.getProcessor(ISTextTypes.EMAIL);
		f1 = processor.getFeatures(environment);
		msg = "TestDirection #3";
		data = "ABC,#DEF,HOST,com";
		data = "#ABC.#DEF:HOST.com";
		lean = toUT16(data);
		full = STextEngine.leanToFullText(processor, null, lean, null);
		model = "ABC@,#DEF@,HOST,com";
		assertEquals("TestDirection #9 full", model, toPseudo(full));

		data = "ABC,DEF,HOST,com";
		lean = toUT16(data);
		full = STextEngine.leanToFullText(processor, null, lean, null);
		model = "ABC@,DEF@,HOST,com";
		assertEquals("TestDirection #10 full", model, toPseudo(full));

		dirArabic = RTL;
		f1 = processor.getFeatures(environment);
		msg = "TestDirection #10.5";
		dirA = processor.getDirection(null, toUT16("###"), null);
		dirH = processor.getDirection(null, toUT16("ABC"), null);
		assertTrue(msg, dirA == RTL && dirH == LTR);
		STextEnvironment environment = new STextEnvironment(null, true, STextEnvironment.ORIENT_LTR);
		data = "ABC,#DEF,HOST,com";
		lean = toUT16(data);
		full = STextEngine.leanToFullText(processor, environment, lean, null);
		model = "<&ABC,#DEF,HOST,com&^";
		assertEquals("TestDirection #11 full", model, toPseudo(full));

		data = "ABc,#DEF,HOSt,COM";
		lean = toUT16(data);
		full = STextEngine.leanToFullText(processor, environment, lean, null);
		model = "<&ABc,#DEF,HOSt,COM&^";
		assertEquals("TestDirection #12 full", model, toPseudo(full));

		data = "ABc,#DEF,HOSt,";
		lean = toUT16(data);
		full = STextEngine.leanToFullText(processor, environment, lean, null);
		model = "<&ABc,#DEF,HOSt,&^";
		assertEquals("TestDirection #13 full", model, toPseudo(full));

		data = "ABC,DEF,HOST,com";
		lean = toUT16(data);
		full = STextEngine.leanToFullText(processor, environment, lean, null);
		model = "ABC@,DEF@,HOST,com";
		assertEquals("TestDirection #14 full", model, toPseudo(full));

		data = "--,---,----";
		lean = toUT16(data);
		full = STextEngine.leanToFullText(processor, environment, lean, null);
		model = "--,---,----";
		assertEquals("TestDirection #15 full", model, toPseudo(full));

		data = "ABC,|DEF,HOST,com";
		lean = toUT16(data);
		full = STextEngine.leanToFullText(processor, environment, lean, null);
		model = "ABC,|DEF@,HOST,com";
		assertEquals("TestDirection #16 full", model, toPseudo(full));

		data = "ABc,|#DEF,HOST,com";
		lean = toUT16(data);
		full = STextEngine.leanToFullText(processor, envRTLMIR, lean, null);
		model = "ABc,|#DEF,HOST,com";
		assertEquals("TestDirection #17 full", model, toPseudo(full));
		assertEquals("Test curDirection", RTL, STextEngine.getCurDirection(processor, envRTLMIR, lean));
	}

	public void testMethods() {


		doTestOrientation();

		processor = STextEngine.PROC_COMMA_DELIMITED;
		doTestOrient("Methods #1 ", "", "", "", "");
		doTestOrient("Methods #2 ", "abc", "abc", ">@abc@^", "abc");
		doTestOrient("Methods #3 ", "ABC", "ABC", ">@ABC@^", "@ABC");
		doTestOrient("Methods #4 ", "bcd,ef", "bcd,ef", ">@bcd,ef@^", "bcd,ef");
		doTestOrient("Methods #5 ", "BCD,EF", "BCD@,EF", ">@BCD@,EF@^", "@BCD@,EF");
		doTestOrient("Methods #6 ", "cde,FG", "cde,FG", ">@cde,FG@^", "cde,FG");
		doTestOrient("Methods #7 ", "CDE,fg", "CDE,fg", ">@CDE,fg@^", "@CDE,fg");
		doTestOrient("Methods #8 ", "12..def,GH", "12..def,GH", ">@12..def,GH@^", "12..def,GH");
		doTestOrient("Methods #9 ", "34..DEF,gh", "34..DEF,gh", ">@34..DEF,gh@^", "@34..DEF,gh");
		doTestOrient(f, "Methods #9 ", "34..DEF,gh", "34..DEF,gh", ">@34..DEF,gh@^", "@34..DEF,gh");

		doTestSkipProcessing();

		doTestLeanOffsets();



		doTestDirection();

		processor = STextEngine.PROC_COMMA_DELIMITED;
		String data = "A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z";
		String lean = toUT16(data);
		String full = STextEngine.leanToFullText(processor, null, lean, null);
		String model = "A@,B@,C@,D@,E@,F@,G@,H@,I@,J@,K@,L@,M@,N@,O@,P@,Q@,R@,S@,T@,U@,V@,W@,X@,Y@,Z";
		assertEquals("many inserts", model, toPseudo(full));
	}




package org.eclipse.equinox.bidi.internal.tests;

import org.eclipse.equinox.bidi.STextEngine;
import org.eclipse.equinox.bidi.custom.STextFeatures;

/**
 * Tests RTL arithmetic

	static final int[] EMPTY_INT_ARRAY = new int[0];

	public void testNullProcessor() {
		String full = STextEngine.leanToFullText(null, null, "abc", null);
		assertEquals("leanToFullText", "abc", full);
		int[] state = new int[1];
		state[0] = 3;
		full = STextEngine.leanToFullText(null, null, "abc", state);
		assertEquals("leanToFullText with state", "abc", full);
		int[] offsets = STextEngine.leanBidiCharOffsets(null, null, "abc", null);
		assertEquals("leanBidiCharOffsets", 0, offsets.length);
		offsets = STextEngine.fullBidiCharOffsets(null, null, "abc", null);
		assertEquals("fullBidiCharOffsets", 0, offsets.length);
		String lean = STextEngine.fullToLeanText(null, null, "abc", null);
		assertEquals("fullToLeanText", "abc", lean);
		lean = STextEngine.fullToLeanText(null, null, "abc", state);
		assertEquals("fullToLeanText with state", "abc", lean);
		int[] map = STextEngine.leanToFullMap(null, null, "abc", null);
		int[] model = {0, 1, 2};
		assertEquals("leanToFullMap", array_display(model), array_display(map));
		map = STextEngine.fullToLeanMap(null, null, "abc", null);
		assertEquals("fullToLeanMap", array_display(model), array_display(map));
		int direction = STextEngine.getCurDirection(null, null, "abc");
		assertEquals("getCurDirection", STextEngine.DIR_LTR, direction);
	}
}





import org.eclipse.equinox.bidi.STextEngine;
import org.eclipse.equinox.bidi.STextEnvironment;
import org.eclipse.equinox.bidi.custom.STextProcessor;

/**
 * Tests some weird cases


	final static STextEnvironment env1 = new STextEnvironment("en_US", false, STextEnvironment.ORIENT_LTR);
	final static STextEnvironment env2 = new STextEnvironment("he", false, STextEnvironment.ORIENT_LTR);
	final static STextFeatures myFeatures = new STextFeatures(null, 1, -1, -1, false, false);

	class Processor1 extends STextProcessor {

		public int getSpecialsCount(STextEnvironment env, String text, byte[] dirProps) {
			return 1;
		}

		public int indexOfSpecial(STextEnvironment env, String text, byte[] dirProps, int[] offsets, int caseNumber, int fromIndex) {
			return fromIndex;
		}

		public int processSpecial(STextEnvironment env, String text, byte[] dirProps, int[] offsets, int[] state, int caseNumber, int separLocation) {
			int len = text.length();
			for (int i = len - 1; i >= 0; i--) {
				STextProcessor.insertMark(text, dirProps, offsets, i);


	class Processor2 extends STextProcessor {

		public int getSpecialsCount(STextEnvironment env, String text, byte[] dirProps) {
			return 1;
		}

	}

	class Processor3 extends STextProcessor {

		public int getSpecialsCount(STextEnvironment env, String text, byte[] dirProps) {
			return 1;
		}

		public int indexOfSpecial(STextEnvironment env, String text, byte[] dirProps, int[] offsets, int caseNumber, int fromIndex) {
			return fromIndex;
		}
	}


		assertFalse(env1.isBidi());
		assertTrue(env2.isBidi());

		STextProcessor processor = new Processor1();
		String full = STextEngine.leanToFullText(processor, env1, "abcd", null);
		assertEquals("@a@b@c@d", toPseudo(full));

		processor = new Processor2();
		boolean catchFlag = false;
		try {
			full = STextEngine.leanToFullText(processor, env1, "abcd", null);
		} catch (IllegalStateException e) {
			catchFlag = true;
		}

		processor = new Processor3();
		catchFlag = false;
		try {
			full = STextEngine.leanToFullText(processor, env1, "abcd", null);
		} catch (IllegalStateException e) {
			catchFlag = true;
		}





package org.eclipse.equinox.bidi.internal.tests;

import org.eclipse.equinox.bidi.STextEngine;
import org.eclipse.equinox.bidi.STextStringRecord;
import org.eclipse.equinox.bidi.custom.STextStringProcessor;

/**
 *	Tests the StringRecord class	

	public void testStringRecord() {
		STextStringRecord sr;
		boolean catchFlag;
		short[] badTriplet1 = new short[] {0, 0};
		short[] badTriplet2 = new short[] {0, 0, -1};
		short[] badTriplet3 = new short[] {0, 0, 999};
		short[] goodTriplet = new short[] {0, 3, 2};
		short[] triplets;
		short type;
		String strType;
		// check handling of invalid arguments
		catchFlag = false;
		try {
			sr = STextStringRecord.addRecord(null, 1, STextEngine.PROC_EMAIL, 0, 1);
		} catch (IllegalArgumentException e) {
			catchFlag = true;
		}
		assertTrue("Catch null string argument", catchFlag);
		catchFlag = false;
		try {
			sr = STextStringRecord.addRecord("abc", 1, null, 0, 1);
		} catch (IllegalArgumentException e) {
			catchFlag = true;
		}
		assertTrue("Catch null processor argument", catchFlag);
		catchFlag = false;
		try {
			sr = STextStringRecord.addRecord("abc", 0, STextEngine.PROC_EMAIL, 0, 1);
		} catch (IllegalArgumentException e) {
			catchFlag = true;
		}
		assertTrue("Catch invalid segment count argument", catchFlag);
		catchFlag = false;
		try {
			sr = STextStringRecord.addRecord("abc", 1, STextEngine.PROC_EMAIL, -1, 1);
		} catch (IllegalArgumentException e) {
			catchFlag = true;
		}
		assertTrue("Catch invalid start argument", catchFlag);
		catchFlag = false;
		try {
			sr = STextStringRecord.addRecord("abc", 1, STextEngine.PROC_EMAIL, 4, 1);
		} catch (IllegalArgumentException e) {
			catchFlag = true;
		}
		assertTrue("Catch invalid start argument", catchFlag);
		catchFlag = false;
		try {
			sr = STextStringRecord.addRecord("abc", 1, STextEngine.PROC_EMAIL, 0, 0);
		} catch (IllegalArgumentException e) {
			catchFlag = true;
		}
		assertTrue("Catch invalid limit argument", catchFlag);
		catchFlag = false;
		try {
			sr = STextStringRecord.addRecord("abc", 1, STextEngine.PROC_EMAIL, 0, 5);
		} catch (IllegalArgumentException e) {
			catchFlag = true;
		}
		assertTrue("Catch invalid limit argument", catchFlag);

		String[] types = STextStringProcessor.getKnownTypes();
		for (int i = 0; i < types.length; i++) {
			type = STextStringRecord.typeStringToShort(types[i]);
			assertFalse(type == -1);
			strType = STextStringRecord.typeShortToString(type);
			assertEquals(types[i], strType);
		}
		type = STextStringRecord.typeStringToShort("dummy");
		assertEquals(-1, type);
		strType = STextStringRecord.typeShortToString((short) 999);
		assertEquals(null, strType);
		int poolSize = STextStringRecord.POOLSIZE;
		int lim = poolSize / 2;
		sr = STextStringRecord.getRecord("XXX");
		assertEquals(null, sr);
		for (int i = 0; i < lim; i++) {
			String str = Integer.toString(i);
			sr = STextStringRecord.addRecord(str, 1, STextEngine.PROC_EMAIL, 0, 1);
			STextStringRecord.add(sr);
		}
		sr = STextStringRecord.getRecord(null);
		assertEquals(null, sr);
		sr = STextStringRecord.getRecord("");
		assertEquals(null, sr);

		for (int i = 0; i < poolSize; i++) {
			String str = Integer.toString(i);
			sr = STextStringRecord.getRecord(str);
			if (i < lim)
				assertFalse(null == sr);
			else
				assertTrue(null == sr);
		}

		for (int i = lim; i <= poolSize; i++) {
			String str = Integer.toString(i);
			sr = STextStringRecord.addRecord(str, 1, STextEngine.PROC_EMAIL, 0, 1);
			STextStringRecord.add(sr);
		}
		for (int i = 1; i <= poolSize; i++) {
			String str = Integer.toString(i);
			sr = STextStringRecord.getRecord(str);
			assertFalse(null == sr);
		}
		sr = STextStringRecord.getRecord("0");
		assertEquals(null, sr);
		sr = STextStringRecord.addRecord("thisisalongstring", 3, STextEngine.PROC_EMAIL, 0, 2);
		sr.addSegment(STextEngine.PROC_JAVA, 4, 5);
		sr.addSegment(STextEngine.PROC_FILE, 6, 7);
		catchFlag = false;
		try {
			sr.addSegment(STextEngine.PROC_EMAIL, 10, 13);
		} catch (IllegalStateException e) {
			catchFlag = true;
		}
		assertTrue("Catch too many segments", catchFlag);
		assertEquals(3, sr.getSegmentCount());
		assertEquals(STextEngine.PROC_EMAIL, sr.getProcessor(0));
		assertEquals(STextEngine.PROC_JAVA, sr.getProcessor(1));
		assertEquals(STextEngine.PROC_FILE, sr.getProcessor(2));
		assertEquals(0, sr.getStart(0));
		assertEquals(4, sr.getStart(1));
		assertEquals(6, sr.getStart(2));
		assertEquals(2, sr.getLimit(0));
		assertEquals(5, sr.getLimit(1));
		assertEquals(7, sr.getLimit(2));
		catchFlag = false;
		try {
			sr.getLimit(3);
		} catch (IllegalArgumentException e) {
			catchFlag = true;
		}
		assertTrue("Catch segment number too large", catchFlag);

		STextStringRecord.clear();
		for (int i = 0; i <= poolSize; i++) {
			String str = Integer.toString(i);
			sr = STextStringRecord.getRecord(str);
			assertEquals(null, sr);
		}
	}
}




package org.eclipse.equinox.bidi.internal.tests;

import org.eclipse.equinox.bidi.STextEnvironment;
import org.eclipse.equinox.bidi.custom.STextProcessor;
import org.eclipse.equinox.bidi.custom.ISTextProcessor;

public class STextTest extends STextProcessor {

	public String getSeparators(STextEnvironment env, String text, byte[] dirProps) {
		return "-=.:";
	public STextFeatures getFeatures(STextEnvironment env) {
		return FEATURES;
	}

	public int indexOfSpecial(STextFeatures features, String text, byte[] dirProps, int[] offsets, int caseNumber, int fromIndex) {
		throw new IllegalStateException();
	}

	public int processSpecial(STextFeatures features, String text, byte[] dirProps, int[] offsets, int[] state, int caseNumber, int separLocation) {
		throw new IllegalStateException();
	}

}




package org.eclipse.equinox.bidi.internal.tests;

import java.util.Locale;
import org.eclipse.equinox.bidi.STextEngine;
import org.eclipse.equinox.bidi.STextUtil;
import org.eclipse.equinox.bidi.ISTextTypes;

/**
 * Tests methods in BidiComplexUtil

	}

	private void doTest3(String msg, String data, String result, String resLean) {
		String full = STextUtil.processTyped(toUT16(data), STextEngine.PROC_COMMA_DELIMITED);
		assertEquals(msg + "full", result, toPseudo(full));
		String lean = STextUtil.deprocess(full, STextEngine.PROC_COMMA_DELIMITED);
		assertEquals(msg + "lean", resLean, toPseudo(lean));
	}


		doTest3("Util #3.1 - ", "ABC,DEF,G", ">@ABC@,DEF@,G@^");
		doTest3("Util #3.2 - ", "", "");
		doTest3("Util #3.3 - ", ">@DEF@^", ">@DEF@^", "DEF");
		boolean catchFlag = false;
		try {
			STextUtil.deprocess(toUT16("ABC,DE"), "wrong_type");
		} catch (IllegalArgumentException e) {
			catchFlag = true;
		}
		assertTrue("Catch invalid type on deprocess", catchFlag);
		catchFlag = false;
		try {
			STextUtil.processTyped("abc", "wrong_type");
		} catch (IllegalArgumentException e) {
			catchFlag = true;
		}
		assertTrue("Catch invalid type on process", catchFlag);
		// Test insertMarks()
		doTest4("Util #4.1 - ", "ABCDEFG", new int[] {3, 6}, 0, false, "ABC@DEF@G");
		doTest4("Util #4.2 - ", "ABCDEFG", new int[] {3, 6}, 0, true, ">@ABC@DEF@G@^");

Return to bug 183164

Lines 11-19 Link Here

(-)src/org/eclipse/equinox/bidi/internal/tests/STextExtensionsTest.java (-32 / +74 lines)
11		11
12	package org.eclipse.equinox.bidi.internal.tests;	12	package org.eclipse.equinox.bidi.internal.tests;
13		13
		14	import java.util.Locale;
14	import org.eclipse.equinox.bidi.STextEngine;	15	import org.eclipse.equinox.bidi.STextEngine;
15	import org.eclipse.equinox.bidi.ISTextTypes;	16	import org.eclipse.equinox.bidi.custom.STextProcessor;
16	import org.eclipse.equinox.bidi.custom.*;
17		17
18	/**	18	/**
19	* Tests all plug-in extensions	19	* Tests all plug-in extensions
Lines 21-67 Link Here
21		21
22	public class STextExtensionsTest extends STextTestBase {	22	public class STextExtensionsTest extends STextTestBase {
23		23
24	ISTextProcessor processor;	24	STextProcessor processor;
25	STextFeatures f1, f2;
26	int[] state = new int[1];	25	int[] state = new int[1];
27	String type;
28		26
29	private void doTest1(String label, String data, String result) {	27	private void doTest1(String label, String data, String result) {
30	String full;	28	String full;
31	full = STextEngine.leanToFullText(type, null, null, toUT16(data), state);	29	full = STextEngine.leanToFullText(processor, null, toUT16(data), state);
32	assertEquals(label + " data = " + data, result, toPseudo(full));	30	assertEquals(label + " data = " + data, result, toPseudo(full));
33	}	31	}
34		32
35	private void doTest2(String label, String data, String result) {	33	private void doTest2(String label, String data, String result) {
36	String full;	34	String full;
37	full = STextEngine.leanToFullText(type, null, null, data, state);	35	full = STextEngine.leanToFullText(processor, null, data, state);
38	assertEquals(label + " data = " + data, result, toPseudo(full));	36	assertEquals(label + " data = " + data, result, toPseudo(full));
39	}	37	}
40		38
41	private void doTest3(String label, String data, String result) {	39	private void doTest3(String label, String data, String result) {
42	String full;	40	String full;
43	full = STextEngine.leanToFullText(processor, f2, null, toUT16(data), state);	41	full = STextEngine.leanToFullText(processor, null, toUT16(data), state);
44	assertEquals(label + " data = " + data, result, toPseudo(full));	42	assertEquals(label + " data = " + data, result, toPseudo(full));
45	}	43	}
46		44
47	public void testExtensions() {	45	public void testExtensions() {
48		46
49	type = ISTextTypes.COMMA_DELIMITED;	47	String data;
		48	processor = STextEngine.PROC_COMMA_DELIMITED;
50	state[0] = STextEngine.STATE_INITIAL;	49	state[0] = STextEngine.STATE_INITIAL;
51	doTest1("Comma #1", "ab,cd, AB, CD, EFG", "ab,cd, AB@, CD@, EFG");	50	doTest1("Comma #1", "ab,cd, AB, CD, EFG", "ab,cd, AB@, CD@, EFG");
52		51
53	type = ISTextTypes.EMAIL;	52	processor = STextEngine.PROC_EMAIL;
54	state[0] = STextEngine.STATE_INITIAL;	53	state[0] = STextEngine.STATE_INITIAL;
55	doTest1("Email #1", "abc.DEF:GHI", "abc.DEF@:GHI");	54	doTest1("Email #1", "abc.DEF:GHI", "abc.DEF@:GHI");
56	doTest1("Email #2", "DEF.GHI \"A.B\":JK ", "DEF@.GHI @\"A.B\"@:JK ");	55	doTest1("Email #2", "DEF.GHI \"A.B\":JK ", "DEF@.GHI @\"A.B\"@:JK ");
57	doTest1("Email #3", "DEF,GHI (A,B);JK ", "DEF@,GHI @(A,B)@;JK ");	56	doTest1("Email #3", "DEF,GHI (A,B);JK ", "DEF@,GHI @(A,B)@;JK ");
58	doTest1("Email #4", "DEF.GHI (A.B :JK ", "DEF@.GHI @(A.B :JK ");	57	doTest1("Email #4", "DEF.GHI (A.B :JK ", "DEF@.GHI @(A.B :JK ");
		58	Locale.setDefault(new Locale("ar"));
		59	doTest1("Email #5", "#EF.GHI \"A.B\":JK ", "<&#EF.GHI \"A.B\":JK &^");
		60	doTest1("Email #6", "#EF,GHI (A,B);JK ", "<&#EF,GHI (A,B);JK &^");
		61	doTest1("Email #7", "#EF.GHI (A.B :JK ", "<&#EF.GHI (A.B :JK &^");
		62	data = toUT16("peter.pan") + "@" + toUT16("#EF.GHI");
		63	doTest2("Email #8", data, "<&peter&.pan@#EF.GHI&^");
		64	Locale.setDefault(new Locale("he"));
		65	data = toUT16("peter.pan") + "@" + toUT16("DEF.GHI");
		66	doTest2("Email #9", data, "peter.pan@DEF@.GHI");
59		67
60	type = ISTextTypes.FILE;	68	processor = STextEngine.PROC_FILE;
61	state[0] = STextEngine.STATE_INITIAL;	69	state[0] = STextEngine.STATE_INITIAL;
62	doTest1("File #1", "c:\\A\\B\\FILE.EXT", "c:\\A@\\B@\\FILE@.EXT");	70	doTest1("File #1", "c:\\A\\B\\FILE.EXT", "c:\\A@\\B@\\FILE@.EXT");
63		71
64	type = ISTextTypes.JAVA;	72	processor = STextEngine.PROC_JAVA;
65	state[0] = STextEngine.STATE_INITIAL;	73	state[0] = STextEngine.STATE_INITIAL;
66	doTest1("Java #1", "A = B + C;", "A@ = B@ + C;");	74	doTest1("Java #1", "A = B + C;", "A@ = B@ + C;");
67	doTest1("Java #2", "A = B + C;", "A@ = B@ + C;");	75	doTest1("Java #2", "A = B + C;", "A@ = B@ + C;");
Lines 74-87 Link Here
74	doTest1("Java #9", "A = //B+C* D;", "A@ = //B+C* D;");	82	doTest1("Java #9", "A = //B+C* D;", "A@ = //B+C* D;");
75	doTest1("Java #10", "A = //B+C`\|D+E;", "A@ = //B+C`\|D@+E;");	83	doTest1("Java #10", "A = //B+C`\|D+E;", "A@ = //B+C`\|D@+E;");
76		84
77	type = ISTextTypes.PROPERTY;	85	processor = STextEngine.PROC_PROPERTY;
78	state[0] = STextEngine.STATE_INITIAL;	86	state[0] = STextEngine.STATE_INITIAL;
79	doTest1("Property #0", "NAME,VAL1,VAL2", "NAME,VAL1,VAL2");	87	doTest1("Property #0", "NAME,VAL1,VAL2", "NAME,VAL1,VAL2");
80	doTest1("Property #1", "NAME=VAL1,VAL2", "NAME@=VAL1,VAL2");	88	doTest1("Property #1", "NAME=VAL1,VAL2", "NAME@=VAL1,VAL2");
81	doTest1("Property #2", "NAME=VAL1,VAL2=VAL3", "NAME@=VAL1,VAL2=VAL3");	89	doTest1("Property #2", "NAME=VAL1,VAL2=VAL3", "NAME@=VAL1,VAL2=VAL3");
82		90
83	String data;	91	processor = STextEngine.PROC_REGEXP;
84	type = ISTextTypes.REGEXP;
85	state[0] = STextEngine.STATE_INITIAL;	92	state[0] = STextEngine.STATE_INITIAL;
86	data = toUT16("ABC(?") + "#" + toUT16("DEF)GHI");	93	data = toUT16("ABC(?") + "#" + toUT16("DEF)GHI");
87	doTest2("Regex #0.0", data, "A@B@C@(?#DEF)@G@H@I");	94	doTest2("Regex #0.0", data, "A@B@C@(?#DEF)@G@H@I");
Lines 127-133 Link Here
127	doTest1("Regex #17.6", "aB123", "aB@123");	134	doTest1("Regex #17.6", "aB123", "aB@123");
128	doTest1("Regex #17.7", "aB567", "aB@567");	135	doTest1("Regex #17.7", "aB567", "aB@567");
129		136
130	type = ISTextTypes.SQL;	137	Locale.setDefault(new Locale("ar"));
		138	data = toUT16("#BC(?") + "#" + toUT16("DEF)GHI");
		139	doTest2("Regex #0.0", data, "<&#BC(?#DEF)GHI&^");
		140	data = toUT16("#BC(?") + "#" + toUT16("DEF");
		141	doTest2("Regex #0.1", data, "<&#BC(?#DEF&^");
		142	doTest1("Regex #0.2", "#HI)JKL", "<&#HI)JKL&^");
		143	data = toUT16("#BC(?") + "<" + toUT16("DEF") + ">" + toUT16("GHI");
		144	doTest2("Regex #1", data, "<&#BC(?<DEF>GHI&^");
		145	doTest1("Regex #2.0", "#BC(?'DEF'GHI", "<&#BC(?'DEF'GHI&^");
		146	doTest1("Regex #2.1", "#BC(?'DEFGHI", "<&#BC(?'DEFGHI&^");
		147	data = toUT16("#BC(?(") + "<" + toUT16("DEF") + ">" + toUT16(")GHI");
		148	doTest2("Regex #3", data, "<&#BC(?(<DEF>)GHI&^");
		149	doTest1("Regex #4", "#BC(?('DEF')GHI", "<&#BC(?('DEF')GHI&^");
		150	doTest1("Regex #5", "#BC(?(DEF)GHI", "<&#BC(?(DEF)GHI&^");
		151	data = toUT16("#BC(?") + "&" + toUT16("DEF)GHI");
		152	doTest2("Regex #6", data, "<&#BC(?&DEF)GHI&^");
		153	data = toUT16("#BC(?") + "P<" + toUT16("DEF") + ">" + toUT16("GHI");
		154	doTest2("Regex #7", data, "<&#BC(?p<DEF>GHI&^");
		155	data = toUT16("#BC\\k") + "<" + toUT16("DEF") + ">" + toUT16("GHI");
		156	doTest2("Regex #8", data, "<&#BC\\k<DEF>GHI&^");
		157	doTest1("Regex #9", "#BC\\k'DEF'GHI", "<&#BC\\k'DEF'GHI&^");
		158	doTest1("Regex #10", "#BC\\k{DEF}GHI", "<&#BC\\k{DEF}GHI&^");
		159	data = toUT16("#BC(?") + "P=" + toUT16("DEF)GHI");
		160	doTest2("Regex #11", data, "<&#BC(?p=DEF)GHI&^");
		161	doTest1("Regex #12", "#BC\\g{DEF}GHI", "<&#BC\\g{DEF}GHI&^");
		162	data = toUT16("#BC\\g") + "<" + toUT16("DEF") + ">" + toUT16("GHI");
		163	doTest2("Regex #13", data, "<&#BC\\g<DEF>GHI&^");
		164	doTest1("Regex #14", "#BC\\g'DEF'GHI", "<&#BC\\g'DEF'GHI&^");
		165	data = toUT16("#BC(?(") + "R&" + toUT16("DEF)GHI");
		166	doTest2("Regex #15", data, "<&#BC(?(r&DEF)GHI&^");
		167	data = toUT16("#BC") + "\\Q" + toUT16("DEF") + "\\E" + toUT16("GHI");
		168	doTest2("Regex #16.0", data, "<&#BC\\qDEF\\eGHI&^");
		169	data = toUT16("#BC") + "\\Q" + toUT16("DEF");
		170	doTest2("Regex #16.1", data, "<&#BC\\qDEF&^");
		171	data = toUT16("#HI") + "\\E" + toUT16("JKL");
		172	doTest2("Regex #16.2", data, "<&#HI\\eJKL&^");
		173	Locale.setDefault(new Locale("he"));
		174
		175	processor = STextEngine.PROC_SQL;
131	state[0] = STextEngine.STATE_INITIAL;	176	state[0] = STextEngine.STATE_INITIAL;
132	doTest1("SQL #0", "abc GHI", "abc GHI");	177	doTest1("SQL #0", "abc GHI", "abc GHI");
133	doTest1("SQL #1", "abc DEF GHI", "abc DEF@ GHI");	178	doTest1("SQL #1", "abc DEF GHI", "abc DEF@ GHI");
Lines 146-182 Link Here
146	doTest1("SQL #12", "ABC\"DEF \"\" G I\" JKL,MN", "ABC@\"DEF \"\" G I\"@ JKL@,MN");	191	doTest1("SQL #12", "ABC\"DEF \"\" G I\" JKL,MN", "ABC@\"DEF \"\" G I\"@ JKL@,MN");
147	doTest1("SQL #13", "ABC--DEF GHI`\|JKL MN", "ABC@--DEF GHI`\|JKL@ MN");	192	doTest1("SQL #13", "ABC--DEF GHI`\|JKL MN", "ABC@--DEF GHI`\|JKL@ MN");
148		193
149	type = ISTextTypes.SYSTEM_USER;	194	processor = STextEngine.PROC_SYSTEM_USER;
150	state[0] = STextEngine.STATE_INITIAL;	195	state[0] = STextEngine.STATE_INITIAL;
151	doTest1("System #1", "HOST(JACK)", "HOST@(JACK)");	196	doTest1("System #1", "HOST(JACK)", "HOST@(JACK)");
152		197
153	type = ISTextTypes.UNDERSCORE;	198	processor = STextEngine.PROC_UNDERSCORE;
154	state[0] = STextEngine.STATE_INITIAL;	199	state[0] = STextEngine.STATE_INITIAL;
155	doTest1("Underscore #1", "A_B_C_d_e_F_G", "A@_B@_C_d_e_F@_G");	200	doTest1("Underscore #1", "A_B_C_d_e_F_G", "A@_B@_C_d_e_F@_G");
156		201
157	type = ISTextTypes.URL;	202	processor = STextEngine.PROC_URL;
158	state[0] = STextEngine.STATE_INITIAL;	203	state[0] = STextEngine.STATE_INITIAL;
159	doTest1("URL #1", "WWW.DOMAIN.COM/DIR1/DIR2/dir3/DIR4", "WWW@.DOMAIN@.COM@/DIR1@/DIR2/dir3/DIR4");	204	doTest1("URL #1", "WWW.DOMAIN.COM/DIR1/DIR2/dir3/DIR4", "WWW@.DOMAIN@.COM@/DIR1@/DIR2/dir3/DIR4");
160		205
161	type = ISTextTypes.XPATH;	206	processor = STextEngine.PROC_XPATH;
162	state[0] = STextEngine.STATE_INITIAL;	207	state[0] = STextEngine.STATE_INITIAL;
163	doTest1("Xpath #1", "abc(DEF)GHI", "abc(DEF@)GHI");	208	doTest1("Xpath #1", "abc(DEF)GHI", "abc(DEF@)GHI");
164	doTest1("Xpath #2", "DEF.GHI \"A.B\":JK ", "DEF@.GHI@ \"A.B\"@:JK ");	209	doTest1("Xpath #2", "DEF.GHI \"A.B\":JK ", "DEF@.GHI@ \"A.B\"@:JK ");
165	doTest1("Xpath #3", "DEF!GHI 'A!B'=JK ", "DEF@!GHI@ 'A!B'@=JK ");	210	doTest1("Xpath #3", "DEF!GHI 'A!B'=JK ", "DEF@!GHI@ 'A!B'@=JK ");
166	doTest1("Xpath #4", "DEF.GHI 'A.B :JK ", "DEF@.GHI@ 'A.B :JK ");	211	doTest1("Xpath #4", "DEF.GHI 'A.B :JK ", "DEF@.GHI@ 'A.B :JK ");
167		212
168	type = ISTextTypes.EMAIL;	213	processor = STextEngine.PROC_EMAIL;
169	processor = STextStringProcessor.getProcessor(type);
170	state[0] = STextEngine.STATE_INITIAL;	214	state[0] = STextEngine.STATE_INITIAL;
171	f1 = processor.getFeatures(null);	215	assertEquals("<>.:,;@", processor.getSeparators(null, "", null));
172	assertEquals("<>.:,;@", f1.getSeparators());	216	doTest3("DelimsEsc #1", "abc.DEF.GHI", "abc.DEF@.GHI");
173	f2 = new STextFeatures("+-*/", f1.getSpecialsCount(), f1.getDirArabic(), f1.getDirHebrew(), f1.getIgnoreArabic(), f1.getIgnoreHebrew());	217	doTest3("DelimsEsc #2", "DEF.GHI (A:B);JK ", "DEF@.GHI @(A:B)@;JK ");
174	doTest3("DelimsEsc #1", "abc+DEF-GHI", "abc+DEF@-GHI");	218	doTest3("DelimsEsc #3", "DEF.GHI (A:B);JK ", "DEF@.GHI @(A:B)@;JK ");
175	doTest3("DelimsEsc #2", "DEF-GHI (AB)/JK ", "DEF@-GHI @(AB)@/JK ");	219	doTest3("DelimsEsc #4", "DEF.GHI (A:B\\):C) ;JK ", "DEF@.GHI @(A:B\\):C) @;JK ");
176	doTest3("DelimsEsc #3", "DEF-GHI (AB)/JK ", "DEF@-GHI @(AB)@/JK ");	220	doTest3("DelimsEsc #5", "DEF.GHI (A\\\\\\):C) ;JK ", "DEF@.GHI @(A\\\\\\):C) @;JK ");
177	doTest3("DelimsEsc #4", "DEF-GHI (AB\\)C) /JK ", "DEF@-GHI @(AB\\)C) @/JK ");	221	doTest3("DelimsEsc #6", "DEF.GHI (A\\\\):C ;JK ", "DEF@.GHI @(A\\\\)@:C @;JK ");
178	doTest3("DelimsEsc #5", "DEF-GHI (A\\\\\\)C) /JK ", "DEF@-GHI @(A\\\\\\)C) @/JK ");	222	doTest3("DelimsEsc #7", "DEF.GHI (A\\):C ;JK ", "DEF@.GHI @(A\\):C ;JK ");
179	doTest3("DelimsEsc #6", "DEF-GHI (A\\\\)C /JK ", "DEF@-GHI @(A\\\\)@C @/JK ");
180	doTest3("DelimsEsc #7", "DEF-GHI (A\\)C /JK ", "DEF@-GHI @(A\\)C /JK ");
181	}	223	}
182	}	224	}