Breake words by dots

za-arthur · za-arthur · commit 00e1ca897c14 · 2017-08-07T19:26:00.000+03:00
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,3 @@
+*.o
+*.so
+results
diff --git a/expected/pg_tsparser.out b/expected/pg_tsparser.out
@@ -40,6 +40,9 @@ SELECT * FROM ts_parse('tsparser', '345 qwe@efd.r '' http://www.com/ http://aew.
     12 |  ' 
     14 | http://
      6 | www.com
+     1 | www
+    12 | .
+     1 | com
     12 | / 
     14 | http://
      5 | aew.werc.ewr/?ad=qwe&dw
@@ -51,6 +54,12 @@ SELECT * FROM ts_parse('tsparser', '345 qwe@efd.r '' http://www.com/ http://aew.
     18 | /?ad=qwe&dw
     12 |  
      6 | 2aew.werc.ewr
+     3 | 2aew
+    12 | .
+     6 | werc.ewr
+     1 | werc
+    12 | .
+     1 | ewr
     12 |  
     14 | http://
      5 | 3aew.werc.ewr/?ad=qwe&dw
@@ -59,6 +68,12 @@ SELECT * FROM ts_parse('tsparser', '345 qwe@efd.r '' http://www.com/ http://aew.
     12 |  
     14 | http://
      6 | 4aew.werc.ewr
+     3 | 4aew
+    12 | .
+     6 | werc.ewr
+     1 | werc
+    12 | .
+     1 | ewr
     12 |  
     14 | http://
      5 | 5aew.werc.ewr:8100/?
@@ -177,7 +192,7 @@ SELECT * FROM ts_parse('tsparser', '345 qwe@efd.r '' http://www.com/ http://aew.
     12 |  
     12 | <> 
      1 | qwerty
-(143 rows)
+(158 rows)
 
 -- Test text search configuration with parser
 CREATE TEXT SEARCH CONFIGURATION english_ts (
@@ -209,3 +224,15 @@ SELECT to_tsvector('english_ts', '12-abc');
  '12':2 '12-abc':1 'abc':3
 (1 row)
 
+SELECT to_tsvector('english_ts', 'test.com');
+          to_tsvector          
+-------------------------------
+ 'com':3 'test':2 'test.com':1
+(1 row)
+
+SELECT to_tsvector('english_ts', 'test2.com');
+           to_tsvector           
+---------------------------------
+ 'com':3 'test2':2 'test2.com':1
+(1 row)
+
diff --git a/sql/pg_tsparser.sql b/sql/pg_tsparser.sql
@@ -24,3 +24,5 @@ ALTER TEXT SEARCH CONFIGURATION english_ts
 SELECT to_tsvector('english_ts', 'pg_trgm');
 SELECT to_tsvector('english_ts', '12_abc');
 SELECT to_tsvector('english_ts', '12-abc');
+SELECT to_tsvector('english_ts', 'test.com');
+SELECT to_tsvector('english_ts', 'test2.com');
diff --git a/tsparser.c b/tsparser.c
@@ -171,6 +171,8 @@ typedef enum
 	TPS_InPort,
 	TPS_InHostFirstAN,
 	TPS_InHost,
+	TPS_InHostAsciiWord,
+	TPS_InHostNumWord,
 	TPS_InEmail,
 	TPS_InFileFirst,
 	TPS_InFileTwiddle,
@@ -1443,7 +1445,7 @@ static const TParserStateActionItem actionTPS_InHostDomainSecond[] = {
 };
 
 static const TParserStateActionItem actionTPS_InHostDomain[] = {
-	{p_isEOF, 0, A_BINGO | A_CLRALL, TPS_Base, HOST, NULL},
+	{p_isEOF, 0, A_BINGO | A_CLRALL, TPS_InHostAsciiWord, HOST, SpecialHyphen},
 	{p_isasclet, 0, A_NEXT, TPS_InHostDomain, 0, NULL},
 	{p_isdigit, 0, A_PUSH, TPS_InHost, 0, NULL},
 	{p_iseqC, ':', A_PUSH, TPS_InPortFirst, 0, NULL},
@@ -1454,7 +1456,22 @@ static const TParserStateActionItem actionTPS_InHostDomain[] = {
 	{p_isdigit, 0, A_POP, TPS_Null, 0, NULL},
 	{p_isstophost, 0, A_BINGO | A_CLRALL, TPS_InURLPathStart, HOST, NULL},
 	{p_iseqC, '/', A_PUSH, TPS_InFURL, 0, NULL},
-	{NULL, 0, A_BINGO | A_CLRALL, TPS_Base, HOST, NULL}
+	{NULL, 0, A_BINGO | A_CLRALL, TPS_InHostAsciiWord, HOST, SpecialHyphen}
+};
+
+static const TParserStateActionItem actionTPS_InHostAsciiWord[] = {
+	{p_isEOF, 0, A_BINGO, TPS_Base, ASCIIWORD, NULL},
+	{p_isasclet, 0, A_NEXT, TPS_Null, 0, NULL},
+	{p_isdigit, 0, A_NEXT, TPS_InHostNumWord, 0, NULL},
+	{p_isalpha, 0, A_NEXT, TPS_InWord, 0, NULL},
+	{p_isspecial, 0, A_NEXT, TPS_InWord, 0, NULL},
+	{NULL, 0, A_BINGO, TPS_Base, ASCIIWORD, NULL}
+};
+
+static const TParserStateActionItem actionTPS_InHostNumWord[] = {
+	{p_isEOF, 0, A_BINGO, TPS_Base, NUMWORD, NULL},
+	{p_isalnum, 0, A_NEXT, TPS_InHostNumWord, 0, NULL},
+	{NULL, 0, A_BINGO, TPS_Base, NUMWORD, NULL}
 };
 
 static const TParserStateActionItem actionTPS_InPortFirst[] = {
@@ -1782,6 +1799,8 @@ static const TParserStateAction Actions[] = {
 	TPARSERSTATEACTION(TPS_InPort),
 	TPARSERSTATEACTION(TPS_InHostFirstAN),
 	TPARSERSTATEACTION(TPS_InHost),
+	TPARSERSTATEACTION(TPS_InHostAsciiWord),
+	TPARSERSTATEACTION(TPS_InHostNumWord),
 	TPARSERSTATEACTION(TPS_InEmail),
 	TPARSERSTATEACTION(TPS_InFileFirst),
 	TPARSERSTATEACTION(TPS_InFileTwiddle),
