regex.h 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739
  1. // Tencent is pleased to support the open source community by making RapidJSON available.
  2. //
  3. // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
  4. //
  5. // Licensed under the MIT License (the "License"); you may not use this file except
  6. // in compliance with the License. You may obtain a copy of the License at
  7. //
  8. // http://opensource.org/licenses/MIT
  9. //
  10. // Unless required by applicable law or agreed to in writing, software distributed
  11. // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
  12. // CONDITIONS OF ANY KIND, either express or implied. See the License for the
  13. // specific language governing permissions and limitations under the License.
  14. #ifndef RAPIDJSON_INTERNAL_REGEX_H_
  15. #define RAPIDJSON_INTERNAL_REGEX_H_
  16. #include "../allocators.h"
  17. #include "../stream.h"
  18. #include "stack.h"
  19. #ifdef __clang__
  20. RAPIDJSON_DIAG_PUSH
  21. RAPIDJSON_DIAG_OFF(padded)
  22. RAPIDJSON_DIAG_OFF(switch-enum)
  23. #elif defined(_MSC_VER)
  24. RAPIDJSON_DIAG_PUSH
  25. RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated
  26. #endif
  27. #ifdef __GNUC__
  28. RAPIDJSON_DIAG_PUSH
  29. RAPIDJSON_DIAG_OFF(effc++)
  30. #endif
  31. #ifndef RAPIDJSON_REGEX_VERBOSE
  32. #define RAPIDJSON_REGEX_VERBOSE 0
  33. #endif
  34. RAPIDJSON_NAMESPACE_BEGIN
  35. namespace internal {
  36. ///////////////////////////////////////////////////////////////////////////////
  37. // DecodedStream
  38. template <typename SourceStream, typename Encoding>
  39. class DecodedStream {
  40. public:
  41. DecodedStream(SourceStream& ss) : ss_(ss), codepoint_() { Decode(); }
  42. unsigned Peek() { return codepoint_; }
  43. unsigned Take() {
  44. unsigned c = codepoint_;
  45. if (c) // No further decoding when '\0'
  46. Decode();
  47. return c;
  48. }
  49. private:
  50. void Decode() {
  51. if (!Encoding::Decode(ss_, &codepoint_))
  52. codepoint_ = 0;
  53. }
  54. SourceStream& ss_;
  55. unsigned codepoint_;
  56. };
  57. ///////////////////////////////////////////////////////////////////////////////
  58. // GenericRegex
  59. static const SizeType kRegexInvalidState = ~SizeType(0); //!< Represents an invalid index in GenericRegex::State::out, out1
  60. static const SizeType kRegexInvalidRange = ~SizeType(0);
  61. template <typename Encoding, typename Allocator>
  62. class GenericRegexSearch;
  63. //! Regular expression engine with subset of ECMAscript grammar.
  64. /*!
  65. Supported regular expression syntax:
  66. - \c ab Concatenation
  67. - \c a|b Alternation
  68. - \c a? Zero or one
  69. - \c a* Zero or more
  70. - \c a+ One or more
  71. - \c a{3} Exactly 3 times
  72. - \c a{3,} At least 3 times
  73. - \c a{3,5} 3 to 5 times
  74. - \c (ab) Grouping
  75. - \c ^a At the beginning
  76. - \c a$ At the end
  77. - \c . Any character
  78. - \c [abc] Character classes
  79. - \c [a-c] Character class range
  80. - \c [a-z0-9_] Character class combination
  81. - \c [^abc] Negated character classes
  82. - \c [^a-c] Negated character class range
  83. - \c [\b] Backspace (U+0008)
  84. - \c \\| \\\\ ... Escape characters
  85. - \c \\f Form feed (U+000C)
  86. - \c \\n Line feed (U+000A)
  87. - \c \\r Carriage return (U+000D)
  88. - \c \\t Tab (U+0009)
  89. - \c \\v Vertical tab (U+000B)
  90. \note This is a Thompson NFA engine, implemented with reference to
  91. Cox, Russ. "Regular Expression Matching Can Be Simple And Fast (but is slow in Java, Perl, PHP, Python, Ruby,...).",
  92. https://swtch.com/~rsc/regexp/regexp1.html
  93. */
  94. template <typename Encoding, typename Allocator = CrtAllocator>
  95. class GenericRegex {
  96. public:
  97. typedef Encoding EncodingType;
  98. typedef typename Encoding::Ch Ch;
  99. template <typename, typename> friend class GenericRegexSearch;
  100. GenericRegex(const Ch* source, Allocator* allocator = 0) :
  101. ownAllocator_(allocator ? 0 : RAPIDJSON_NEW(Allocator)()), allocator_(allocator ? allocator : ownAllocator_),
  102. states_(allocator_, 256), ranges_(allocator_, 256), root_(kRegexInvalidState), stateCount_(), rangeCount_(),
  103. anchorBegin_(), anchorEnd_()
  104. {
  105. GenericStringStream<Encoding> ss(source);
  106. DecodedStream<GenericStringStream<Encoding>, Encoding> ds(ss);
  107. Parse(ds);
  108. }
  109. ~GenericRegex()
  110. {
  111. RAPIDJSON_DELETE(ownAllocator_);
  112. }
  113. bool IsValid() const {
  114. return root_ != kRegexInvalidState;
  115. }
  116. private:
  117. enum Operator {
  118. kZeroOrOne,
  119. kZeroOrMore,
  120. kOneOrMore,
  121. kConcatenation,
  122. kAlternation,
  123. kLeftParenthesis
  124. };
  125. static const unsigned kAnyCharacterClass = 0xFFFFFFFF; //!< For '.'
  126. static const unsigned kRangeCharacterClass = 0xFFFFFFFE;
  127. static const unsigned kRangeNegationFlag = 0x80000000;
  128. struct Range {
  129. unsigned start; //
  130. unsigned end;
  131. SizeType next;
  132. };
  133. struct State {
  134. SizeType out; //!< Equals to kInvalid for matching state
  135. SizeType out1; //!< Equals to non-kInvalid for split
  136. SizeType rangeStart;
  137. unsigned codepoint;
  138. };
  139. struct Frag {
  140. Frag(SizeType s, SizeType o, SizeType m) : start(s), out(o), minIndex(m) {}
  141. SizeType start;
  142. SizeType out; //!< link-list of all output states
  143. SizeType minIndex;
  144. };
  145. State& GetState(SizeType index) {
  146. RAPIDJSON_ASSERT(index < stateCount_);
  147. return states_.template Bottom<State>()[index];
  148. }
  149. const State& GetState(SizeType index) const {
  150. RAPIDJSON_ASSERT(index < stateCount_);
  151. return states_.template Bottom<State>()[index];
  152. }
  153. Range& GetRange(SizeType index) {
  154. RAPIDJSON_ASSERT(index < rangeCount_);
  155. return ranges_.template Bottom<Range>()[index];
  156. }
  157. const Range& GetRange(SizeType index) const {
  158. RAPIDJSON_ASSERT(index < rangeCount_);
  159. return ranges_.template Bottom<Range>()[index];
  160. }
  161. template <typename InputStream>
  162. void Parse(DecodedStream<InputStream, Encoding>& ds) {
  163. Stack<Allocator> operandStack(allocator_, 256); // Frag
  164. Stack<Allocator> operatorStack(allocator_, 256); // Operator
  165. Stack<Allocator> atomCountStack(allocator_, 256); // unsigned (Atom per parenthesis)
  166. *atomCountStack.template Push<unsigned>() = 0;
  167. unsigned codepoint;
  168. while (ds.Peek() != 0) {
  169. switch (codepoint = ds.Take()) {
  170. case '^':
  171. anchorBegin_ = true;
  172. break;
  173. case '$':
  174. anchorEnd_ = true;
  175. break;
  176. case '|':
  177. while (!operatorStack.Empty() && *operatorStack.template Top<Operator>() < kAlternation)
  178. if (!Eval(operandStack, *operatorStack.template Pop<Operator>(1)))
  179. return;
  180. *operatorStack.template Push<Operator>() = kAlternation;
  181. *atomCountStack.template Top<unsigned>() = 0;
  182. break;
  183. case '(':
  184. *operatorStack.template Push<Operator>() = kLeftParenthesis;
  185. *atomCountStack.template Push<unsigned>() = 0;
  186. break;
  187. case ')':
  188. while (!operatorStack.Empty() && *operatorStack.template Top<Operator>() != kLeftParenthesis)
  189. if (!Eval(operandStack, *operatorStack.template Pop<Operator>(1)))
  190. return;
  191. if (operatorStack.Empty())
  192. return;
  193. operatorStack.template Pop<Operator>(1);
  194. atomCountStack.template Pop<unsigned>(1);
  195. ImplicitConcatenation(atomCountStack, operatorStack);
  196. break;
  197. case '?':
  198. if (!Eval(operandStack, kZeroOrOne))
  199. return;
  200. break;
  201. case '*':
  202. if (!Eval(operandStack, kZeroOrMore))
  203. return;
  204. break;
  205. case '+':
  206. if (!Eval(operandStack, kOneOrMore))
  207. return;
  208. break;
  209. case '{':
  210. {
  211. unsigned n, m;
  212. if (!ParseUnsigned(ds, &n))
  213. return;
  214. if (ds.Peek() == ',') {
  215. ds.Take();
  216. if (ds.Peek() == '}')
  217. m = kInfinityQuantifier;
  218. else if (!ParseUnsigned(ds, &m) || m < n)
  219. return;
  220. }
  221. else
  222. m = n;
  223. if (!EvalQuantifier(operandStack, n, m) || ds.Peek() != '}')
  224. return;
  225. ds.Take();
  226. }
  227. break;
  228. case '.':
  229. PushOperand(operandStack, kAnyCharacterClass);
  230. ImplicitConcatenation(atomCountStack, operatorStack);
  231. break;
  232. case '[':
  233. {
  234. SizeType range;
  235. if (!ParseRange(ds, &range))
  236. return;
  237. SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, kRangeCharacterClass);
  238. GetState(s).rangeStart = range;
  239. *operandStack.template Push<Frag>() = Frag(s, s, s);
  240. }
  241. ImplicitConcatenation(atomCountStack, operatorStack);
  242. break;
  243. case '\\': // Escape character
  244. if (!CharacterEscape(ds, &codepoint))
  245. return; // Unsupported escape character
  246. // fall through to default
  247. RAPIDJSON_DELIBERATE_FALLTHROUGH;
  248. default: // Pattern character
  249. PushOperand(operandStack, codepoint);
  250. ImplicitConcatenation(atomCountStack, operatorStack);
  251. }
  252. }
  253. while (!operatorStack.Empty())
  254. if (!Eval(operandStack, *operatorStack.template Pop<Operator>(1)))
  255. return;
  256. // Link the operand to matching state.
  257. if (operandStack.GetSize() == sizeof(Frag)) {
  258. Frag* e = operandStack.template Pop<Frag>(1);
  259. Patch(e->out, NewState(kRegexInvalidState, kRegexInvalidState, 0));
  260. root_ = e->start;
  261. #if RAPIDJSON_REGEX_VERBOSE
  262. printf("root: %d\n", root_);
  263. for (SizeType i = 0; i < stateCount_ ; i++) {
  264. State& s = GetState(i);
  265. printf("[%2d] out: %2d out1: %2d c: '%c'\n", i, s.out, s.out1, (char)s.codepoint);
  266. }
  267. printf("\n");
  268. #endif
  269. }
  270. }
  271. SizeType NewState(SizeType out, SizeType out1, unsigned codepoint) {
  272. State* s = states_.template Push<State>();
  273. s->out = out;
  274. s->out1 = out1;
  275. s->codepoint = codepoint;
  276. s->rangeStart = kRegexInvalidRange;
  277. return stateCount_++;
  278. }
  279. void PushOperand(Stack<Allocator>& operandStack, unsigned codepoint) {
  280. SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, codepoint);
  281. *operandStack.template Push<Frag>() = Frag(s, s, s);
  282. }
  283. void ImplicitConcatenation(Stack<Allocator>& atomCountStack, Stack<Allocator>& operatorStack) {
  284. if (*atomCountStack.template Top<unsigned>())
  285. *operatorStack.template Push<Operator>() = kConcatenation;
  286. (*atomCountStack.template Top<unsigned>())++;
  287. }
  288. SizeType Append(SizeType l1, SizeType l2) {
  289. SizeType old = l1;
  290. while (GetState(l1).out != kRegexInvalidState)
  291. l1 = GetState(l1).out;
  292. GetState(l1).out = l2;
  293. return old;
  294. }
  295. void Patch(SizeType l, SizeType s) {
  296. for (SizeType next; l != kRegexInvalidState; l = next) {
  297. next = GetState(l).out;
  298. GetState(l).out = s;
  299. }
  300. }
  301. bool Eval(Stack<Allocator>& operandStack, Operator op) {
  302. switch (op) {
  303. case kConcatenation:
  304. RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag) * 2);
  305. {
  306. Frag e2 = *operandStack.template Pop<Frag>(1);
  307. Frag e1 = *operandStack.template Pop<Frag>(1);
  308. Patch(e1.out, e2.start);
  309. *operandStack.template Push<Frag>() = Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex));
  310. }
  311. return true;
  312. case kAlternation:
  313. if (operandStack.GetSize() >= sizeof(Frag) * 2) {
  314. Frag e2 = *operandStack.template Pop<Frag>(1);
  315. Frag e1 = *operandStack.template Pop<Frag>(1);
  316. SizeType s = NewState(e1.start, e2.start, 0);
  317. *operandStack.template Push<Frag>() = Frag(s, Append(e1.out, e2.out), Min(e1.minIndex, e2.minIndex));
  318. return true;
  319. }
  320. return false;
  321. case kZeroOrOne:
  322. if (operandStack.GetSize() >= sizeof(Frag)) {
  323. Frag e = *operandStack.template Pop<Frag>(1);
  324. SizeType s = NewState(kRegexInvalidState, e.start, 0);
  325. *operandStack.template Push<Frag>() = Frag(s, Append(e.out, s), e.minIndex);
  326. return true;
  327. }
  328. return false;
  329. case kZeroOrMore:
  330. if (operandStack.GetSize() >= sizeof(Frag)) {
  331. Frag e = *operandStack.template Pop<Frag>(1);
  332. SizeType s = NewState(kRegexInvalidState, e.start, 0);
  333. Patch(e.out, s);
  334. *operandStack.template Push<Frag>() = Frag(s, s, e.minIndex);
  335. return true;
  336. }
  337. return false;
  338. case kOneOrMore:
  339. if (operandStack.GetSize() >= sizeof(Frag)) {
  340. Frag e = *operandStack.template Pop<Frag>(1);
  341. SizeType s = NewState(kRegexInvalidState, e.start, 0);
  342. Patch(e.out, s);
  343. *operandStack.template Push<Frag>() = Frag(e.start, s, e.minIndex);
  344. return true;
  345. }
  346. return false;
  347. default:
  348. // syntax error (e.g. unclosed kLeftParenthesis)
  349. return false;
  350. }
  351. }
  352. bool EvalQuantifier(Stack<Allocator>& operandStack, unsigned n, unsigned m) {
  353. RAPIDJSON_ASSERT(n <= m);
  354. RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag));
  355. if (n == 0) {
  356. if (m == 0) // a{0} not support
  357. return false;
  358. else if (m == kInfinityQuantifier)
  359. Eval(operandStack, kZeroOrMore); // a{0,} -> a*
  360. else {
  361. Eval(operandStack, kZeroOrOne); // a{0,5} -> a?
  362. for (unsigned i = 0; i < m - 1; i++)
  363. CloneTopOperand(operandStack); // a{0,5} -> a? a? a? a? a?
  364. for (unsigned i = 0; i < m - 1; i++)
  365. Eval(operandStack, kConcatenation); // a{0,5} -> a?a?a?a?a?
  366. }
  367. return true;
  368. }
  369. for (unsigned i = 0; i < n - 1; i++) // a{3} -> a a a
  370. CloneTopOperand(operandStack);
  371. if (m == kInfinityQuantifier)
  372. Eval(operandStack, kOneOrMore); // a{3,} -> a a a+
  373. else if (m > n) {
  374. CloneTopOperand(operandStack); // a{3,5} -> a a a a
  375. Eval(operandStack, kZeroOrOne); // a{3,5} -> a a a a?
  376. for (unsigned i = n; i < m - 1; i++)
  377. CloneTopOperand(operandStack); // a{3,5} -> a a a a? a?
  378. for (unsigned i = n; i < m; i++)
  379. Eval(operandStack, kConcatenation); // a{3,5} -> a a aa?a?
  380. }
  381. for (unsigned i = 0; i < n - 1; i++)
  382. Eval(operandStack, kConcatenation); // a{3} -> aaa, a{3,} -> aaa+, a{3.5} -> aaaa?a?
  383. return true;
  384. }
  385. static SizeType Min(SizeType a, SizeType b) { return a < b ? a : b; }
  386. void CloneTopOperand(Stack<Allocator>& operandStack) {
  387. const Frag src = *operandStack.template Top<Frag>(); // Copy constructor to prevent invalidation
  388. SizeType count = stateCount_ - src.minIndex; // Assumes top operand contains states in [src->minIndex, stateCount_)
  389. State* s = states_.template Push<State>(count);
  390. memcpy(s, &GetState(src.minIndex), count * sizeof(State));
  391. for (SizeType j = 0; j < count; j++) {
  392. if (s[j].out != kRegexInvalidState)
  393. s[j].out += count;
  394. if (s[j].out1 != kRegexInvalidState)
  395. s[j].out1 += count;
  396. }
  397. *operandStack.template Push<Frag>() = Frag(src.start + count, src.out + count, src.minIndex + count);
  398. stateCount_ += count;
  399. }
  400. template <typename InputStream>
  401. bool ParseUnsigned(DecodedStream<InputStream, Encoding>& ds, unsigned* u) {
  402. unsigned r = 0;
  403. if (ds.Peek() < '0' || ds.Peek() > '9')
  404. return false;
  405. while (ds.Peek() >= '0' && ds.Peek() <= '9') {
  406. if (r >= 429496729 && ds.Peek() > '5') // 2^32 - 1 = 4294967295
  407. return false; // overflow
  408. r = r * 10 + (ds.Take() - '0');
  409. }
  410. *u = r;
  411. return true;
  412. }
  413. template <typename InputStream>
  414. bool ParseRange(DecodedStream<InputStream, Encoding>& ds, SizeType* range) {
  415. bool isBegin = true;
  416. bool negate = false;
  417. int step = 0;
  418. SizeType start = kRegexInvalidRange;
  419. SizeType current = kRegexInvalidRange;
  420. unsigned codepoint;
  421. while ((codepoint = ds.Take()) != 0) {
  422. if (isBegin) {
  423. isBegin = false;
  424. if (codepoint == '^') {
  425. negate = true;
  426. continue;
  427. }
  428. }
  429. switch (codepoint) {
  430. case ']':
  431. if (start == kRegexInvalidRange)
  432. return false; // Error: nothing inside []
  433. if (step == 2) { // Add trailing '-'
  434. SizeType r = NewRange('-');
  435. RAPIDJSON_ASSERT(current != kRegexInvalidRange);
  436. GetRange(current).next = r;
  437. }
  438. if (negate)
  439. GetRange(start).start |= kRangeNegationFlag;
  440. *range = start;
  441. return true;
  442. case '\\':
  443. if (ds.Peek() == 'b') {
  444. ds.Take();
  445. codepoint = 0x0008; // Escape backspace character
  446. }
  447. else if (!CharacterEscape(ds, &codepoint))
  448. return false;
  449. // fall through to default
  450. RAPIDJSON_DELIBERATE_FALLTHROUGH;
  451. default:
  452. switch (step) {
  453. case 1:
  454. if (codepoint == '-') {
  455. step++;
  456. break;
  457. }
  458. // fall through to step 0 for other characters
  459. RAPIDJSON_DELIBERATE_FALLTHROUGH;
  460. case 0:
  461. {
  462. SizeType r = NewRange(codepoint);
  463. if (current != kRegexInvalidRange)
  464. GetRange(current).next = r;
  465. if (start == kRegexInvalidRange)
  466. start = r;
  467. current = r;
  468. }
  469. step = 1;
  470. break;
  471. default:
  472. RAPIDJSON_ASSERT(step == 2);
  473. GetRange(current).end = codepoint;
  474. step = 0;
  475. }
  476. }
  477. }
  478. return false;
  479. }
  480. SizeType NewRange(unsigned codepoint) {
  481. Range* r = ranges_.template Push<Range>();
  482. r->start = r->end = codepoint;
  483. r->next = kRegexInvalidRange;
  484. return rangeCount_++;
  485. }
  486. template <typename InputStream>
  487. bool CharacterEscape(DecodedStream<InputStream, Encoding>& ds, unsigned* escapedCodepoint) {
  488. unsigned codepoint;
  489. switch (codepoint = ds.Take()) {
  490. case '^':
  491. case '$':
  492. case '|':
  493. case '(':
  494. case ')':
  495. case '?':
  496. case '*':
  497. case '+':
  498. case '.':
  499. case '[':
  500. case ']':
  501. case '{':
  502. case '}':
  503. case '\\':
  504. *escapedCodepoint = codepoint; return true;
  505. case 'f': *escapedCodepoint = 0x000C; return true;
  506. case 'n': *escapedCodepoint = 0x000A; return true;
  507. case 'r': *escapedCodepoint = 0x000D; return true;
  508. case 't': *escapedCodepoint = 0x0009; return true;
  509. case 'v': *escapedCodepoint = 0x000B; return true;
  510. default:
  511. return false; // Unsupported escape character
  512. }
  513. }
  514. Allocator* ownAllocator_;
  515. Allocator* allocator_;
  516. Stack<Allocator> states_;
  517. Stack<Allocator> ranges_;
  518. SizeType root_;
  519. SizeType stateCount_;
  520. SizeType rangeCount_;
  521. static const unsigned kInfinityQuantifier = ~0u;
  522. // For SearchWithAnchoring()
  523. bool anchorBegin_;
  524. bool anchorEnd_;
  525. };
  526. template <typename RegexType, typename Allocator = CrtAllocator>
  527. class GenericRegexSearch {
  528. public:
  529. typedef typename RegexType::EncodingType Encoding;
  530. typedef typename Encoding::Ch Ch;
  531. GenericRegexSearch(const RegexType& regex, Allocator* allocator = 0) :
  532. regex_(regex), allocator_(allocator), ownAllocator_(0),
  533. state0_(allocator, 0), state1_(allocator, 0), stateSet_()
  534. {
  535. RAPIDJSON_ASSERT(regex_.IsValid());
  536. if (!allocator_)
  537. ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)();
  538. stateSet_ = static_cast<unsigned*>(allocator_->Malloc(GetStateSetSize()));
  539. state0_.template Reserve<SizeType>(regex_.stateCount_);
  540. state1_.template Reserve<SizeType>(regex_.stateCount_);
  541. }
  542. ~GenericRegexSearch() {
  543. Allocator::Free(stateSet_);
  544. RAPIDJSON_DELETE(ownAllocator_);
  545. }
  546. template <typename InputStream>
  547. bool Match(InputStream& is) {
  548. return SearchWithAnchoring(is, true, true);
  549. }
  550. bool Match(const Ch* s) {
  551. GenericStringStream<Encoding> is(s);
  552. return Match(is);
  553. }
  554. template <typename InputStream>
  555. bool Search(InputStream& is) {
  556. return SearchWithAnchoring(is, regex_.anchorBegin_, regex_.anchorEnd_);
  557. }
  558. bool Search(const Ch* s) {
  559. GenericStringStream<Encoding> is(s);
  560. return Search(is);
  561. }
  562. private:
  563. typedef typename RegexType::State State;
  564. typedef typename RegexType::Range Range;
  565. template <typename InputStream>
  566. bool SearchWithAnchoring(InputStream& is, bool anchorBegin, bool anchorEnd) {
  567. DecodedStream<InputStream, Encoding> ds(is);
  568. state0_.Clear();
  569. Stack<Allocator> *current = &state0_, *next = &state1_;
  570. const size_t stateSetSize = GetStateSetSize();
  571. std::memset(stateSet_, 0, stateSetSize);
  572. bool matched = AddState(*current, regex_.root_);
  573. unsigned codepoint;
  574. while (!current->Empty() && (codepoint = ds.Take()) != 0) {
  575. std::memset(stateSet_, 0, stateSetSize);
  576. next->Clear();
  577. matched = false;
  578. for (const SizeType* s = current->template Bottom<SizeType>(); s != current->template End<SizeType>(); ++s) {
  579. const State& sr = regex_.GetState(*s);
  580. if (sr.codepoint == codepoint ||
  581. sr.codepoint == RegexType::kAnyCharacterClass ||
  582. (sr.codepoint == RegexType::kRangeCharacterClass && MatchRange(sr.rangeStart, codepoint)))
  583. {
  584. matched = AddState(*next, sr.out) || matched;
  585. if (!anchorEnd && matched)
  586. return true;
  587. }
  588. if (!anchorBegin)
  589. AddState(*next, regex_.root_);
  590. }
  591. internal::Swap(current, next);
  592. }
  593. return matched;
  594. }
  595. size_t GetStateSetSize() const {
  596. return (regex_.stateCount_ + 31) / 32 * 4;
  597. }
  598. // Return whether the added states is a match state
  599. bool AddState(Stack<Allocator>& l, SizeType index) {
  600. RAPIDJSON_ASSERT(index != kRegexInvalidState);
  601. const State& s = regex_.GetState(index);
  602. if (s.out1 != kRegexInvalidState) { // Split
  603. bool matched = AddState(l, s.out);
  604. return AddState(l, s.out1) || matched;
  605. }
  606. else if (!(stateSet_[index >> 5] & (1u << (index & 31)))) {
  607. stateSet_[index >> 5] |= (1u << (index & 31));
  608. *l.template PushUnsafe<SizeType>() = index;
  609. }
  610. return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation.
  611. }
  612. bool MatchRange(SizeType rangeIndex, unsigned codepoint) const {
  613. bool yes = (regex_.GetRange(rangeIndex).start & RegexType::kRangeNegationFlag) == 0;
  614. while (rangeIndex != kRegexInvalidRange) {
  615. const Range& r = regex_.GetRange(rangeIndex);
  616. if (codepoint >= (r.start & ~RegexType::kRangeNegationFlag) && codepoint <= r.end)
  617. return yes;
  618. rangeIndex = r.next;
  619. }
  620. return !yes;
  621. }
  622. const RegexType& regex_;
  623. Allocator* allocator_;
  624. Allocator* ownAllocator_;
  625. Stack<Allocator> state0_;
  626. Stack<Allocator> state1_;
  627. uint32_t* stateSet_;
  628. };
  629. typedef GenericRegex<UTF8<> > Regex;
  630. typedef GenericRegexSearch<Regex> RegexSearch;
  631. } // namespace internal
  632. RAPIDJSON_NAMESPACE_END
  633. #ifdef __GNUC__
  634. RAPIDJSON_DIAG_POP
  635. #endif
  636. #if defined(__clang__) || defined(_MSC_VER)
  637. RAPIDJSON_DIAG_POP
  638. #endif
  639. #endif // RAPIDJSON_INTERNAL_REGEX_H_