Reject a compiled program too big for its offset type

Concatenated large repeats reach a program past INT_MAX from a few hundred
kilobytes of source, and the offsets emitted into it wrap.  Cap the size
while the program is emitted, which does not depend on the length pass that
overflows along with it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

include/ruby/onigmo.h

@@ -689,6 +689,7 @@ ONIG_EXTERN const OnigSyntaxType*   OnigDefaultSyntax;
 #define ONIGERR_INVALID_CHAR_PROPERTY_NAME                   -223
 #define ONIGERR_TOO_MANY_RANGE_REPEAT                        -224
 #define ONIGERR_TOO_MANY_NULL_CHECK                          -225
+#define ONIGERR_TOO_BIG_COMPILED_PROGRAM                     -226
 #define ONIGERR_INVALID_CODE_POINT_VALUE                     -400
 #define ONIGERR_INVALID_WIDE_CHAR_VALUE                      -400
 #define ONIGERR_TOO_BIG_WIDE_CHAR_VALUE                      -401

regcomp.c

@@ -223,6 +223,10 @@ unset_addr_list_add(UnsetAddrList* uslist, int offset, struct _Node* node)
 static int
 add_opcode(regex_t* reg, int opcode)
 {
+  /* Every instruction passes here, so this is where a program that would
+     outgrow its offset type is stopped, while it is still being emitted. */
+  if (reg->used > MAX_COMPILED_PROGRAM_SIZE)
+    return ONIGERR_TOO_BIG_COMPILED_PROGRAM;
   BBUF_ADD1(reg, opcode);
   return 0;
 }

regerror.c

@@ -183,6 +183,8 @@ onig_error_code_to_format(OnigPosition code)
     p = "too many range repeat"; break;
   case ONIGERR_TOO_MANY_NULL_CHECK:
     p = "too many null check"; break;
+  case ONIGERR_TOO_BIG_COMPILED_PROGRAM:
+    p = "too big compiled program"; break;
   case ONIGERR_TOO_MANY_CAPTURE_GROUPS:
     p = "too many capture groups are specified"; break;
   case ONIGERR_INVALID_CODE_POINT_VALUE:

regint.h

@@ -378,6 +378,11 @@ typedef unsigned int  BitStatusType;
 #define INT_MAX_LIMIT           ((1UL << (SIZEOF_INT * 8 - 1)) - 1)
 #define LONG_MAX_LIMIT           ((1UL << (SIZEOF_LONG * 8 - 1)) - 1)
 
+/* Program offsets are RelAddrType, and the compile_length_*() functions sum
+   into int, so a larger program would wrap both.  The quarter leaves room
+   for the sums the offset arithmetic makes out of those lengths. */
+#define MAX_COMPILED_PROGRAM_SIZE  ((unsigned int )(INT_MAX_LIMIT / 4))
+
 #define DIGITVAL(code)    ((code) - '0')
 #define ODIGITVAL(code)   DIGITVAL(code)
 #define XDIGITVAL(enc,code) \