avfilter/vf_stack: reject a grid with more than INT_MAX cells

Regression since: aa8905a1b1
NULL pointer dereference Replicated through UnModified FFmpeg with ASAN

Found during review of the fixes for the security report GIRN0tvt58WQ

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavfilter/vf_stack.c

@@ -116,8 +116,11 @@ static av_cold int init(AVFilterContext *ctx)
             }
         }
 
-        if (is_grid)
+        if (is_grid) {
+            if (s->nb_grid_rows > INT_MAX / s->nb_grid_columns)
+                return AVERROR(EINVAL);
             s->nb_inputs = s->nb_grid_rows * s->nb_grid_columns;
+        }
     }
 
     s->frames = av_calloc(s->nb_inputs, sizeof(*s->frames));