GH-117066: Tier 2 optimizer: Don't throw away good traces if we can't optimize them perfectly. (GH-117067)

Author: markshannon

Python/optimizer_analysis.c

@@ -406,24 +406,28 @@ optimize_uops(
 out_of_space:
     DPRINTF(3, "\n");
     DPRINTF(1, "Out of space in abstract interpreter\n");
-    _Py_uop_abstractcontext_fini(ctx);
-    return 0;
-
+    goto done;
 error:
     DPRINTF(3, "\n");
     DPRINTF(1, "Encountered error in abstract interpreter\n");
     _Py_uop_abstractcontext_fini(ctx);
-    return 0;
+    return -1;
 
 hit_bottom:
     // Attempted to push a "bottom" (contradition) symbol onto the stack.
     // This means that the abstract interpreter has hit unreachable code.
-    // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but it's
-    // simpler to just admit failure and not create the executor.
+    // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting
+    // bottom indicates type instability, so we are probably better off
+    // retrying later.
     DPRINTF(3, "\n");
     DPRINTF(1, "Hit bottom in abstract interpreter\n");
     _Py_uop_abstractcontext_fini(ctx);
     return 0;
+done:
+    /* Cannot optimize further, but there would be no benefit
+     * in retrying later */
+    _Py_uop_abstractcontext_fini(ctx);
+    return 1;
 }
 
 

Python/optimizer_bytecodes.c

@@ -546,7 +546,9 @@ dummy_func(void) {
         PyFunctionObject *func = (PyFunctionObject *)(this_instr + 2)->operand;
         DPRINTF(3, "func: %p ", func);
         if (func == NULL) {
-            goto error;
+            DPRINTF(3, "\n");
+            DPRINTF(1, "Missing function\n");
+            goto done;
         }
         PyCodeObject *co = (PyCodeObject *)func->func_code;