Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
tools/mesa: vcvkk's AIR front end and two ir3 fixes, the ones hakios-metal's compiler is written against
shimoku7409ebcf8a9354
diff --git a/tools/mesa/air-ir3.patch b/tools/mesa/air-ir3.patchindex 823d288..2a496df 100644--- a/tools/mesa/air-ir3.patch+++ b/tools/mesa/air-ir3.patch@@ -29,7 +29,7 @@ dimensions -- and hides on the rest, which is presumably why it survived.#include "pipe/p_context.h"-@@ -247,10 +248,62 @@+@@ -247,10 +248,69 @@return nir;}@@ -68,9 +68,16 @@ dimensions -- and hides on the rest, which is presumably why it survived.++ struct air_constant_value *fc;+ unsigned num_fc = air_module_function_constants(NULL, bitcode, bitcode_size, &fc);++ /* An air.visible library function is not an entry point; the wrapper makes++ * one that calls it, so that these 135 modules can be compiled for the++ * Adreno at all (nir_air.h, docs/air.md section 5). Always on here: this++ * tool exists to answer "does ir3 accept it", and a module that is not an++ * entry point has no other way to be asked.++ */+ const struct air_to_nir_options options = {+ .constants = fc,+ .num_constants = num_fc,++ .library_wrapper = true,+ };+ struct air_shader_info info;+ nir_shader *nir = air_to_nir(NULL, bitcode, bitcode_size, NULL, &options,diff --git a/tools/mesa/air/air2nir.c b/tools/mesa/air/air2nir.cindex c4f89ea..f719cf4 100644--- a/tools/mesa/air/air2nir.c+++ b/tools/mesa/air/air2nir.c@@ -10,6 +10,8 @@* air2nir --ll FILE [FUNCTION] print what the bitcode reader read, as LLVM assembly* air2nir --list FILE.metallib the functions of a container* air2nir --report FILE... one line per module, for tools/air/corpus.py+ * air2nir --interface FILE [FUNC] what the shader expects of the runtime+ * (air_shader_info), key=value per line** FILE is a .metallib (then FUNCTION picks one; without it, every function)* or one module's bitcode, wrapped or not. --report forks for every module,@@ -111,6 +113,20 @@ stage_name(int stage)}}+/* --stitch: a graph of air.visible functions compiled into one entry point.+ *+ * The graph is Metal's own (MTLFunctionStitchingGraph and its nodes) and is+ * typed on the command line the way air_stitch_parse reads it:+ *+ * "_pf_invertRed(in0); _pf_redContour(n0); _pf_textureDiff(n0, n1)"+ *+ * The functions it names are looked up in the container by name, because the+ * nodes of one graph live in different modules of one .metallib.+ */+static const struct air_stitch_graph *stitch = NULL;+static struct air_stitch_function *stitch_fns = NULL;+static unsigned num_stitch_fns = 0;+/* -- evaluating a shader by folding it -------------------------------------** For the class of shader that only reads buffers and computes -- 256 of the@@ -151,7 +167,13 @@ library_byte(unsigned i)static uint8_tpattern_byte(unsigned i){- return library_wrapper ? library_byte(i) : (i * 7 + 13) & 0xff;+ /* A stitched graph reads the wrapper's argument slots, so it is filled the+ * wrapper's way. Not a detail: with the byte pattern a one-node graph gave+ * `1 0 0 1` where the wrapper gave `4.5 0 0 1` for the same function, which+ * reads exactly like a mistranslation and was two harnesses handing the+ * same shader different numbers.+ */+ return (library_wrapper || stitch) ? library_byte(i) : (i * 7 + 13) & 0xff;}struct eval_state {@@ -293,6 +315,55 @@ print_buffer_stores(nir_shader *nir, const struct air_shader_info *info)return printed;}+/* THE CONSTANTS A PIPELINE WOULD HAVE SUPPLIED, for `--fc INDEX=VALUE`.+ *+ * Without this the only constant set this tool can translate against is the+ * one air_module_function_constants builds -- every declared constant, value+ * zero, `supplied` false -- and that is not any pipeline's. It matters because+ * which blocks a shader even HAS now depends on them (plan_global_folding in+ * air_to_nir.c), so "does this module translate" is a question about a+ * configuration and not about the module alone: fixed_frag_lph_cph stops in+ * three different places under three different constant sets.+ *+ * air_run has had this option for a while and is the wrong place to reproduce+ * a translation from: it runs the shader, so a module softpipe cannot execute+ * -- which this one cannot -- aborts in the Gallium driver before anything is+ * printed, and the translation's own verdict is lost with the buffer.+ *+ * 64 and not 32 because fixed_frag_lph_cph declares 58 (the same bound as+ * fc_set in air_run.c and RW_MAX_CONSTANTS in air_metal.h).+ */+static struct {+ unsigned index;+ uint64_t bits;+} fc_set[64];+static unsigned num_fc_set;++/* An index the module does not declare is refused rather than ignored: an+ * option that silently sets nothing would make every run agree with every+ * other one for the wrong reason. Mirrors air_run's apply_fc_values.+ */+static bool+apply_fc_set(struct air_constant_value *fc, unsigned num_fc)+{+ for (unsigned i = 0; i < num_fc_set; i++) {+ unsigned j;+ for (j = 0; j < num_fc; j++) {+ if (fc[j].index == fc_set[i].index) {+ fc[j].bits = fc_set[i].bits;+ fc[j].supplied = true;+ break;+ }+ }+ if (j == num_fc) {+ fprintf(stderr, "air2nir: this module declares no function constant %u "+ "(it declares %u)\n", fc_set[i].index, num_fc);+ return false;+ }+ }+ return true;+}+static inteval_module(const struct module_ref *mod){@@ -300,8 +371,13 @@ eval_module(const struct module_ref *mod)const char *err = NULL;struct air_constant_value *fc;unsigned num_fc = air_module_function_constants(ctx, mod->bitcode, mod->size, &fc);+ if (!apply_fc_set(fc, num_fc))+ return false;struct air_to_nir_options options = { .constants = fc, .num_constants = num_fc,- .library_wrapper = library_wrapper };+ .library_wrapper = library_wrapper,+ .stitch = stitch,+ .stitch_functions = stitch_fns,+ .num_stitch_functions = num_stitch_fns };struct air_shader_info info;nir_shader *nir = air_to_nir(ctx, mod->bitcode, mod->size, NULL, &options,&default_options, &info, &err);@@ -324,8 +400,30 @@ eval_module(const struct module_ref *mod)struct eval_state st = { .info = &info };nir_shader_intrinsics_pass(nir, fold_buffer_loads, nir_metadata_control_flow, &st);- /* fold what the loads made foldable, and nothing else: no algebraic- * rewriting, so the arithmetic is IEEE and comparable with LLVM's+ /* Fold what the loads made foldable, and nothing else: no algebraic+ * rewriting, so the arithmetic is IEEE and comparable with LLVM's.+ *+ * Control flow is simplified, though, and that is not a rewriting of any+ * arithmetic: once a comparison has folded, the branch under it is dead and+ * the phi below it has one live source. Without this, 20 of the corpus's+ * library functions had no folded value at all -- every one of them is+ * straight-line arithmetic behind one `if` -- and a shader with a branch+ * could never be compared against LLVM, which has no such trouble.+ *+ * Dead vector components are dropped for the same reason, and it is the+ * same kind of reason: an `insertelement undef, x, 0` arrives as+ * `vec3 x, undef.y, undef.z`, and a vector with one undefined component in+ * it is not a constant, so the fold stopped there and took everything+ * downstream with it -- although nothing ever reads that component.+ * nir_opt_shrink_vectors removes exactly the components nothing reads and+ * rewrites the swizzles; the lanes that remain are computed by the same+ * instruction over the same bits, so no arithmetic is touched. Four more of+ * the library functions reduce to a value with it: the LLVM comparison of+ * the exactly-defined ones went from 54 of 57 to 57 of 57 (and the three+ * missing sides were ours, not LLVM's), and the softpipe comparison from+ * 120 to 124 -- with the 255 entry points that already agreed with LLVM+ * still agreeing, bit for bit, which is what says the pass changed no+ * number anywhere.*/bool progress;unsigned rounds = 0;@@ -335,6 +433,9 @@ eval_module(const struct module_ref *mod)NIR_PASS(progress, nir, nir_opt_constant_folding);NIR_PASS(progress, nir, nir_opt_dce);NIR_PASS(progress, nir, nir_lower_vars_to_ssa);+ NIR_PASS(progress, nir, nir_opt_dead_cf);+ NIR_PASS(progress, nir, nir_opt_if, nir_opt_if_optimize_phi_true_false);+ NIR_PASS(progress, nir, nir_opt_shrink_vectors, false);} while (progress && ++rounds < 16);printf("eval: the outputs this shader stores:\n");@@ -369,6 +470,87 @@ one_line(const char *s, FILE *fp)fputc(*s == '\n' || *s == '\t' ? ' ' : *s, fp);}+/* --interface: what air_shader_info says the shader expects of the runtime,+ * which nir_print_shader cannot show because it is not in the NIR. Its own+ * mode rather than a line added to the others: the report's lines are parsed+ * by tools/air/corpus.py and the translated NIR by tools/air/fbfetchcheck.py,+ * and a field added to either would move numbers that are a baseline.+ *+ * One `key=value` per line, tab-separated, so a check can read it.+ */+static int+interface_module(const struct module_ref *mod)+{+ void *ctx = ralloc_context(NULL);+ const char *err = NULL;+ struct air_constant_value *fc;+ unsigned num_fc = air_module_function_constants(ctx, mod->bitcode, mod->size, &fc);+ if (!apply_fc_set(fc, num_fc))+ return false;+ struct air_to_nir_options options = { .constants = fc, .num_constants = num_fc,+ .library_wrapper = library_wrapper,+ .stitch = stitch,+ .stitch_functions = stitch_fns,+ .num_stitch_functions = num_stitch_fns };+ struct air_shader_info info;+ nir_shader *nir = air_to_nir(ctx, mod->bitcode, mod->size, NULL, &options,+ &default_options, &info, &err);+ if (!nir) {+ fprintf(stderr, "air2nir: %s: %s\n", mod->name, err);+ ralloc_free(ctx);+ return 1;+ }++ printf("entry=%s\tstage=%s\n", info.entry_point, gl_shader_stage_name(info.stage));+ printf("inputs_read=0x%" PRIx64 "\n", (uint64_t)nir->info.inputs_read);+ printf("vertex_attributes=%u\tvertex_attributes_read=0x%x\n",+ info.num_vertex_attributes, info.vertex_attributes_read);+ for (unsigned i = 0; i < info.num_vertex_attributes; i++) {+ const struct air_vertex_attribute *a = &info.vertex_attributes[i];+ printf("attribute\tindex=%u\tlocation=%s\tdriver_location=%d\tcomponents=%u"+ "\tbits=%u\ttype=%s\tname=%s\tdecl=%s\tused=%s\tfc=%s\n",+ a->index, gl_vert_attrib_name((gl_vert_attrib)a->location),+ a->driver_location == ~0u ? -1 : (int)a->driver_location, a->components,+ a->bit_size,+ a->base_type == AIR_ATTRIB_FLOAT ? "float" :+ a->base_type == AIR_ATTRIB_UINT ? "uint" : "int",+ a->name ? a->name : "-", a->type_name ? a->type_name : "-",+ a->used ? "yes" : "no",+ a->function_constant ? a->function_constant : "-");+ }+ for (unsigned i = 0; i < info.num_varyings; i++)+ printf("varying\tslot=%u\tcomponents=%u\tname=%s\n", info.varyings[i].slot,+ info.varyings[i].components,+ info.varyings[i].name ? info.varyings[i].name : "-");+ for (unsigned i = 0; i < info.num_resources; i++) {+ const struct air_resource *r = &info.resources[i];+ /* set and id are the Metal name of an argument-buffer member, which is+ * the one case where index is this frontend's number and not Metal's --+ * so a check can see both halves of the mapping and not just the one it+ * would have guessed (tools/air/argbufcheck.py)+ *+ * write and ir_write are two readings of one fact and are printed apart+ * for the same reason: write is what Metal declares the shader may do to+ * the buffer, ir_write is what the translated IR does to it, and a+ * runtime that serves an unaligned `device` buffer by uploading a copy+ * needs the second one with the first beside it as a cross-check.+ */+ printf("resource\tkind=%d\tindex=%u\tsize=%u\tname=%s\tset=%d\tid=%d"+ "\tread=%d\twrite=%d\tir_write=%d\n",+ r->kind, r->index, r->size, r->name ? r->name : "-",+ r->argbuf_set == ~0u ? -1 : (int)r->argbuf_set,+ r->argbuf_id == ~0u ? -1 : (int)r->argbuf_id, r->read, r->write,+ r->ir_write);+ }+ for (unsigned i = 0; i < info.num_notes; i++) {+ fputs("note\t", stdout);+ one_line(info.notes[i], stdout);+ fputc('\n', stdout);+ }+ ralloc_free(ctx);+ return 0;+}+/* tools/air/corpus.py reads these lines: tab-separated key=value */static voidreport_one(const struct module_ref *mod)@@ -400,8 +582,13 @@ report_one(const struct module_ref *mod)*/struct air_constant_value *fc;unsigned num_fc = air_module_function_constants(ctx, mod->bitcode, mod->size, &fc);+ if (!apply_fc_set(fc, num_fc))+ return;struct air_to_nir_options options = { .constants = fc, .num_constants = num_fc,- .library_wrapper = library_wrapper };+ .library_wrapper = library_wrapper,+ .stitch = stitch,+ .stitch_functions = stitch_fns,+ .num_stitch_functions = num_stitch_fns };if (num_fc)printf("\tfc=%u", num_fc);@@ -423,7 +610,18 @@ report_one(const struct module_ref *mod)instrs++;}}- printf("\ttranslate=ok\tnir_instrs=%u\n", instrs);+ printf("\ttranslate=ok\tnir_instrs=%u", instrs);+ /* What was translated with a caveat: the shader was accepted and something+ * other than exactly what it asked for was done (air_shader_info.notes).+ * On the report line rather than on stderr, because the report is the only+ * output anything parses, and a caveat nobody can count is the silence this+ * channel exists to end.+ */+ for (unsigned i = 0; i < info.num_notes; i++) {+ fputs(i ? ";" : "\tnotes=", stdout);+ one_line(info.notes[i], stdout);+ }+ printf("\n");ralloc_free(nir);ralloc_free(ctx);}@@ -431,10 +629,35 @@ report_one(const struct module_ref *mod)intmain(int argc, char **argv){- enum { TRANSLATE, LL, LIST, REPORT, EVAL } mode = TRANSLATE;+ enum { TRANSLATE, LL, LIST, REPORT, EVAL, INTERFACE } mode = TRANSLATE;int i = 1;- while (i < argc && !strcmp(argv[i], "--library"))- library_wrapper = true, i++;+ const char *stitch_text = NULL;+ while (i < argc && (!strcmp(argv[i], "--library") ||+ (!strcmp(argv[i], "--fc") && i + 1 < argc) ||+ (!strcmp(argv[i], "--stitch") && i + 1 < argc))) {+ if (!strcmp(argv[i], "--library")) {+ library_wrapper = true, i++;+ } else if (!strcmp(argv[i], "--fc")) {+ unsigned idx = 0;+ unsigned long long val = 0;+ if (sscanf(argv[i + 1], "%u=0x%llx", &idx, &val) != 2 &&+ sscanf(argv[i + 1], "%u=%llu", &idx, &val) != 2) {+ fprintf(stderr, "air2nir: --fc wants INDEX=VALUE, decimal or 0x\n");+ return 2;+ }+ if (num_fc_set >= ARRAY_SIZE(fc_set)) {+ fprintf(stderr, "air2nir: more than %u function constants\n",+ (unsigned)ARRAY_SIZE(fc_set));+ return 2;+ }+ fc_set[num_fc_set].index = idx;+ fc_set[num_fc_set].bits = val;+ num_fc_set++;+ i += 2;+ } else {+ stitch_text = argv[i + 1], i += 2;+ }+ }if (i < argc && !strcmp(argv[i], "--ll"))mode = LL, i++;else if (i < argc && !strcmp(argv[i], "--list"))@@ -443,10 +666,15 @@ main(int argc, char **argv)mode = REPORT, i++;else if (i < argc && !strcmp(argv[i], "--eval"))mode = EVAL, i++;+ else if (i < argc && !strcmp(argv[i], "--interface"))+ mode = INTERFACE, i++;if (i >= argc) {fprintf(stderr,- "usage: air2nir [--library] [--ll | --list | --report | --eval] "- "FILE [FUNCTION]\n");+ "usage: air2nir [--library] [--stitch GRAPH] "+ "[--ll | --list | --report | --eval | --interface] "+ "FILE [FUNCTION]\n"+ " GRAPH is \"f(in0); g(n0)\": one node per `;', arguments inN, nN, "+ "texN, sampN, bufN\n");return 2;}@@ -454,6 +682,28 @@ main(int argc, char **argv)void *ctx = ralloc_context(NULL);int status = 0;+ /* The module translated is the OUTPUT node's: a stitched graph is still+ * handed to air_to_nir as one module plus a graph, and that module has to+ * be one of the graph's own functions.+ */+ const char *only = NULL;+ if (stitch_text) {+ char why[256] = { 0 };+ size_t size = 0;+ const uint8_t *data = map_file(argv[i], &size);+ stitch = air_stitch_parse(ctx, stitch_text, why, sizeof(why));+ if (stitch && data)+ num_stitch_fns = air_stitch_functions_of(ctx, stitch, data, size, &stitch_fns,+ why, sizeof(why));+ if (!stitch || !data || !num_stitch_fns) {+ fprintf(stderr, "air2nir: --stitch: %s\n", why[0] ? why : "no functions");+ ralloc_free(ctx);+ glsl_type_singleton_decref();+ return 2;+ }+ only = stitch->nodes[stitch->output_node].function;+ }+if (mode == REPORT) {setvbuf(stdout, NULL, _IOLBF, 0);for (; i < argc; i++) {@@ -482,7 +732,8 @@ main(int argc, char **argv)}struct module_ref *mods;- unsigned n = modules_of(ctx, argv[i], i + 1 < argc ? argv[i + 1] : NULL, &mods);+ unsigned n = modules_of(ctx, argv[i],+ only ? only : i + 1 < argc ? argv[i + 1] : NULL, &mods);if (n == 0)return 1;@@ -492,6 +743,10 @@ main(int argc, char **argv)status |= eval_module(&mods[k]);continue;}+ if (mode == INTERFACE) {+ status |= interface_module(&mods[k]);+ continue;+ }if (mode == LIST) {printf("%-60s %-12s %zu\n", mods[k].name,air_metallib_type_name(mods[k].container_type), mods[k].size);@@ -509,8 +764,13 @@ main(int argc, char **argv)}struct air_constant_value *fc;unsigned num_fc = air_module_function_constants(ctx, mods[k].bitcode, mods[k].size, &fc);+ if (!apply_fc_set(fc, num_fc))+ return 1;struct air_to_nir_options options = { .constants = fc, .num_constants = num_fc,- .library_wrapper = library_wrapper };+ .library_wrapper = library_wrapper,+ .stitch = stitch,+ .stitch_functions = stitch_fns,+ .num_stitch_functions = num_stitch_fns };struct air_shader_info info;nir_shader *nir = air_to_nir(ctx, mods[k].bitcode, mods[k].size, NULL, &options,&default_options, &info, &err);diff --git a/tools/mesa/air/air_intrinsics.c b/tools/mesa/air/air_intrinsics.cindex b9b5e31..11e26ce 100644--- a/tools/mesa/air/air_intrinsics.c+++ b/tools/mesa/air/air_intrinsics.c@@ -28,16 +28,31 @@/* the dot-separated parts of an air.* name, after "air." */#define AIR_MAX_PARTS 8+/* 48 and not 32: air.normalize_function_constant_predicate's first part is 37+ * characters, and at 32 split_name refused the whole name rather than the+ * operation -- "a name this cannot take apart", which reads like a malformed+ * module and was a buffer. It is the longest part in Apple's own vocabulary+ * list (section 3.2); the longest multi-part name there,+ * air.next_intersection_query.instancing.triangle_data, has no part over 14.+ */struct air_name {unsigned num;- char part[AIR_MAX_PARTS][32];+ char part[AIR_MAX_PARTS][48];};static boolsplit_name(const char *name, struct air_name *out){out->num = 0;+ out->part[0][0] = 0;const char *p = name + 4; /* past "air." */+ /* "air." with nothing after it names no operation. Without this the loop+ * below never runs, split_name returns true all the same, and the+ * dispatcher compares a part[0] nothing wrote -- which gcc points out as+ * soon as the first comparison is made unconditionally.+ */+ if (!*p)+ return false;while (*p && out->num < AIR_MAX_PARTS) {const char *dot = strchr(p, '.');size_t len = dot ? (size_t)(dot - p) : strlen(p);@@ -59,6 +74,109 @@ is_signed_part(const char *s)return !strcmp(s, "s");}+/* Metal's fmod: the remainder of x/y, with the SIGN OF x, correctly rounded.+ *+ * nir_frem is not that function. Its constant expression is, literally,+ *+ * src0 - src1 * truncf(src0 / src1) (nir_opcodes.py)+ *+ * and ir3 lowers it to the same three instructions (ir3_compiler.c sets+ * .lower_fmod). Two things are wrong with the formula, and+ * tools/air/intrincheck.py counted both: of 264 cases of air.fmod.f32 it+ * reported 47 wrong, 25 of them a sign and 22 of them a number.+ *+ * - THE SIGN OF A ZERO IS LOST. fmod(-0, 5) is -0 and fmod(-1.5, 0.5) is+ * -0, and the subtraction gives +0 for both: -0 - (0.5 * -0) is -0 - -0,+ * which is +0. This is the shape of the air.sign bug recorded further+ * down this file -- a sign of zero that a formula does not carry -- and+ * it is the half of this that is not about precision at all.+ * - THE PRODUCT IS ROUNDED BEFORE THE SUBTRACTION. y * trunc(x/y) needs up+ * to 48 significant bits and a float has 24, so the subtraction is exact+ * but of the wrong product: fmod(1.5, 0x1.fffffep-2) came out 1.19e-07+ * against 8.94e-08, one whole ulp of the dividend, and fmod(1e8, 3) came+ * out 0 where the answer is 1. A number, not a sign.+ *+ * Whose definition is followed here, and why it is Metal's: the Metal Shading+ * Language Specification's table entry for fmod gives the same subtraction+ * formula, but it inherits OpenCL's accuracy table along with it, where fmod+ * is 0 ULP -- an exactly specified function, not a licensed approximation --+ * and OpenCL's edge cases are C99 Annex F's, which say fmod(+-0, y) is +-0.+ * Two references on this machine agree with that reading and none with the+ * formula: this machine's libm answers -0 for fmodf(-0, 0.5) and for+ * fmodf(-1.5, 0.5), and LLVM's own `frem` -- JIT-executed by lli, which is+ * the second engine and not a reading of a document -- answers -0 for both+ * and for frem(-2.5, -0.5). The corpus itself says nothing either way: not+ * one of its 128 libraries contains a single `frem` instruction, because+ * Apple's compiler always emits the call.+ *+ * So:+ *+ * q = trunc(x / y)+ * r = fma(-y, q, x) one rounding, not two+ * if (|r| >= |y|) r -= copysign(|y|, r) q was one too small+ * if (sign r != sign x) r += copysign(|y|, x) q was one too large+ * if (r == 0) r = copysign(0, x) a zero keeps x's sign+ *+ * The fma is the whole of the precision argument: when q is an exact integer+ * the true remainder x - q*y is itself exactly representable (that is why+ * IEEE 754 can require `remainder` to be exact), so ONE rounding of it is no+ * rounding at all. The two corrections are there because fl(x/y) can round+ * across an integer and move trunc by one either way, and because ir3 does+ * not divide: it lowers fdiv to fmul(a, frcp(b)) and frcp is approximate, so+ * a quotient off by one has to be survivable rather than assumed away.+ *+ * WHAT THIS STILL DOES NOT DO, measured and not guessed: q must be exactly+ * representable, which means |x/y| < 2^24 for a float and 2^11 for a half.+ * Above that no correction of a bounded size can help -- fmod(1e8, 1e-8)+ * wants a quotient of 1e16 -- and the exact answer needs an iterated+ * reduction: scale y up by 2^k so the partial quotient stays under 2^23,+ * subtract, repeat. Each pass removes only 23 - 1 bits of the exponent+ * difference, that difference reaches 253 for two normal floats, so twelve+ * passes of about ten instructions each would be the price, and its exactness+ * ON THE ADRENO could not be established from this machine at all for the+ * frcp reason above. It is not done here; section 4 of docs/air.md carries+ * the condition and the measurement.+ *+ * Over the reference pool this leaves 7 of 264 cases differing, every one of+ * them with |x/y| >= 2^24; over 195 906 random pairs of normal floats every+ * single disagreement with this machine's libm has |x/y| >= 2^24, and over+ * 376 438 random pairs whose quotient IS representable there is none.+ */+static nir_def *+air_fmod(nir_builder *b, nir_def *x, nir_def *y)+{+ uint64_t sign_bit = 1ull << (x->bit_size - 1);+ /* The sign of a float is taken and planted with integer bit operations and+ * never with a multiply by zero or a copysign built out of one: this+ * builder runs with exact off (Metal's fast library), and+ * nir_opt_algebraic's inexact rule (('~fmul', a, 0.0), 0.0) would throw+ * away exactly the bit being carried. iand and ior cannot be rewritten by+ * a float rule because they are not float operations.+ */+ nir_def *sx = nir_iand_imm(b, x, sign_bit);+ nir_def *ay = nir_fabs(b, y);+ nir_def *y_as_x = nir_ior(b, ay, sx); /* copysign(|y|, x) */++ nir_def *q = nir_ftrunc(b, nir_fdiv(b, x, y));+ nir_def *r = nir_ffma(b, nir_fneg(b, y), q, x);++ /* |r| >= |y|: the quotient was one too small. A NaN fails this comparison+ * (an ordered fge is false for one) and fails the feq against zero at the+ * end; the middle correction may well fire for a NaN, and adding to a NaN+ * gives a NaN, so a NaN comes out of all three unchanged.+ */+ nir_def *y_as_r = nir_ior(b, ay, nir_iand_imm(b, r, sign_bit));+ r = nir_bcsel(b, nir_fge(b, nir_fabs(b, r), ay), nir_fsub(b, r, y_as_r), r);++ /* r's sign is not x's: the quotient was one too large */+ nir_def *wrong_sign = nir_ine(b, nir_iand_imm(b, r, sign_bit), sx);+ r = nir_bcsel(b, nir_iand(b, wrong_sign, nir_fneu_imm(b, r, 0.0)),+ nir_fadd(b, r, y_as_x), r);++ /* and a zero result is the dividend's zero */+ return nir_bcsel(b, nir_feq_imm(b, r, 0.0), sx, r);+}+/* -- the families ------------------------------------------------------- */static bool@@ -130,6 +248,119 @@ math_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n,r = nir_fsin(&c->b, a[0]);} else if (!strcmp(op, "fast_cos") || !strcmp(op, "cos")) {r = nir_fcos(&c->b, a[0]);+ } else if (!strcmp(op, "fast_cospi") || !strcmp(op, "cospi") ||+ !strcmp(op, "fast_sinpi") || !strcmp(op, "sinpi")) {+ /* cos(pi*x) and sin(pi*x), which are their own operations in Apple's+ * vocabulary because multiplying by pi is where the accuracy goes.+ *+ * APPLE'S OWN NAMES, and not the 137 of section 3.2: `air.cospi`,+ * `air.sinpi`, `air.tanpi`, their three `air.fast_*` forms and+ * `air.acospi`, `air.asinpi`, `air.atanpi`, `air.atan2pi` are C strings+ * in the device's shared cache, in the AIR front end's own builtin table+ * (dyld_shared_cache_arm64.17, `air.cospi` at file offset 95 195 301 --+ * the same region carries AIRNTEmitDylibImage and Air.DylibScript). That+ * table spells the whole name, prefix and all, which is why it holds 911+ * single-segment entries where AGXCompilerCore's holds 137 suffixes.+ * `tanpi` and the four inverse ones are deliberately NOT here: a+ * declaration scan over every .metallib of the image finds only+ * cospi.{f16,f32}, fast_cospi.f32 and sinpi.{f16,f32}, and tanpi's poles+ * and zero signs are four more places to be silently wrong for no+ * customer.+ *+ * STEP ONE, AND IT IS EXACT: r = x - 2*rint(x/2) lies in [-1, 1] and is+ * exactly representable for EVERY x, which is why this is not fmul(x, pi)+ * followed by a cosine. Above 2^(p+1) every float is an even integer, so+ * r is zero; below it x and 2*rint(x/2) are within a factor of two of+ * each other, where a subtraction is exact. An infinity reduces to+ * inf - inf = NaN, which is what cospi(inf) is, so there is no edge case+ * to add and none is added.+ *+ * STEP TWO IS A QUARTER AND NOT A HALF, and that is the part worth the+ * instructions. Both functions are near a zero somewhere in [-1, 1], and+ * a cosine evaluated near its own zero has no relative accuracy left:+ * r*pi carries about 1e-7 of absolute error from pi's own rounding, which+ * is nothing beside 1 and everything beside cospi(0.49999997). Measured,+ * on the pool this suite already runs: the half-period form answers+ * 7.55e-08 where the host libm's cospi says 9.36e-08 -- 2 551 410 f32 ulp,+ * on the one input class the operation exists for. So |r| is folded into+ * [0, 1/4], where neither function is near a zero, and which of sin and+ * cos answers is chosen by the quarter it came from. Both are evaluated,+ * because a bcsel is not a branch; the ARGUMENTS are selected first, so+ * the cost is one sine and one cosine and not three.+ *+ * A HALF IS WIDENED TO f32 FIRST, measured and not assumed. pi rounded to+ * f16 is 3.140625, a relative error of 3e-4, and near the zeros that is+ * the whole answer: computed entirely at f16 the half-period form is+ * bit-exact against the host libm's cospi on 50 064 of the 65 536 half+ * inputs with 1 614 wrong by more than four half-ulp (sinpi 42 494) --+ * which is a C model of that formula and not this code, since the+ * un-widened lowering was never built.+ * `tools/air/intrincheck.py --domain16` sweeps that whole domain -- the+ * only width where "does it ever disagree" can be answered rather than+ * sampled -- and this lowering is bit-exact on 65 536 of 65 536 for both.+ *+ * THE SIGN OF sinpi's ZERO is read off the argument's sign BIT and not+ * compared, because -0 == 0. sinpi is zero at every integer with the sign+ * of its argument, which the formula cannot produce on its own: for+ * x = -2 the reduction gives r = +0 and for x = 3 it gives r = -1, so+ * both the even and the odd integers need it. cospi needs no such case --+ * at |r| = 1/2 the middle quarter evaluates sin(0), which is exactly+ * zero.+ */+ bool is_sin = !strcmp(op, "sinpi") || !strcmp(op, "fast_sinpi");+ nir_def *x = a[0];+ bool narrow = a[0]->bit_size == 16;+ if (narrow)+ x = nir_f2f32(&c->b, x);+ unsigned bits = x->bit_size;+ nir_def *n = nir_fround_even(&c->b, nir_fmul_imm(&c->b, x, 0.5));+ nir_def *red = nir_fsub(&c->b, x, nir_fmul_imm(&c->b, n, 2.0));+ nir_def *u = nir_fabs(&c->b, red);+ nir_def *one = nir_imm_floatN_t(&c->b, 1.0, bits);+ /* 1/2 - |r| and 1 - |r| are both exact: Sterbenz, since |r| <= 1 */+ nir_def *half_less = nir_fsub(&c->b, nir_imm_floatN_t(&c->b, 0.5, bits), u);+ nir_def *one_less = nir_fsub(&c->b, one, u);+ /* |r| <= 1/4 and |r| >= 3/4, spelled with the two comparisons the+ * builder has: there is no nir_fle_imm, and `not (u > 1/4)` is not the+ * same predicate as `u <= 1/4` for a NaN -- which is the right way round+ * here, since a NaN must reach the cosine and come back a NaN rather than+ * be routed by a comparison that answered false.+ */+ nir_def *lo = nir_inot(&c->b, nir_flt_imm(&c->b, nir_fneg(&c->b, u), -0.25));+ nir_def *hi = nir_fge_imm(&c->b, u, 0.75);+ if (is_sin) {+ /* |r| in [0, 1/4] or (3/4, 1]: sin of the distance to the nearer+ * integer; in between: cos of the distance to the nearer half+ */+ nir_def *mid = nir_inot(&c->b, nir_ior(&c->b, lo, hi));+ nir_def *as = nir_bcsel(&c->b, lo, u, one_less);+ nir_def *v =+ nir_bcsel(&c->b, mid,+ nir_fcos(&c->b, nir_fmul_imm(&c->b, half_less, M_PI)),+ nir_fsin(&c->b, nir_fmul_imm(&c->b, as, M_PI)));+ v = nir_bcsel(&c->b, nir_flt_imm(&c->b, red, 0.0), nir_fneg(&c->b, v), v);+ r = nir_bcsel(&c->b,+ nir_ior(&c->b, nir_feq_imm(&c->b, u, 0.0),+ nir_feq_imm(&c->b, u, 1.0)),+ nir_iand_imm(&c->b, x, 1ull << (bits - 1)), v);+ } else {+ /* |r| in [0, 1/4] or [3/4, 1]: cos of the distance to the nearer+ * integer, negated past a half; in between: sin of the distance to+ * the nearer half+ */+ nir_def *mid = nir_inot(&c->b, nir_ior(&c->b, lo, hi));+ nir_def *ac = nir_bcsel(&c->b, lo, u, one_less);+ nir_def *v =+ nir_bcsel(&c->b, mid,+ nir_fsin(&c->b, nir_fmul_imm(&c->b, half_less, M_PI)),+ nir_fcos(&c->b, nir_fmul_imm(&c->b, ac, M_PI)));+ /* `hi` alone and not `hi && !mid`: mid is the complement of+ * `lo || hi`, so the two can never both hold+ */+ r = nir_bcsel(&c->b, hi, nir_fneg(&c->b, v), v);+ }+ if (narrow)+ r = nir_f2f16(&c->b, r);} else if (!strcmp(op, "fast_tan") || !strcmp(op, "tan")) {r = nir_fdiv(&c->b, nir_fsin(&c->b, a[0]), nir_fcos(&c->b, a[0]));} else if (!strcmp(op, "fast_atan2") || !strcmp(op, "atan2")) {@@ -162,6 +393,106 @@ math_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n,nir_def *at_origin =nir_bcsel(&c->b, nir_ilt_imm(&c->b, y, 0), nir_fneg(&c->b, mag), mag);r = nir_bcsel(&c->b, origin, at_origin, nir_atan2(&c->b, y, x));+ } else if (!strcmp(op, "fast_asin") || !strcmp(op, "asin") ||+ !strcmp(op, "fast_acos") || !strcmp(op, "acos")) {+ /* The inverse sine and cosine, reduced to atan2, which this file+ * already has and which Mesa implements:+ *+ * asin(x) = atan2(x, sqrt(1 - x*x))+ * acos(x) = atan2(sqrt(1 - x*x), x)+ *+ * The reduction is exact rather than an approximation of its own, which+ * is why it is preferred to the polynomial in Mesa's SPIR-V front end+ * (build_asin in vtn_glsl450.c): that one's own comment says it "isn't+ * precise enough to meet half-float precision requirements" and names+ * this formula as the alternative it rejected for cost. Cost is the+ * right thing to spend here -- air.fast_acos is called by 183 of+ * Resident Evil 4's modules and the shape of a terminator's lighting+ * depends on it.+ *+ * (1 - x)*(1 + x) and not 1 - x*x: the two are the same number in exact+ * arithmetic and not in a float. Near |x| = 1 the subtraction of two+ * close values loses most of the significand -- at x = 1 - 2^-20 the+ * product form keeps the result to a few ulp where 1 - x*x has none of+ * the low bits left -- and |x| near 1 is exactly where a normal dotted+ * with a light direction spends its time.+ *+ * The edges fall out of atan2 rather than needing a case: the second+ * argument is a square root and so never negative, both forms reach+ * atan2(0, +-1) = {0, pi} at x = +-1, |x| > 1 takes the root of a+ * negative and gives NaN as the domain demands, and neither form can+ * reach atan2(0, 0) -- x and 1 - x*x are not both zero -- so the origin+ * this file corrects for plain atan2 is unreachable from here.+ */+ nir_def *x = a[0];+ nir_def *root = nir_fsqrt(&c->b,+ nir_fmul(&c->b, nir_fsub(&c->b, nir_imm_floatN_t(&c->b, 1.0, x->bit_size), x),+ nir_fadd_imm(&c->b, x, 1.0)));+ bool is_asin = !strcmp(op, "fast_asin") || !strcmp(op, "asin");+ r = is_asin ? nir_atan2(&c->b, x, root) : nir_atan2(&c->b, root, x);+ } else if (!strcmp(op, "fast_atan") || !strcmp(op, "atan")) {+ /* one argument: the arc tangent of a ratio already taken, which is+ * Metal's atan(y_over_x) and GLSL's atan(y_over_x) both+ */+ r = nir_atan(&c->b, a[0]);+ } else if (!strcmp(op, "fast_sinh") || !strcmp(op, "sinh")) {+ /* 0.5 * (e^x - e^-x), which is Mesa's own lowering of GLSL sinh+ * (vtn_glsl450.c), with the small-argument case taken out of it.+ *+ * A DIFFERENCE OF TWO EXPONENTIALS HAS NO LOW BITS LEFT NEAR ZERO: at+ * x = 1e-8 both e^x and e^-x round to 1.0 in a float, so the formula+ * answers 0 where sinh(1e-8) is 1e-8 -- not an imprecise answer but a+ * wrong one, and the first thing tools/air/intrincheck.py said about+ * this lowering. sinh(x) = x + x^3/6, so below the x where x^3/6 falls+ * under the type's own precision the argument IS the answer, exactly;+ * 2^-12 is that point for a float (x^2/6 < 2^-24) and 2^-5 for a half.+ * Above it the formula keeps a few thousand ulp of relative error near+ * the threshold, which is what a difference of exponentials costs and+ * what the fast_ prefix licenses.+ */+ unsigned bits = a[0]->bit_size;+ nir_def *big = nir_fge_imm(&c->b, nir_fabs(&c->b, a[0]),+ bits > 16 ? 0x1p-12 : 0x1p-5);+ r = nir_bcsel(&c->b, big,+ nir_fmul_imm(&c->b,+ nir_fsub(&c->b, nir_fexp(&c->b, a[0]),+ nir_fexp(&c->b, nir_fneg(&c->b, a[0]))), 0.5),+ a[0]);+ } else if (!strcmp(op, "fast_cosh") || !strcmp(op, "cosh")) {+ r = nir_fmul_imm(&c->b, nir_fadd(&c->b, nir_fexp(&c->b, a[0]),+ nir_fexp(&c->b, nir_fneg(&c->b, a[0]))), 0.5);+ } else if (!strcmp(op, "fast_tanh") || !strcmp(op, "tanh")) {+ /* (e^x - e^-x) / (e^x + e^-x), with the two guards Mesa's own lowering+ * of GLSL tanh carries and for the same reasons (vtn_glsl450.c):+ *+ * - the argument is clamped to +-10, above which e^x overflows while+ * the answer has been 1.0 to the last bit of a float since |x| = 9;+ * - and the clamp would turn a NaN into 10 and so a NaN into 1, so the+ * result is taken only where the argument is large enough and handed+ * back otherwise. That also keeps tanh(-0) = -0, which the division+ * would give as +0... and this one is EXACTLY the shape of the+ * air.sign bug recorded further up this file: a sign of zero that+ * NIR's folder keeps and a backend need not.+ *+ * The threshold is 2^-12 (2^-5 for a half) and not zero, which is where+ * this differs from Mesa's: tanh(x) = x - x^3/3, so below that point+ * the argument is the answer to the last bit, and above it the+ * difference of exponentials loses the low bits the same way sinh's+ * does. With Mesa's threshold of zero, tanh(1e-8) came back 0.+ */+ unsigned bits = a[0]->bit_size;+ nir_def *x = nir_fmin(&c->b, nir_fmax(&c->b, a[0],+ nir_imm_floatN_t(&c->b, bits > 16 ? -10.0 : -4.2, bits)),+ nir_imm_floatN_t(&c->b, bits > 16 ? 10.0 : 4.2, bits));+ nir_def *ex = nir_fexp(&c->b, x), *enx = nir_fexp(&c->b, nir_fneg(&c->b, x));+ bool was = c->b.exact;+ c->b.exact = true;+ nir_def *regular = nir_fge_imm(&c->b, nir_fabs(&c->b, a[0]),+ bits > 16 ? 0x1p-12 : 0x1p-5);+ c->b.exact = was;+ r = nir_bcsel(&c->b, regular,+ nir_fdiv(&c->b, nir_fsub(&c->b, ex, enx), nir_fadd(&c->b, ex, enx)),+ a[0]);} else if (!strcmp(op, "fast_exp") || !strcmp(op, "exp")) {/* the natural one: Apple's compiler names exp and exp2 separately */r = nir_fexp2(&c->b, nir_fmul_imm(&c->b, a[0], M_LOG2E));@@ -181,7 +512,17 @@ math_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n,r = nir_fpow(&c->b, a[0], a[1]);} else if (!strcmp(op, "fast_fmod") || !strcmp(op, "fmod") ||!strcmp(op, "fast_mod") || !strcmp(op, "mod")) {- r = nir_frem(&c->b, a[0], a[1]);+ /* One lowering for the fast form as well as the plain one. Metal's+ * fast-math does license a lower-precision fmod and does license+ * ignoring the sign of a zero -- 131 of the corpus's 139 fmod+ * declarations are air.fast_fmod.f32, so the cheap form would be legal+ * for nearly all of them -- but a reference that is right for one+ * spelling and relaxed for another is a reference nobody can read, and+ * the extra instructions are about a dozen. If the cost ever matters,+ * the one-step form is nir_frem plus the zero's sign, and the condition+ * under which the two differ is written in air_fmod.+ */+ r = air_fmod(&c->b, a[0], a[1]);} else if (!strcmp(op, "fast_divide") || !strcmp(op, "divide")) {r = nir_fdiv(&c->b, a[0], a[1]);} else if (!strcmp(op, "saturate") || !strcmp(op, "fast_saturate")) {@@ -219,6 +560,34 @@ math_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n,} else if (!strcmp(op, "fast_fmax") || !strcmp(op, "fmax") || !strcmp(op, "max")) {r = is_float ? nir_fmax(&c->b, a[0], a[1]): (is_signed ? nir_imax(&c->b, a[0], a[1]) : nir_umax(&c->b, a[0], a[1]));+ } else if (!strcmp(op, "fast_fmin3") || !strcmp(op, "fmin3") ||+ !strcmp(op, "fast_fmax3") || !strcmp(op, "fmax3")) {+ /* The least or the greatest of THREE, which Metal spells min3/max3 and+ * the Adreno has an instruction for. Built out of the two-argument form+ * above rather than given one of its own, which is exact and not merely+ * convenient: min and max are associative over a total order, and the+ * order the project's reference uses puts -0.0 below +0.0 and has no NaN+ * to order (every module here is compiled nnan), so min(min(a,b),c) is+ * the same value whichever way it is grouped. `clamp` just above is the+ * same composition of the same two operations.+ *+ * 11 of RenderBox's 47 modules call air.fmax3.f16 -- it is how a shape+ * function takes the widest of three coverage estimates -- and+ * air.fmin3.f16 beside it. tools/air/intrincheck.py now carries the+ * reference for both, so each is folded and drawn against it like every+ * other operation here, and tools/air/airdefs.py emits an LLVM body so+ * that a whole module calling one can still be compared on the CPU.+ */+ bool lo = strstr(op, "min") != NULL; /* fmin3 or fmax3 */+ if (is_float)+ r = lo ? nir_fmin(&c->b, nir_fmin(&c->b, a[0], a[1]), a[2])+ : nir_fmax(&c->b, nir_fmax(&c->b, a[0], a[1]), a[2]);+ else if (is_signed)+ r = lo ? nir_imin(&c->b, nir_imin(&c->b, a[0], a[1]), a[2])+ : nir_imax(&c->b, nir_imax(&c->b, a[0], a[1]), a[2]);+ else+ r = lo ? nir_umin(&c->b, nir_umin(&c->b, a[0], a[1]), a[2])+ : nir_umax(&c->b, nir_umax(&c->b, a[0], a[1]), a[2]);} else if (!strcmp(op, "fast_clamp") || !strcmp(op, "clamp")) {if (is_float)r = nir_fmin(&c->b, nir_fmax(&c->b, a[0], a[1]), a[2]);@@ -276,6 +645,163 @@ math_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n,r = nir_ball(&c->b, a[0]);} else if (!strcmp(op, "any")) {r = nir_bany(&c->b, a[0]);+ } else if (!strcmp(op, "simd_sum") || !strcmp(op, "simd_min") ||+ !strcmp(op, "simd_max") || !strcmp(op, "simd_prefix_exclusive_sum")) {+ /* A REDUCTION OVER THE SIMD GROUP, given to every lane. NIR's reduce with+ * cluster_size 0 is that, and ir3 has it (nir_intrinsic_reduce in+ * ir3_compiler_nir.c, emit_intrinsic_reduce) and splits a vector into+ * scalars on the way (nir_lower_subgroups with lower_to_scalar). The+ * prefix scan is nir_exclusive_scan, which the same ir3 function emits+ * from the same `scan` instruction's third destination.+ *+ * FOUR OF APPLE'S SEVENTEEN, and the four are the ones with a customer.+ * This used to read "only the sum, because only the sum occurs", which+ * was true of the corpus and never checked against the image: of the+ * image's 10 242 modules, 9 call air.simd_sum, 4 air.simd_max, 3+ * air.simd_min and 2 air.simd_prefix_exclusive_sum, and the last three+ * names were the FIRST wall of 4 of them (Portrait.framework and+ * MPSImage). The thirteen still missing -- product, and, or, xor, ballot,+ * the inclusive scans, the prefix products -- are called by nothing in+ * either, so each would still be code with no case behind it.+ *+ * What no amount of this can be checked against: a float reduction has no+ * defined order in Metal or in NIR, so a sum over the group is not a+ * number any reference can predict bit-exactly, on the phone or+ * anywhere. Min and max are better off -- they are order-independent for+ * every input that is not a NaN -- and the prefix sum is worse, because+ * it is an order written into the operation's name over a group whose+ * width is the hardware's.+ */+ nir_op rop = is_float ? nir_op_fadd : nir_op_iadd;+ if (!strcmp(op, "simd_min"))+ rop = is_float ? nir_op_fmin : is_signed ? nir_op_imin : nir_op_umin;+ else if (!strcmp(op, "simd_max"))+ rop = is_float ? nir_op_fmax : is_signed ? nir_op_imax : nir_op_umax;+ air_note(c, "air.%s is taken over the SIMD group this implementation has, which "+ "is not Metal's 32 lanes: ir3 runs 64 or 128 of them on this "+ "hardware, so the result is a reduction over more lanes than the "+ "shader was compiled for", op);+ r = !strcmp(op, "simd_prefix_exclusive_sum")+ ? nir_exclusive_scan(&c->b, a[0], .reduction_op = rop)+ : nir_reduce(&c->b, a[0], .reduction_op = rop);+ } else if (!strcmp(op, "normalize_function_constant_predicate")) {+ /* Metal's compiler stores the predicates that say whether a dependent+ * function constant applies in `__metal_implicit_fc_pred_N` bytes, and+ * puts every one through this before storing it. The name is in Apple's+ * own vocabulary (it sits between air.nextafter and air.pack.snorm1x8 in+ * AGXCompilerCore's list of operation names, section 3.2), but no string+ * or symbol in that binary says what it computes, so the reading here is+ * C's: a byte is a predicate if it is not zero.+ *+ * THE CORPUS CANNOT TELL THAT APART FROM THE IDENTITY, and saying so is+ * the point: every argument it is given in these modules is a+ * `zext i1 ... to i8`, already 0 or 1, and any function that agrees with+ * the identity on {0, 1} gives the same answer. A byte of 2 would+ * discriminate, and nothing produces one here -- so this is a reading+ * that no measurement available on this machine can confirm or refute,+ * and a shader that fed it a wider byte would be the first test of it.+ */+ r = nir_b2iN(&c->b, nir_ine_imm(&c->b, a[0], 0),+ air_nir_bit_size(c, in->base.type));+ } else if (!strcmp(op, "unpack") && n->num >= 2 &&+ !strcmp(n->part[1], "unorm4x8")) {+ /* four unsigned bytes of a word, each divided by 255. Component 0 is the+ * LOW byte, which is both GLSL's unpackUnorm4x8 and Metal's+ * unpack_unorm4x8_to_float, so NIR's opcode is the operation and not an+ * approximation of it. ir3 has no instruction for it and does not need+ * one: it sets lower_unpack_unorm_4x8, and nir_opt_algebraic turns it+ * into four extract_u8 and a divide.+ */+ r = nir_unpack_unorm_4x8(&c->b, a[0]);+ if (air_nir_bit_size(c, in->base.type) == 16)+ r = nir_f2f16(&c->b, r);+ } else if (!strcmp(op, "pack") && n->num >= 3 &&+ !strcmp(n->part[1], "unorm") && !strcmp(n->part[2], "rgb565")) {+ /* COMPONENT 0 IS AT BIT 0, and that is Apple's own answer rather than a+ * reading of the name. The module and its metadata fix the widths and+ * that the 6-bit field is in the middle; which end component 0 is at is+ * in neither, and the two readings exchange red and blue. A draw through+ * air_run cannot separate them either: the pack and the number read back+ * would both be ours. (docs/air.md 5e has what a draw on the DEVICE+ * could and could not say, which is not what the refusal here used to+ * claim.) What settles it is the lowering in AGXCompilerCore -- this+ * device's own, ios165/20F66__iPhone10,3_6 -- read as instructions:+ *+ * AGCLLVMAirBuiltins::initializePackUnpackMap 0x1bbe5c0cc+ * maps the name suffix "unorm.rgb565.v3f" to kind 2+ * AGCLLVMAirBuiltins::buildPack 0x1bbe5be08+ * kind 2 -> buildPackFormat(PackedType 0, normalized, unsigned)+ * AGCLLVMTargetLowerer::buildPackFormat 0x1bbed0934+ * PackedType 0..3 -> buildPackFloatFormat, which keeps the type+ * AGCLLVMTargetLowerer::buildPackFloatFormat 0x1bbed3204+ * indexes a 9-byte row of `formats` (0x1bbf1977f) by PackedType:+ * one count byte, then four WIDTHS, then four SHIFTS. Row 0 is+ * count 3, widths {5, 6, 5}, shifts {0, 5, 11}, and the loop is+ * `acc |= packFromFloat(extractelement(v, i), width[i], shift[i])`+ * from a constant 0, then truncateToSmall to the declared i16.+ * AGCLLVMTargetLowerer::buildPackFromFloat 0x1bbed2e38+ * fpext to f32; buildFClamp(0.0, 1.0), which for exactly those two+ * bounds dispatches to buildSaturate (0x1bbe0e428); fmul by+ * (1 << width) - 1; llvm.agx{1,2}.pack.rne.s32, whose prefix is the+ * GPU generation's (Gen2's constructor stores agx1 and Gen3's agx2,+ * both round to nearest even); shl by `shift`.+ *+ * THE SHIFTS ARE THE RUNNING SUM OF THE WIDTHS in all seven rows of that+ * table -- 565 {0,5,11}, 4444 {0,4,8,12}, 1555 {0,1,6,11},+ * 5551 {0,5,10,15}, rgb10 {0,10,20}, rgb10a2 {0,10,20,30} and a second+ * copy of that last layout, rows 5 and 6 being the float and the uint+ * variants of 1010102 -- so the fields are laid out from bit 0+ * upward in component order and the 565 row cannot be read the other way+ * round without breaking the six beside it. Each row's widths also sum+ * to its word: 16 for the four 16-bit ones, 30 and 32 for the others.+ *+ * The inverse lowering is the second witness, and it reads the same row+ * through the opposite argument order: buildUnpackFloatFormat+ * (0x1bbe001ec) hands buildUnpackToFloat (0x1bbed2c10) the SHIFT first+ * and the width second, where the pack side passes them the other way+ * round, and that callee begins with `lshr` by its first argument and+ * masks with (1 << width) - 1. A misreading of which byte-run of the row+ * is widths and which is shifts would have to survive both call sites.+ *+ * So, for i in 0..2 with width {5,6,5} and shift {0,5,11}:+ * acc |= rne(saturate(f32(v[i])) * ((1 << width[i]) - 1)) << shift[i]+ *+ * In f32 and not in f16, because Apple extends first, and that is+ * measured and not asserted: 61 half values round differently if the+ * product is computed at f16, two of them are in intrincheck.py's pool+ * for this operation, and removing the extension here duly makes 2 of+ * its 24 cases differ. nir_fsat IS saturate and nir_fround_even IS+ * Apple's rne, so neither step here is an approximation of the one it+ * stands for.+ *+ * ROUND-TO-NEAREST-EVEN IS NOT OBSERVABLE at this width, which is worth+ * knowing rather than relying on: over all 65 536 half bit patterns the+ * only exactly half-way products are 0.5*31 = 15.5 and 0.5*63 = 31.5,+ * and nearest-even and half-away-from-zero both answer 16 and 32 there.+ * So no input this operation can be given separates Apple's rounding+ * mode from the other one, and it is spelled rne because that is what+ * Apple's code names. Truncation does differ, on 7 551 of them, and that+ * is the mistake the check can catch (tools/air/intrincheck.py, and+ * tools/air/pack565.ll for four colours readable by eye).+ */+ static const unsigned pack565_width[3] = { 5, 6, 5 };+ static const unsigned pack565_shift[3] = { 0, 5, 11 };+ /* exact: a pack is a bit pattern, not a shader's arithmetic, and the+ * fast-math this function turns on for everything else has no business+ * reassociating the one multiply it contains+ */+ c->b.exact = true;+ nir_def *acc = nir_imm_int(&c->b, 0);+ for (unsigned k = 0; k < 3; k++) {+ nir_def *x = nir_channel(&c->b, a[0], k);+ if (x->bit_size != 32)+ x = nir_f2f32(&c->b, x);+ x = nir_fmul_imm(&c->b, nir_fsat(&c->b, x),+ (double)((1u << pack565_width[k]) - 1));+ nir_def *f = nir_f2u32(&c->b, nir_fround_even(&c->b, x));+ acc = nir_ior(&c->b, acc, nir_ishl_imm(&c->b, f, pack565_shift[k]));+ }+ r = nir_u2uN(&c->b, acc, air_nir_bit_size(c, in->base.type));} else if (!strcmp(op, "convert")) {/* air.convert.<dst sign>.<dst type>.<src sign>.<src type> */if (n->num < 5) {@@ -291,7 +817,19 @@ math_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n,nir_def *x = a[0];if (!src_float && air_type_bit_size(air_type_scalar(st)) == 1)x = src_signed ? nir_ineg(&c->b, nir_b2i32(&c->b, x)) : nir_b2i32(&c->b, x);- if (dst_float && src_float)+ /* A BOOL DESTINATION IS NOT A CONVERSION. C's rule, which MSL inherits,+ * is `x != 0`, and in NIR a boolean is a 1-bit def -- so asking+ * nir_f2uN for one bit produces f2u1, an opcode ir3 has no instruction+ * for and no lowering of. That is the whole of why+ * variable_blur_frag_{lpf,lph} were the only two modules of QuartzCore's+ * library to reach the backend and die there: "Unhandled ALU op: f2u1",+ * from air.convert.u.i1.f.f32. The comparison is also the right answer+ * rather than a way round the backend: a conversion would have a+ * rounding mode, and bool(0.5) is true, not false.+ */+ if (bits == 1)+ r = src_float ? nir_fneu_imm(&c->b, x, 0.0) : nir_ine_imm(&c->b, x, 0);+ else if (dst_float && src_float)r = nir_f2fN(&c->b, x, bits);else if (dst_float)r = src_signed ? nir_i2fN(&c->b, x, bits) : nir_u2fN(&c->b, x, bits);@@ -312,6 +850,53 @@ math_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n,return true;}+/* -- function constants ------------------------------------------------- */++/* air.is_function_constant_defined(ptr to the constant's global)+ *+ * Metal's companion to `[[function_constant(n)]]`: a shader may ask whether+ * the pipeline supplied a value for a constant, and guard the code that reads+ * it. The answer is known at translation time -- it is exactly whether the+ * caller passed a value for that index in air_to_nir_options -- so this folds+ * to a constant and the guarded branch folds away with it.+ *+ * It is handled here, before math_op, because its argument is a POINTER and+ * math_op resolves every argument as a number: the four fixed_vert_* modules+ * stopped at "a pointer where a number belongs" with no instruction named,+ * because the failure was in reading the arguments and not in the operation.+ */+static bool+fc_defined_op(struct air_ctx *c, const struct air_inst *in)+{+ if (in->num_operands != 2)+ return air_error(c, "intrinsic: air.is_function_constant_defined of %u arguments",+ in->num_operands - 1);+ struct air_val *p = air_get(c, in->operands[1]);+ if (!p)+ return false;+ if (p->kind != AIR_VAL_PTR || p->ptr.class != AIR_PTR_FUNCTION_CONSTANT)+ return air_error(c, "intrinsic: air.is_function_constant_defined of something "+ "that is not a function constant");+ /*+ * Defined means SUPPLIED BY THE PIPELINE, not "declared by the module".+ *+ * This loop used to answer true whenever the index was in the array, and the array is+ * every constant the module declares -- so the answer was always true, and the value+ * handed over with it was the zero nobody had filled in. The question these 80+ * constants of the corpus ask is exactly the one that was being answered wrongly:+ * each is guarded by this intrinsic and has a branch for not having been supplied.+ */+ bool defined = false;+ for (unsigned i = 0; c->options && i < c->options->num_constants; i++) {+ if (c->options->constants[i].index == p->ptr.binding) {+ defined = c->options->constants[i].supplied;+ break;+ }+ }+ air_set(c, &in->base, air_val_ssa(c, nir_imm_bool(&c->b, defined)));+ return true;+}+/* -- atomics ------------------------------------------------------------ */static bool@@ -340,6 +925,38 @@ atomic_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *nnir_def *off = p.offset ? p.offset : nir_imm_int(&c->b, 0);+ /* Compare-exchange, refused before the memory order is read -- because its+ * arguments are in different places and reading them as the others' gave a+ * sentence about the wrong thing. Its signature, decoded from the call+ * sites so that implementing it needs no second reading:+ *+ * i32 air.atomic.<global|local>.cmpxchg.weak.i32(+ * ptr object, ptr expected, i32 desired,+ * i32 success_order, i32 failure_order, i32 scope, i1 volatile)+ *+ * so the order this used to read as argument 3 is the DESIRED VALUE, which+ * is not a constant, and 4 modules of Resident Evil 4 were told "with a+ * memory order that is not a constant" about an operation this does not+ * implement at all.+ *+ * THE RESULT IS THE OBJECT'S OLD VALUE and not a boolean, which is read off+ * what the callers do with it rather than from the name: Metal's+ * atomic_compare_exchange_weak_explicit returns bool, and every call site+ * here instead feeds the result to `icmp ne i32 %expected, %result` and+ * then re-loads *expected -- the weak-CAS retry loop HLSL's+ * InterlockedCompareExchange compiles to. A lowering is three operations+ * (load *expected, nir_atomic_op_cmpxchg, store the old value back through+ * `expected`, whose conditional-on-failure store may be unconditional+ * because on success it writes the value already there), and what stops it+ * is not the operation but the check: nothing in tools/air/atomiccheck.py+ * can make a wrong compare-exchange show, and an atomic that is only+ * compiled is the one kind this project will not claim.+ */+ if (!strcmp(op, "cmpxchg"))+ return air_error(c, "intrinsic: air.atomic.%s.cmpxchg -- compare-exchange, whose "+ "result is the old value and whose `expected` is a pointer "+ "this does not write back yet", where);+/* The memory order, which this lowering is right for exactly one value of.* That argument is Metal's memory_order, and every atomic in the corpus* passes 0 -- memory_order_relaxed. A relaxed load or store of an aligned@@ -403,11 +1020,40 @@ atomic_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n/* -- barriers ----------------------------------------------------------- */static bool-barrier_op(struct air_ctx *c, const struct air_inst *in)+barrier_op(struct air_ctx *c, const struct air_inst *in, bool simdgroup){- /* air.wg.barrier(i32 flags, i32 scope): the flags say which memory is- * ordered. 2 is threadgroup memory, 1 device, 4 texture; the corpus only- * ever passes 2 with scope 1 (the threadgroup).+ /* air.wg.barrier(i32 flags, i32 scope) and air.simdgroup.barrier(i32 flags,+ * i32 scope). BOTH ARGUMENTS ARE NOW APPLE'S OWN WORDS rather than a+ * census: the Metal headers shipped with this device's own shader compiler+ * (GPUCompiler.framework/.../include/metal/metal_compute) define the two+ * library functions as+ *+ * threadgroup_barrier(mem_flags f) { __metal_wg_barrier(int(f),+ * __METAL_MEMORY_SCOPE_THREADGROUP__); }+ * simdgroup_barrier(mem_flags f) { __metal_simdgroup_barrier(int(f),+ * __METAL_MEMORY_SCOPE_THREADGROUP__); }+ *+ * so the second argument is the MEMORY scope and is the threadgroup in+ * both -- the 1 the corpus passes, now sourced rather than observed -- and+ * the only difference between the two operations is the EXECUTION scope:+ * the threadgroup for one, the SIMD-group for the other. That is the whole+ * of what `simdgroup` selects below.+ *+ * The flags are `enum class mem_flags` of the same header: mem_none,+ * mem_device, mem_threadgroup, mem_threadgroup_imageblock, mem_texture,+ * mem_object_data. The numbers are compiler built-ins and not in the+ * header, so 1 = device, 2 = threadgroup, 4 = texture, 8 = imageblock stays+ * a reading of the calls -- but `mem_threadgroup_imageblock` NAMES flag 8's+ * memory as threadgroup memory, which is the thing the fallback below used+ * to be guessing.+ *+ * The flag census of the corpus, kept because it is what a reader can+ * check: 46 of the 852 modules call air.wg.barrier at all, and they pass 1+ * in 3 of them (6 calls), 2 in 25 (43 calls) and 8 in 15 (15 calls). The+ * fifteen are exactly the fifteen CMImaging modules with an explicit+ * imageblock, which stop at the interface (air_to_nir.c) and so never+ * arrive here. No module of the corpus calls air.simdgroup.barrier; 509 of+ * the image's 10 242 do, and 106 of those stop here (docs/air.md 10).*/uint64_t flags = 2;if (in->num_operands > 1)@@ -420,12 +1066,568 @@ barrier_op(struct air_ctx *c, const struct air_inst *in)modes |= nir_var_mem_shared;if (flags & 4)modes |= nir_var_image;- if (!modes)- modes = nir_var_mem_shared;+ /* Flag 8 is mem_threadgroup_imageblock, and the header's name for it is why+ * it gets threadgroup memory rather than the old catch-all fallback: an+ * explicit imageblock is per-threadgroup storage and Apple calls it that.+ * It still cannot reach this function -- an imageblock input is refused at+ * the interface -- so this is a mode for a case that does not arrive, which+ * is better than a default that answers for it silently.+ */+ if (flags & 8)+ modes |= nir_var_mem_shared;- nir_barrier(&c->b, .execution_scope = SCOPE_WORKGROUP,- .memory_scope = SCOPE_WORKGROUP, .memory_semantics = NIR_MEMORY_ACQ_REL,+ /* mem_none == 0 ORDERS NO MEMORY, and that is the one place the old code+ * said something Apple's header contradicts. It read `if (!modes) modes =+ * nir_var_mem_shared;` and called that right-for-the-wrong-reason about+ * flag 8; but 0 is `mem_none`, a value the enumeration defines, and six+ * calls in the image pass it (five in MLCompute, one in+ * CameraColorProcessing, all of them to simdgroup.barrier). For those the+ * fallback invented a threadgroup-memory fence the shader did not ask for.+ * It was conservative and therefore not a wrong answer, but it was an+ * answer to a question nobody asked, and a barrier that orders nothing is+ * exactly what an execution-only barrier is.+ */+ nir_barrier(&c->b,+ .execution_scope = simdgroup ? SCOPE_SUBGROUP : SCOPE_WORKGROUP,+ .memory_scope = modes ? SCOPE_WORKGROUP : SCOPE_NONE,+ .memory_semantics = modes ? NIR_MEMORY_ACQ_REL : 0,.memory_modes = modes);++ /* The SIMD-group's width is the hardware's and not Metal's 32 (simd_op+ * below has the whole argument). For a barrier that costs less than it does+ * for a shuffle -- a barrier over MORE lanes than the shader expects still+ * orders the ones it expects -- but a shader that uses a simdgroup barrier+ * INSTEAD of a threadgroup one because it knows its threadgroup is one+ * 32-lane SIMD-group is relying on the width, and on this hardware the+ * group is 64 or 128, so a threadgroup of 64 that Apple's compiler would+ * have split into two SIMD-groups is one here. That changes which lanes the+ * barrier covers, in the safe direction.+ */+ if (simdgroup)+ air_note(c, "air.simdgroup.barrier is taken over the SIMD group this "+ "implementation has, which is not Metal's 32 lanes: ir3 runs 64 or "+ "128 of them on this hardware, so the barrier covers at least the "+ "lanes the shader expects and possibly more");+ return true;+}++/* -- fragment flow control ---------------------------------------------- */++/* air.discard_fragment()+ *+ * NIR'S `demote` AND NOT NIR'S `terminate`, and that choice is the whole of+ * this function. The two differ in what the thread does afterwards: terminate+ * ends the invocation, so the lane stops contributing to its quad's+ * derivatives and performs no further memory writes, while demote turns it+ * into a HELPER invocation -- it keeps executing, its framebuffer and memory+ * writes are dropped, and it still supplies the neighbour values that a+ * derivative or an implicit-level-of-detail sample in the OTHER three lanes+ * of the quad needs.+ *+ * It is a real distinction in NIR and in every backend that has one: ir3+ * keeps the two apart in its own IR (ir3_compiler_nir.c emits ir3_DEMOTE+ * against ir3_KILL, and they carry different scheduling barriers), and+ * nir_to_tgsi sends them to TGSI's DEMOTE and KILL, which are separate+ * opcodes. ONE PLACE COLLAPSES THEM, AND IT IS WORTH KNOWING ABOUT: the a6xx+ * instruction encoder maps DEMOTE onto the hardware's KILL --+ * `} else if (instr->opc == OPC_DEMOTE) { return OPC_KILL; }` in+ * src/freedreno/isa/encode.c -- and the ISA description has no demote bitset+ * at all (ir3-cat0.xml carries `kill` and nothing beside it). So on this GPU,+ * with this Mesa, both come out as one instruction and the helper lanes Metal+ * promises are not delivered by the hardware path today. That is a gap to+ * name, not a reason to translate this as `terminate`: terminate would lose+ * the same lanes and would be wrong on a backend that does tell them apart.+ * What it costs is measurable only on the phone -- a shader that discards and+ * then takes a derivative -- and that is written as a job rather than assumed.+ *+ * ONE THING IS NOT A DIFFERENCE BETWEEN THE TWO, and saying so is the+ * correction of a sentence that used to stand here: stores. NIR's own+ * definition of demote (nir_intrinsics.py) is "Any stores to memory after+ * this instruction are suppressed and the fragment does not write outputs to+ * the framebuffer. Unlike discard, demote needs to ensure that derivatives+ * will still work for invocations that were not demoted." So demote drops the+ * lane's later writes as well -- it differs from terminate in the derivatives+ * and in the lane going on executing, not in the memory. Whether METAL drops+ * them is a third question, and nothing on this machine answers it: no header+ * on the device says, and a corpus cannot show it. If Metal keeps them, both+ * of NIR's intrinsics are wrong about it by the same amount.+ *+ * Nothing in a corpus COUNT can decide it -- both translate, both validate,+ * both compile -- so it is decided from two independent readings of what+ * Metal means, and both say the thread lives:+ *+ * 1. The shaders themselves. Resident Evil 4's `PS_DrawByTiles` calls+ * air.discard_fragment at the ninth instruction of a 1 800-line function+ * and then calls `air.simd_is_helper_thread()` EIGHT TIMES further down+ * it. A terminated thread cannot be asked whether it is a helper thread;+ * a demoted one answers yes, which is the only reading under which that+ * shader means anything. 24 of the library's 1 288 discard-calling modules+ * call that function.+ * 2. Apple's own declaration of it. `llvm-dis` of that module reads+ * `declare void @air.discard_fragment() local_unnamed_addr #8` with+ * `attributes #8 = { nounwind }` -- and NOT `noreturn`. LLVM is entitled+ * to delete everything after a call it is told does not return, and+ * Apple's compiler kept the rest of the function; the call is followed by+ * `br` and not `unreachable` in 73 of 85 call sites sampled. A discard+ * that killed the thread would be a noreturn call.+ *+ * And the cost of getting it wrong is not hypothetical here: of 85 sampled+ * call sites, 45 are followed in the same function by an air.sample_* with an+ * implicit level of detail and 6 by air.dfdx, both of which read the other+ * lanes of the quad. Under `terminate` those neighbours would read a dead+ * lane.+ *+ * uses_discard is set by hand because nir_shader_gather_info does not run on+ * this path (ir3 calls it only for a binning variant), exactly as the+ * framebuffer-fetch flags are set in air_to_nir.c. It is not bookkeeping: it+ * is what nir_to_tgsi_info turns into `uses_kill`, which is what a Gallium+ * driver reads before it allows an early depth test, and what freedreno reads+ * before it writes LRZ. A discarding shader whose interface says it discards+ * nothing gets its depth written for pixels it threw away.+ */+static bool+discard_op(struct air_ctx *c, const struct air_inst *in)+{+ if (c->nir->info.stage != MESA_SHADER_FRAGMENT)+ return air_error(c, "intrinsic: air.discard_fragment in a %s shader",+ gl_shader_stage_name(c->nir->info.stage));+ nir_demote(&c->b);+ c->nir->info.fs.uses_discard = true;+ return true;+}++/* -- SIMD groups and quads ----------------------------------------------- */++/* air.simd_* and air.quad_*: Metal's cross-lane operations.+ *+ * THE SIMD-GROUP HERE IS THE IMPLEMENTATION'S, NOT A 32-LANE ONE, and that is+ * the only way these can be translated honestly. Metal does not fix the width+ * -- a shader asks for it with air.get_simd_width (this file answers with+ * nir_load_subgroup_size) and a host asks MTLComputePipelineState for+ * threadExecutionWidth -- and it is 32 on every Apple GPU, which is why a+ * shader compiled for one may have the number built into it. This hardware's+ * is 64 or 128: `threadsize_base` is 64 for the A635 this port targets and ir3+ * doubles it per variant in the compute and fragment stages+ * (ir3_shader_get_subgroup_size, IR3_SINGLE_OR_DOUBLE).+ *+ * So the width a translated shader is TOLD and the width its votes are taken+ * over are the same number, whatever that number is: a shader that reads+ * air.get_simd_width and works from it is right, and a shader with 32 written+ * into it is wrong -- as it would be on any Metal implementation whose+ * SIMD-group is not 32 wide. Each of these leaves a note saying so, because it+ * is a thing the hardware cannot do exactly rather than a thing this frontend+ * chose.+ *+ * One more thing decides whether that consistency holds, and it is a property+ * of the GPU and not of this file: ir3 declares a fragment or vertex+ * subgroup to be ONE LANE and lowers a vote to its own argument on a device+ * without `has_getfiberid` (ir3_nir.c: `options.subgroup_size = 1`,+ * `lower_vote_trivial`). The A635 has it -- it is an a6xx gen4, and+ * freedreno_devices.py sets the flag for gen4 and above -- so the votes here+ * are real in every stage, and air.get_simd_width answers with the real width+ * in the same stages. On an a6xx below gen4 both would read 1, which is still+ * self-consistent and still not Metal's 32.+ *+ * air.simd_ballot is a 64-bit mask with one bit per lane, and it is exact+ * only while a group has at most 64 lanes. That is the driver's to promise,+ * not this frontend's: hakios-metal builds every variant with zeroed+ * shader_options, IR3_SINGLE_ONLY, so ir3 never doubles the threadsize and a+ * group is threadsize_base, 64 lanes, in every stage -- the execution width+ * the device reports. A driver that lets ir3 run 128 would get the low half+ * of the mask; the note below says so where it is emitted.+ *+ * The QUAD operations are a different matter and are exact: a quad is four+ * consecutive lanes on both machines -- it is what a fragment derivative is+ * computed over -- so a swap inside one has the same meaning whatever the+ * SIMD-group's width is, and nir_lower_subgroups leaves the quad intrinsics+ * alone for ir3 (no lower_quad, and lower_vote_trivial does not reach them).+ */+static bool+simd_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n,+ bool *handled)+{+ const char *op = n->part[0];++ /* THE NAME IS DECIDED BEFORE AN ARGUMENT IS TOUCHED, which is the same rule+ * math_op states above it and fc_defined_op exists for -- and getting it+ * wrong here cost 46 modules of the 456 for the length of one commit. This+ * function used to resolve its first argument before asking whether the+ * name was one of its own, and every texture operation passes a texture+ * HANDLE as that argument: air.sample_texture_2d came back "instruction: a+ * pointer where a number belongs", which reads like a broken module and+ * names no operation at all. The count was the only visible symptom and it+ * was briefly attributed to somebody else's commit.+ */+ if (strncmp(op, "simd_", 5) && strncmp(op, "quad_", 5) &&+ strcmp(op, "get_sample_position") && strcmp(op, "is_uniform")) {+ *handled = false;+ return true;+ }+ *handled = true;++ if (!strcmp(op, "simd_is_helper_thread")) {+ /* Exact, and width-independent: a helper thread is one whose results+ * are discarded, which is a property of the thread and not of a group.+ * ir3 runs nir_lower_is_helper_invocation itself, and this is the other+ * half of the discard above -- a demoted lane must answer yes here.+ */+ if (c->nir->info.stage != MESA_SHADER_FRAGMENT)+ return air_error(c, "intrinsic: air.simd_is_helper_thread in a %s shader "+ "(there are no helper threads outside the fragment "+ "stage)", gl_shader_stage_name(c->nir->info.stage));+ air_set(c, &in->base, air_val_ssa(c, nir_is_helper_invocation(&c->b, 1)));+ return true;+ }++ nir_def *a0 = in->num_operands > 1 ? air_get_ssa(c, in->operands[1]) : NULL;+ if (in->num_operands > 1 && !a0)+ return false;++ if (!strcmp(op, "simd_ballot")) {+ /* Resident Evil 4's culling passes elect a lane with it (a ballot of the+ * live lanes, then a popcount of the ones below this lane) */+ if (!a0 || a0->bit_size != 1 || in->base.type->kind != AIR_TYPE_INT ||+ in->base.type->bits != 64)+ return air_error(c, "intrinsic: air.simd_ballot of something other than a "+ "bool into a 64-bit mask");+ air_note(c, "air.simd_ballot is a mask of the SIMD group this implementation "+ "has, 64 lanes when the driver keeps ir3 to its single threadsize "+ "(hakios-metal does), not Metal's 32: a shader that truncates it to "+ "32 bits and indexes it by lane & 31 sees two groups of 32 in it");+ air_set(c, &in->base, air_val_ssa(c, nir_ballot(&c->b, 1, 64, a0)));+ return true;+ }++ if (!strcmp(op, "simd_all") || !strcmp(op, "simd_any")) {+ bool all = !strcmp(op, "simd_all");+ air_note(c, "air.%s is taken over the SIMD group this implementation has, "+ "which is not Metal's 32 lanes: ir3 runs 64 or 128 of them on this "+ "hardware. air.get_simd_width answers with the same number, so a "+ "shader that reads it is right and one with 32 built into it is not",+ n->part[0]);+ air_set(c, &in->base,+ air_val_ssa(c, all ? nir_vote_all(&c->b, 1, a0)+ : nir_vote_any(&c->b, 1, a0)));+ return true;+ }++ /* air.is_uniform (Metal's simd_is_uniform, mediaserverd's CNN kernels): whether+ * every active lane holds the same value -- NIR's vote_ieq / vote_feq, over the+ * group this hardware has (the note above). A 16-bit value is widened first,+ * which keeps equality as it was.+ */+ if (!strcmp(op, "is_uniform")) {+ if (!a0)+ return air_error(c, "intrinsic: air.is_uniform with no value");+ const bool fl = n->num > 1 && n->part[1][0] == 'f';+ nir_def *v = a0->bit_size < 32 ? (fl ? nir_f2f32(&c->b, a0) : nir_u2u32(&c->b, a0)) : a0;+ air_set(c, &in->base, air_val_ssa(c, fl ? nir_vote_feq(&c->b, 1, v) : nir_vote_ieq(&c->b, 1, v)));+ return true;+ }++ if (!strcmp(op, "simd_broadcast_first")) {+ air_note(c, "air.%s reads the first active lane of the SIMD group this "+ "implementation has, which is not Metal's 32 lanes -- the value is "+ "the same one Metal would give wherever it is uniform across the "+ "group, and another lane's where it is not", n->part[0]);+ air_set(c, &in->base, air_val_ssa(c, nir_read_first_invocation(&c->b, a0)));+ return true;+ }++ if (!strcmp(op, "simd_shuffle")) {+ nir_def *lane = air_get_ssa(c, in->operands[2]);+ if (!lane)+ return false;+ air_note(c, "air.%s takes its lane number in the SIMD group this "+ "implementation has, which is not Metal's 32 lanes", n->part[0]);+ air_set(c, &in->base,+ air_val_ssa(c, nir_shuffle(&c->b, a0, nir_u2u32(&c->b, lane))));+ return true;+ }++ if (!strcmp(op, "simd_is_first")) {+ /* simd_is_first(): true in exactly one lane of the SIMD group, the+ * lowest-numbered ACTIVE one. NIR's `elect` is that operation and+ * nothing else -- it is SPIR-V's OpGroupNonUniformElect -- and ir3 emits+ * it directly (nir_intrinsic_elect in ir3_compiler_nir.c), with no+ * lowering in between to disagree with.+ *+ * This is one of the few cross-lane operations whose MEANING does not+ * depend on the group's width: "the first active lane" is the first+ * active lane at 32, 64 or 128. What does depend on the width is what+ * the shader then does with it, and in the two corpus modules that call+ * this the dependence is total: CMISharpnessScore::sumScoreStep{1,2}+ * elect one lane per SIMD-group to write that group's air.simd_sum into+ * `simdGroupSums`, a threadgroup array declared `[32 x float]` and+ * indexed by the SIMD-group's index in the threadgroup, and then read+ * back at `simd_lane_id`. At 64 lanes that read is past the end of the+ * array. So translating this operation makes those two modules COMPILE+ * and does not make them right, and the note says which.+ *+ * The reading "lane 0" would agree with `elect` on every case those two+ * modules can produce, because both calls sit in reconvergent control+ * flow where lane 0 is active -- which is exactly the kind of test that+ * passes for the wrong reason. The reason to prefer `elect` is not the+ * corpus: it is that Apple's own header gates the function on+ * __HAVE_SIMDGROUP_IS_FIRST__ beside simd_is_helper_thread, whose whole+ * subject is which lanes are live, and that a lane-0 reading would need+ * air.get_simd_lane_id, which the operation does not take.+ */+ /* NO SEMICOLON IN A NOTE. air2nir's --report joins the notes of a module+ * with `;` and corpus.py splits them on it, so a note with one in it+ * arrives in the report as two rows, the second of them starting+ * mid-sentence. Found by reading the report after adding this one.+ */+ air_note(c, "air.simd_is_first elects one lane of the SIMD group this "+ "implementation has, which is not Metal's 32 lanes: ir3 runs 64 or "+ "128 of them on this hardware. The election itself is exact at any "+ "width, but a shader that sizes a per-SIMD-group array at 32, or "+ "indexes one by air.get_simd_lane_id, is relying on the width and is "+ "wrong here however this operation is translated");+ air_set(c, &in->base, air_val_ssa(c, nir_elect(&c->b, 1)));+ return true;+ }++ if (!strcmp(op, "simd_broadcast")) {+ /* simd_broadcast(data, lane): the value held by one named lane, given to+ * every lane. NIR's read_invocation, which is the same operation; ir3+ * lowers it to read_invocation_cond_ir3 (lower_read_invocation_to_cond)+ * rather than refusing it.+ */+ nir_def *lane = air_get_ssa(c, in->operands[2]);+ if (!lane)+ return false;+ air_note(c, "air.%s takes its lane number in the SIMD group this "+ "implementation has, which is not Metal's 32 lanes", n->part[0]);+ air_set(c, &in->base,+ air_val_ssa(c, nir_read_invocation(&c->b, a0,+ nir_u2u32(&c->b, lane))));+ return true;+ }++ if (!strcmp(op, "simd_shuffle_up") || !strcmp(op, "simd_shuffle_down") ||+ !strcmp(op, "simd_shuffle_xor")) {+ /* simd_shuffle_{up,down,xor}(data, delta): the value held by the lane+ * `lane - delta`, `lane + delta` or `lane ^ delta`. NIR has all three+ * with the same definition, and in both languages the result is+ * UNDEFINED when the computed lane is outside the group -- SPIR-V's+ * OpGroupNonUniformShuffle{Up,Down,Xor} say so and Metal says so -- so+ * the pair agrees on the defined cases and neither promises anything+ * about the rest. That is why these three are translated and+ * air.simd_shuffle_and_fill_* below is not.+ *+ * THE WIDTH IS WORSE HERE THAN FOR A VOTE, and the note says so. A+ * reduction written as a shuffle ladder -- deltas 16, 8, 4, 2, 1 -- puts+ * the sum of lanes 0..31 in lane 0 whatever the group's width is, because+ * lane 0 never reads above lane 31; but every OTHER lane of a 64- or+ * 128-wide group reads across the 32-lane window the shader believes in,+ * so a shader that uses more than lane 0's answer gets a different number+ * here than on an Apple GPU. Which of the two it is cannot be decided by+ * this function: it is a property of the shader.+ */+ nir_def *delta = air_get_ssa(c, in->operands[2]);+ if (!delta)+ return false;+ delta = nir_u2u32(&c->b, delta);+ air_note(c, "air.%s steps by lanes of the SIMD group this implementation has, "+ "which is not Metal's 32 lanes: ir3 runs 64 or 128 of them on this "+ "hardware, so a lane whose source is outside Apple's 32-lane window "+ "reads a real neighbour here where Metal leaves it undefined",+ n->part[0]);+ nir_def *r = !strcmp(op, "simd_shuffle_up")+ ? nir_shuffle_up(&c->b, a0, delta)+ : !strcmp(op, "simd_shuffle_down")+ ? nir_shuffle_down(&c->b, a0, delta)+ : nir_shuffle_xor(&c->b, a0, delta);+ air_set(c, &in->base, air_val_ssa(c, r));+ return true;+ }++ if (!strcmp(op, "simd_shuffle_and_fill_down") ||+ !strcmp(op, "simd_shuffle_and_fill_up")) {+ /* REFUSED BY NAME, and the refusal says where the answer is rather than+ * that there is none -- which is the difference between a wall and a+ * shrug, and this one has an address.+ *+ * The signature is Apple's: metal_simdgroup on this device declares+ * `simd_shuffle_and_fill_down(T data, T filling_data, ushort delta,+ * ushort modulo)` and a two-argument overload that passes+ * `__metal_get_simdgroup_size()` as the modulo. So `modulo` is a WINDOW+ * width, and the operation shuffles within a window of that many lanes+ * and takes `filling_data` where the source lane would leave it. What the+ * header does not state is which lane of `filling_data` it takes, and the+ * two readings -- the lane `delta` from the window's start, or the same+ * lane the shuffle would have had, counted on past the window -- give+ * different answers for every call.+ *+ * And the modulo is not a formality that could be ignored: Apple's own+ * calls pass a LITERAL. CMISharpnessScore::sharpnessScoreCalculatorFloat+ * passes 32 twice and 4 once; FRC's modules pass 4, 8 and a runtime+ * value. A literal 32 is Apple's group width written into the shader, and+ * on a 64- or 128-lane group a plain shuffle crosses that window instead+ * of filling at it, which is a wrong value and not an undefined one. A+ * literal 4 is not the group width at all and would be wrong on an Apple+ * GPU too if it were read as one.+ *+ * WHERE THE ANSWER IS: AGXCompilerCore in this device's own cache lowers+ * it, and the symbols are not hidden --+ * `AGCLLVMGen::TargetLowerer::buildShuffleAndFill` (`0x1bbefa8a4`),+ * `...::buildShuffleAndFillCore` (`0x1bbefaa1c`) and+ * `...::buildShuffleAndFillIfModuloIsAlpha` (`0x1bbefaf74`). The first+ * reads one byte at offset 0x10 of the `modulo` operand -- where+ * `llvm::Value` keeps its SubclassID -- and takes `...Core` for one value+ * of it and `...IfModuloIsAlpha` for the rest, so a constant modulo has a+ * path of its own, which is the path every call in this image takes. That+ * binary also carries the string `AGC_SIMD_ROTATE_AS_SHUFFLE_AND_FILL`,+ * which is why the rotate refusal below is this same wall and not another+ * one. Reading `...Core` out is what would settle the fill's source lane;+ * it has not been read, and inventing the formula instead is the thing+ * this frontend does not do (docs/air.md 10.3).+ */+ /* INSIDE air_error's 255 BYTES, counted rather than hoped: the name is 30+ * of them and the tail is 198, so the whole sentence arrives. The detail+ * that does not fit is the comment above and docs/air.md 10.3.+ */+ return air_error(c, "intrinsic: air.%s -- the `modulo` lane window (Apple passes "+ "32, 4 and 8). Which lane of `filling_data` fills is settled "+ "by AGXCompilerCore buildShuffleAndFillCore 0x1bbefaa1c, "+ "unread: docs/air.md 10.3", n->part[0]);+ }++ if (!strcmp(op, "simd_shuffle_rotate_down") || !strcmp(op, "simd_shuffle_rotate_up")) {+ /* A rotation inside the SIMD group: `(lane + delta) mod modulo`, which+ * names a different lane at 32 than at 64 in EVERY lane rather than only+ * near the edge. NIR has nir_rotate and ir3 lowers it+ * (lower_rotate_clustered_to_shuffle), so the operation exists on both+ * sides; what does not exist here is the window to rotate within.+ *+ * It is the fill's wall and not a second one, which Apple's own compiler+ * says rather than this file inferring it: AGXCompilerCore carries+ * `AGC_SIMD_ROTATE_AS_SHUFFLE_AND_FILL`, so the rotate is built out of+ * the operation above. No module of either the corpus or the image calls+ * one, so this is a refusal with no customer rather than a gap in reach.+ */+ return air_error(c, "intrinsic: air.%s -- a rotation in the SIMD group, which "+ "AGXCompilerCore builds out of shuffle-and-fill "+ "(AGC_SIMD_ROTATE_AS_SHUFFLE_AND_FILL), so it is that "+ "operation's wall: docs/air.md 10.3", n->part[0]);+ }++ if (!strcmp(op, "quad_shuffle") || !strcmp(op, "quad_broadcast")) {+ /* Apple's two names for one operation, and the header says so:+ * metal_quadgroup declares `quad_shuffle(T data, ushort quad_lane_id)`+ * and `quad_broadcast(T data, ushort broadcast_lane_id)` -- an absolute+ * lane WITHIN the quad in both cases. NIR's quad_broadcast is that, and+ * it is exact at any SIMD-group width because a quad is four consecutive+ * lanes on both machines.+ *+ * NIR inherits SPIR-V's requirement that the index be uniform across the+ * quad, and so does Metal's own quad_broadcast; `quad_shuffle` is the+ * name for the non-uniform reading, which is why both are here and only+ * a constant or a quad-uniform index is honest. Nothing enforces that:+ * the index is not inspected, because a value this frontend cannot prove+ * uniform is not a value it can prove non-uniform either, and refusing+ * every non-constant index would refuse the uniform ones with it.+ */+ nir_def *lane = air_get_ssa(c, in->operands[2]);+ if (!lane)+ return false;+ air_set(c, &in->base,+ air_val_ssa(c, nir_quad_broadcast(&c->b, a0, nir_u2u32(&c->b, lane))));+ return true;+ }++ if (!strcmp(op, "quad_shuffle_down") || !strcmp(op, "quad_shuffle_up")) {+ /* quad_shuffle_{down,up}(data, delta): the lane `delta` along inside the+ * quad, undefined when that leaves the quad. NIR has no quad-restricted+ * shuffle, so this is nir_shuffle_{down,up} over the SIMD group -- which+ * gives the same lane for every case the quad contains, and reads the+ * neighbouring quad for the cases Metal leaves undefined.+ *+ * That substitution is sound in exactly the way the `and_fill` one above+ * is not: there, the out-of-window case has a DEFINED result that this+ * would get wrong; here it has none, so a real neighbour's value is as+ * conforming as anything else. What it is not is identical to Apple's+ * hardware, and a shader written against that hardware's actual+ * behaviour out of range would differ -- which no reading of the bitcode+ * can detect, so it is a note and not a check.+ */+ nir_def *delta = air_get_ssa(c, in->operands[2]);+ if (!delta)+ return false;+ delta = nir_u2u32(&c->b, delta);+ air_note(c, "air.%s is emitted as a SIMD-group shuffle, which agrees with it "+ "inside the quad and reads the neighbouring quad where Metal leaves "+ "the result undefined", n->part[0]);+ air_set(c, &in->base,+ air_val_ssa(c, !strcmp(op, "quad_shuffle_down")+ ? nir_shuffle_down(&c->b, a0, delta)+ : nir_shuffle_up(&c->b, a0, delta)));+ return true;+ }++ if (!strcmp(op, "quad_shuffle_xor")) {+ /* quad_shuffle_xor(data, mask) hands each lane the value of the lane+ * whose number is its own exclusive-ored with the mask. Within a quad+ * that is one of three fixed swaps, which is exactly the three NIR has:+ * 1 swaps the two columns, 2 the two rows, 3 the diagonal. A mask of 0+ * is the identity.+ *+ * A mask that is not a constant, or is 4 or more, is refused rather+ * than translated: the first would need a selection over the three+ * swaps, and the second leaves the quad, where Metal's own definition+ * stops. Every call in Resident Evil 4's three modules passes a+ * constant.+ */+ nir_def *mask = air_get_ssa(c, in->operands[2]);+ if (!mask)+ return false;+ if (!nir_src_is_const(nir_src_for_ssa(mask)))+ return air_error(c, "intrinsic: air.%s with a mask that is not a constant",+ n->part[0]);+ uint64_t m = nir_src_as_uint(nir_src_for_ssa(mask));+ switch (m) {+ case 0: air_set(c, &in->base, air_val_ssa(c, a0)); return true;+ case 1: air_set(c, &in->base,+ air_val_ssa(c, nir_quad_swap_horizontal(&c->b, a0)));+ return true;+ case 2: air_set(c, &in->base,+ air_val_ssa(c, nir_quad_swap_vertical(&c->b, a0)));+ return true;+ case 3: air_set(c, &in->base,+ air_val_ssa(c, nir_quad_swap_diagonal(&c->b, a0)));+ return true;+ default:+ return air_error(c, "intrinsic: air.%s with mask %u, which leaves the quad",+ n->part[0], (unsigned)m);+ }+ }++ /* air.get_sample_position(index, 0): the position of one sample within the+ * pixel, which the hardware's sample pattern decides.+ *+ * REFUSED BY NAME, and the reason is half a pixel. NIR has the operation+ * and the Adreno has the instruction -- nir_load_sample_pos_from_id, which+ * ir3 emits as RGETPOS -- but that intrinsic's convention is an OFFSET+ * from the pixel's centre, in [-0.5, 0.5]: ir3's own lowering of+ * gl_SamplePosition adds (0.5, 0.5) to it and cites the dEQP case that+ * fixes the difference (ir3_nir_lower_load_barycentric_at_sample.c).+ * Metal's convention is not stated by anything on this machine -- not by+ * the metal_graphics header on the device, which only names the function+ * and its MSL section, and not by any string in the compiler libraries --+ * and the two readings differ by exactly half a pixel, which is the kind+ * of wrongness that looks like a slightly soft image rather than like a+ * bug. 4 of Resident Evil 4's 7 879 modules call it, and the way to settle+ * it is a measurement on the phone (draw one multisampled pixel, read the+ * positions back) rather than a choice made here.+ */+ if (!strcmp(op, "get_sample_position"))+ return air_error(c, "intrinsic: air.%s -- NIR's sample position is an offset "+ "from the pixel centre and Metal's convention is not "+ "attested anywhere on this machine; the two differ by half "+ "a pixel, so this is left to a measurement", n->part[0]);++ *handled = false;return true;}@@ -436,6 +1638,10 @@ llvm_op(struct air_ctx *c, const struct air_inst *in, const char *name){if (!strncmp(name, "llvm.lifetime.", 14))return true; /* scope markers: nothing to emit */+ /* a promise to the optimizer that its condition holds (MPS's CNN kernels+ * carry them): it changes nothing a shader computes, so nothing either */+ if (!strcmp(name, "llvm.assume"))+ return true;if (!strncmp(name, "llvm.memcpy.", 12)) {/* a copy between two of our pointers, of a constant length */struct air_val *dst = air_get(c, in->operands[1]);@@ -470,17 +1676,87 @@ llvm_op(struct air_ctx *c, const struct air_inst *in, const char *name)if (dst->kind != AIR_VAL_PTR)return air_error(c, "intrinsic: memset of something that is not a pointer");uint64_t bytes;- if (!air_const_uint(in->operands[3], &bytes) || bytes % 4)- return air_error(c, "intrinsic: memset of a length that is not a multiple of 4");+ if (!air_const_uint(in->operands[3], &bytes))+ return air_error(c, "intrinsic: memset of a run-time length");if (!nir_src_is_const(nir_src_for_ssa(byte)))return air_error(c, "intrinsic: memset of a run-time byte");uint32_t b = nir_src_as_uint(nir_src_for_ssa(byte)) & 0xff;uint32_t word = b | b << 8 | b << 16 | b << 24;- for (uint64_t off = 0; off < bytes; off += 4) {+ /* WHOLE WORDS, THEN WHATEVER IS LEFT. The length used to have to be a+ * multiple of four, and the one llvm.memset in all 852 modules of the+ * corpus is not: DepthProcessorV1's `kernelComputeStage1` zeroes EIGHTEEN+ * bytes of a `device` buffer, which is four words and a short. So that+ * rule never once applied to anything it allowed. There is nothing to+ * decide here -- a memset's meaning is LLVM's own and the bytes it writes+ * are not open to interpretation -- so the tail is written at the width it+ * is, 16 bits and then 8, which air_store_ptr carries because a store is a+ * store of whatever bit size the value has.+ *+ * Three stores and not a byte loop, because the length is a constant and+ * a byte loop would emit eighteen of them. `llvm.memcpy` keeps its+ * multiple-of-four refusal: a tail there needs a LOAD of the same odd+ * width as well, nothing in the image asks for one, and a lowering with+ * no customer is a lowering with no test.+ */+ uint64_t off = 0;+ for (; off + 4 <= bytes; off += 4) {struct air_ptr d = air_ptr_add(c, dst->ptr, nir_imm_int(&c->b, off));if (!air_store_ptr(c, d, nir_imm_int(&c->b, word), 4))return false;}+ if (bytes - off >= 2) {+ struct air_ptr d = air_ptr_add(c, dst->ptr, nir_imm_int(&c->b, off));+ if (!air_store_ptr(c, d, nir_imm_intN_t(&c->b, word & 0xffff, 16), 2))+ return false;+ off += 2;+ }+ if (bytes - off >= 1) {+ struct air_ptr d = air_ptr_add(c, dst->ptr, nir_imm_int(&c->b, off));+ if (!air_store_ptr(c, d, nir_imm_intN_t(&c->b, b, 8), 1))+ return false;+ }+ return true;+ }+ /* llvm.fmuladd(a, b, c): a multiply-add that MAY be fused.+ *+ * Translated as a fused one -- nir_ffma, a single rounding -- and the+ * licence for that is the intrinsic's own definition rather than the+ * module's fast-math flag. LLVM's Language Reference says of llvm.fmuladd+ * that the expression "is equivalent to a * b + c, except that it is+ * unspecified whether rounding will be performed between the+ * multiplication and addition steps", and that a caller who needs the+ * fusion guaranteed must use llvm.fma instead. A producer that emitted+ * llvm.fmuladd rather than `fmul` + `fadd` therefore accepted both answers+ * when it chose the name, and the choice between them is the back end's.+ *+ * SO THE ABSENCE OF air.compile.fast_math_enable CHANGES NOTHING HERE, and+ * that is worth saying because the measurement invites the opposite+ * reading: 383 of a 394-module sample of Resident Evil 4 carry the flag, so+ * contraction is permitted under the stricter reading too -- but the eleven+ * that do not are not a different case for THIS intrinsic. A module that+ * wanted two roundings would carry two instructions.+ *+ * Which answer the Adreno gives is then ir3's: nir_ffma is one rounding,+ * and a backend that has no fused multiply-add would lower it. softpipe+ * does not have one -- TGSI's own specification lets MAD round the product+ * first and its interpreter does -- which is why tools/air/intrincheck.py+ * sets air.fma aside on softpipe and leaves it to the phone. The same+ * applies to every llvm.fmuladd this translates.+ */+ if (!strncmp(name, "llvm.fmuladd.", 13)) {+ nir_def *a = air_get_ssa(c, in->operands[1]);+ nir_def *b = air_get_ssa(c, in->operands[2]);+ nir_def *d = air_get_ssa(c, in->operands[3]);+ if (!a || !b || !d)+ return false;+ air_set(c, &in->base, air_val_ssa(c, nir_ffma(&c->b, a, b, d)));+ return true;+ }+ if (!strncmp(name, "llvm.fabs.", 10)) {+ nir_def *a = air_get_ssa(c, in->operands[1]);+ if (!a)+ return false;+ air_set(c, &in->base, air_val_ssa(c, nir_fabs(&c->b, a)));return true;}if (!strcmp(name, "llvm.trap"))@@ -501,19 +1777,51 @@ llvm_op(struct air_ctx *c, const struct air_inst *in, const char *name)** The signatures are measured from the corpus, not guessed:*- * sample_<kind>_<dim> (tex, samp, [plane,] coord, i1 has_offset, offset,+ * sample_<kind>_<dim> (tex, samp, [plane,] coord, [i32 array,]+ * [i1 has_offset, offset,]* i1 lod_is_level, float lod_or_bias, float min_lod,* i32 flags) -> { <4 x T>, i8 }- * gather_<kind>_<dim> (tex, samp, coord, i1 has_offset, offset,+ * sample_<kind>_<dim>_grad+ * (tex, samp, [plane,] coord, [i32 array,]+ * dPdx, dPdy, float min_lod,+ * [i1 has_offset, offset,] i32 flags)+ * gather_<kind>_<dim> (tex, samp, coord, [i32 array,]+ * [i1 has_offset, offset,]* i32 component, i32 flags) -> { <4 x T>, i8 }- * read_<kind>_<dim> (tex, [plane,] coord, i32 lod_or_sample, i32 flags)- * write_<kind>_<dim> (tex, coord, value, i32 lod, i32 flags)+ * read_<kind>_<dim> (tex, [plane,] coord, [i32 array,]+ * i32 lod_or_sample, i32 flags)+ * write_<kind>_<dim> (tex, coord, [i32 array,] value, i32 lod, i32 flags)* get_<what>_<kind>_<dim> (tex[, i32 lod]) -> i32*+ * Three things in those brackets, and none of them is a guess. They are read+ * off Apple's own metal_stdlib on this device --+ * GPUCompiler.framework/.../include/metal/metal_texture, where every+ * `texture2d_array<T>::sample` and its fifty neighbours are one-line+ * forwarders to the builtin, so the header states the argument ORDER for all+ * nine dimensionalities and six operations -- and then confirmed against the+ * LLVM declaration in the modules that call them. The two agree everywhere:+ *+ * AN ARRAY INDEX IS ITS OWN ARGUMENT, straight after the coordinate, in+ * every one of sample, gather, read and write. It is not inside the+ * coordinate vector, which is what this file used to require of it:+ * air.sample_texture_2d_array takes a <2 x float> and an i32, and the+ * refusal it used to earn said "with a 2-component coordinate, wanting 3".+ *+ * THE GRADIENT FORM REORDERS WHAT FOLLOWS. dPdx, dPdy and the minimum level+ * come before the offset pair, where the plain form has the offset pair+ * first and the level last. A reader that assumed one order for both would+ * take a gradient for an offset flag.+ *+ * A CUBE HAS NO OFFSET PAIR AT ALL -- Metal defines no offset for cube+ * sampling -- so air.sample_texture_cube goes from the coordinate straight+ * to the level flag. Reading the 2d shape there is what made 621 modules+ * stop at "argument 5 is not a constant": the float minimum level was being+ * read as the i1 that says whether the next float is a level.+ ** The i1 before the level is read as "the float is a level, not a bias", so- * (false, 0.0) is a plain sample -- which is what every one of the corpus's- * sample calls passes. The level and bias forms are written here but no- * module exercises them.+ * (false, 0.0) is a plain sample -- which is what every one of the system+ * corpus's sample calls passes. Resident Evil 4 exercises both: 23 823+ * level-form calls against 5 961 bias-form.** The second field of the returned struct is Metal's sparse-residency byte.* No module in the corpus reads it (every extractvalue of a texture result@@ -527,6 +1835,8 @@ struct air_tex_name {bool depth; /* a depth texture rather than a colour one */enum glsl_sampler_dim dim;bool is_array, is_ms, is_buffer;+ bool grad; /* the _grad form: explicit dPdx and dPdy */+ bool no_offset; /* this dimensionality takes no offset argument */};static bool@@ -592,27 +1902,41 @@ parse_tex_name(const char *name, struct air_tex_name *t)else if (!eat(&s, "texture_"))return false;- /* the dimensionality, longest first so that 2d_ms_array beats 2d */+ /* The gradient form appends _grad to the DIMENSIONALITY and not to the+ * operation: Apple's builtin is __metal_sample_texture_2d_array_t_grad, so+ * the air.* name is air.sample_texture_2d_array_grad. Taking it off here+ * rather than giving the table nine more rows is what keeps one name for+ * one dimensionality -- and the exact comparison below is why 858 modules+ * used to stop at "a texture name this cannot take apart".+ */+ size_t len = strlen(s);+ if (len > 5 && !strcmp(s + len - 5, "_grad")) {+ t->grad = true;+ len -= 5;+ }++ /* the dimensionality, and whether it has an offset argument at all */static const struct {const char *suffix;enum glsl_sampler_dim dim;- bool array, ms;+ bool array, ms, no_offset;} dims[] = {- { "cube_array", GLSL_SAMPLER_DIM_CUBE, true, false },- { "2d_ms_array", GLSL_SAMPLER_DIM_MS, true, true },- { "2d_array", GLSL_SAMPLER_DIM_2D, true, false },- { "1d_array", GLSL_SAMPLER_DIM_1D, true, false },- { "2d_ms", GLSL_SAMPLER_DIM_MS, false, true },- { "cube", GLSL_SAMPLER_DIM_CUBE, false, false },- { "1d", GLSL_SAMPLER_DIM_1D, false, false },- { "2d", GLSL_SAMPLER_DIM_2D, false, false },- { "3d", GLSL_SAMPLER_DIM_3D, false, false },+ { "cube_array", GLSL_SAMPLER_DIM_CUBE, true, false, true },+ { "2d_ms_array", GLSL_SAMPLER_DIM_MS, true, true, false },+ { "2d_array", GLSL_SAMPLER_DIM_2D, true, false, false },+ { "1d_array", GLSL_SAMPLER_DIM_1D, true, false, false },+ { "2d_ms", GLSL_SAMPLER_DIM_MS, false, true, false },+ { "cube", GLSL_SAMPLER_DIM_CUBE, false, false, true },+ { "1d", GLSL_SAMPLER_DIM_1D, false, false, false },+ { "2d", GLSL_SAMPLER_DIM_2D, false, false, false },+ { "3d", GLSL_SAMPLER_DIM_3D, false, false, false },};for (unsigned i = 0; i < ARRAY_SIZE(dims); i++) {- if (!strcmp(s, dims[i].suffix)) {+ if (strlen(dims[i].suffix) == len && !strncmp(s, dims[i].suffix, len)) {t->dim = dims[i].dim;t->is_array = dims[i].array;t->is_ms = dims[i].ms;+ t->no_offset = dims[i].no_offset;return true;}}@@ -756,17 +2080,192 @@ all_zero(nir_def *def)return true;}+/* The coordinate, with the array index that follows it folded into it.+ *+ * NIR wants one coordinate with the layer as its last component; AIR passes+ * the two separately (see the signatures above). Which way the layer is+ * converted follows the coordinate's own type, which is also the only place+ * the answer can come from: a sample's coordinate is floating point and NIR's+ * array layer is then a float like the rest of it, while a read's is an+ * integer and the layer stays one.+ *+ * The float conversion is exact and not a rounding to be worried about. Metal+ * hands over an integer layer; u2f of an integer below 2^24 is exact in a+ * float, and 2^24 layers is 16 million slices of a texture array whose limit+ * on this hardware is 2 048. What a backend does with a *fractional* layer --+ * Vulkan says round to even, GLSL says nearest -- therefore never arises+ * here, because the value handed to it is always a whole number.+ *+ * An integer coordinate is widened to 32 bits first so that the layer, which+ * arrives as i32, does not have to be narrowed to meet it: air.read_texture_2d+ * has a 16-bit coordinate variant and the layer in those signatures is 32-bit+ * all the same.+ */+static nir_def *+tex_coord_and_layer(struct air_ctx *c, const struct air_inst *in,+ const struct air_tex_name *t, const struct air_name *n,+ unsigned *arg, unsigned ncoord)+{+ if (in->num_operands <= *arg) {+ air_error(c, "intrinsic: air.%s without a coordinate", n->part[0]);+ return NULL;+ }+ const struct air_value *cv = in->operands[*arg];+ nir_def *coord = air_get_ssa(c, cv);+ if (!coord)+ return NULL;+ (*arg)++;+ bool is_float = air_type_is_float(air_type_scalar(cv->type));+ if (!is_float)+ coord = nir_u2u32(&c->b, coord);++ if (t->is_array) {+ if (in->num_operands <= *arg) {+ air_error(c, "intrinsic: air.%s without the array index its signature has",+ n->part[0]);+ return NULL;+ }+ nir_def *layer = air_get_ssa(c, in->operands[(*arg)++]);+ if (!layer)+ return NULL;+ if (layer->num_components != 1) {+ air_error(c, "intrinsic: air.%s with a %u-component array index",+ n->part[0], layer->num_components);+ return NULL;+ }+ layer = is_float ? nir_u2fN(&c->b, layer, coord->bit_size)+ : nir_u2u32(&c->b, layer);++ nir_def *comps[4];+ unsigned nc = MIN2(coord->num_components, 3);+ for (unsigned i = 0; i < nc; i++)+ comps[i] = nir_channel(&c->b, coord, i);+ comps[nc] = layer;+ coord = nir_vec(&c->b, comps, nc + 1);+ }++ if (coord->num_components != ncoord) {+ air_error(c, "intrinsic: air.%s with a %u-component coordinate, wanting %u",+ n->part[0], coord->num_components, ncoord);+ return NULL;+ }+ return coord;+}+static booltexture_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n){+ /* THE NULL TEXTURE, named before the parser gets to shrug at it. 26 calls of+ * air.get_null_texture_2d and 8 of air.get_null_texture_3d in the system+ * corpus, all of them in QuartzCore's two YCbCr uber shaders -- and+ * fixed_frag_lph_cph is the module this driver is asked for more than any+ * other (104 of 105 pipeline refusals in one boot), so this is the last+ * thing between it and a pipeline.+ *+ * WHAT APPLE'S COMPILER DOES WITH IT, read rather than remembered.+ * AGCLLVMUserObject::replaceTexturesAndSamplers (0x1bbde9754) walks the+ * module's function list (body 0x1bbde9ed8, latch 0x1bbdea168) and looks+ * each name up with llvm::StringMapImpl::FindKey (0x1bbde9f04, libLLVM+ * 0x1bd5ddba0) in getGetNullTextureFunctionSet (0x1bbdf34a0, object+ * 0x1e29980c0), whose 16 names are "air.get_null_" (0x1bbf4867f) joined to+ * each suffix of the table at 0x1d9274398 -- texture_buffer_1d, texture_1d,+ * texture_1d_array, texture_2d, texture_2d_array, texture_3d, texture_cube,+ * texture_cube_array, texture_2d_ms, texture_2d_ms_array, depth_2d,+ * depth_2d_array, depth_cube, depth_cube_array, depth_2d_ms,+ * depth_2d_ms_array. On a hit it takes the function's return type and+ * replaces every call of it with a CONSTANT:+ *+ * with a texture heap ConstantInt::get(intTy, [this+0x84c], false)+ * (0x1bbde9f70, libLLVM 0x1bce62ec8) then+ * ConstantExpr::getIntToPtr(.., retTy, false)+ * (0x1bbde9f7c, libLLVM 0x1bce63694)+ * without one Constant::getNullValue(retTy)+ * (0x1bbdea178, libLLVM 0x1bce62bf4)+ *+ * and [this+0x84c] is written at 0x1bbe988b4 in+ * createTextureHeapIndexArray, from the SmallVector's own Size -- so the+ * heap index is ONE PAST THE LAST real texture.+ *+ * SO THERE IS NO RULE HERE FOR WHAT SAMPLING ONE YIELDS. Apple's compiler+ * substitutes a handle and leaves the answer to the descriptor that handle+ * names; the behaviour is the hardware's. Folding the sample to a value at+ * translation time therefore cannot be justified out of this -- it would+ * need the MSL specification or a measurement on an Apple GPU, and neither+ * is in reach, so it is NOT the road and that is a reading and not a+ * shrug.+ *+ * What the read does give is the other road's precedent, in Apple's own+ * structure: where textures are indices into a table, a null texture is a+ * RESERVED INDEX in that table. This frontend's textures are indices too+ * (Metal's own `[[texture(n)]]`, resolved while translating), so the+ * analogue is exact -- a zero-filled 1x1 texture the runtime binds at an+ * index past the module's last. docs/air.md 3 costs it: for+ * fixed_frag_lph_cph that index is 15, because it declares 0..14, and+ * RWA_MAX_TEXTURES is 16, so it fits with nothing to spare. The frontend's+ * half is small and the runtime's is not this file's.+ */+ if (!strncmp(n->part[0], "get_null_texture", 16)) {+ /* ROAD B, and the frontend's whole half of it: a reserved slot, flagged,+ * for the runtime to bind a 1x1 texture of zeros at. The sample through+ * it is an ordinary sample at an ordinary index, which is why nothing+ * else in this file changes -- and the shape the runtime must make it is+ * derived from that sample like any other slot's (slot_shapes), so a+ * `get_null_texture_3d` reaches the driver as a 3D slot without this+ * function saying so.+ */+ int slot = air_null_texture_slot(c);+ if (slot < 0)+ return false; /* null_texture_slot has said why */+ struct air_ptr p = { .class = AIR_PTR_TEXTURE, .binding = (unsigned)slot };+ air_set(c, &in->base, air_val_ptr(c, p));+ return true;+ }+ if (!strncmp(n->part[0], "is_null_texture", 15)) {+ /* Exact, and a constant, because the question is one this frontend has+ * already answered: a texture is resolved to an index while translating,+ * so "is this the null one" is "is this the reserved slot". Folding it+ * here is not an optimisation -- an argument that did not resolve to a+ * texture never reaches this line, because tex_handle refuses it first.+ */+ struct air_val *t = air_get(c, in->operands[1]);+ if (!t)+ return false;+ if (t->kind != AIR_VAL_PTR || t->ptr.class != AIR_PTR_TEXTURE)+ return air_error(c, "intrinsic: air.%s of something this frontend did not "+ "resolve to a texture", n->part[0]);+ bool is_null = c->null_texture_index >= 0 &&+ t->ptr.binding == (unsigned)c->null_texture_index;+ air_set(c, &in->base, air_val_ssa(c, nir_imm_bool(&c->b, is_null)));+ return true;+ }+struct air_tex_name t;if (!parse_tex_name(n->part[0], &t))return air_error(c, "intrinsic: air.%s (a texture name this cannot take apart)",n->part[0]);if (t.is_buffer)return air_error(c, "intrinsic: air.%s (texture buffers, not yet)", n->part[0]);- if (t.compare)- return air_error(c, "intrinsic: air.%s (compare sampling, not yet)", n->part[0]);+ /* A comparison belongs to a depth texture and to a sample: Apple's header declares+ * sample_compare and gather_compare on depth2d and depth2d_array and on nothing+ * else, so a name claiming either of the other two shapes is a misparse and not a+ * gap. Said by name rather than translated into a plausible instruction.+ */+ if (t.compare && !t.depth)+ return air_error(c, "intrinsic: air.%s (a comparison of a colour texture, which "+ "Metal declares on depth textures only)", n->part[0]);+ if (t.compare && !strcmp(t.op, "gather"))+ return air_error(c, "intrinsic: air.%s (gather_compare: four comparisons in one "+ "instruction, which nothing measured here attests -- no module "+ "of the 456 and none of Resident Evil 4's 7 879 calls one -- so "+ "there is nothing to check an answer against)", n->part[0]);+ /* Only a sample has a gradient form -- Apple's header has+ * __metal_sample_*_t_grad and no gather, read or write counterpart -- and a+ * name that claimed one would be read by the sample branch's rules below,+ * which is the kind of silent misparse this file refuses instead.+ */+ if (t.grad && strcmp(t.op, "sample"))+ return air_error(c, "intrinsic: air.%s (a gradient form of %s, which Metal has no "+ "signature for)", n->part[0], t.op);struct air_ptr tex;if (!tex_handle(c, in, 1, AIR_PTR_TEXTURE, &tex))@@ -832,10 +2331,26 @@ texture_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *}if (!strcmp(t.op, "write")) {- nir_def *coord = air_get_ssa(c, in->operands[arg++]);+ nir_def *coord = tex_coord_and_layer(c, in, &t, n, &arg, ncoord);+ if (!coord)+ return false;+ /* The store's type is the VALUE's, not the call's: a write returns void,+ * so dest_type above (from the return type) can only say "uint", and ir3+ * would then emit stib.b.typed.*.u16/u32 for a half or float value --+ * a typed store converts its source as the type says, so the texture+ * would receive the bit pattern of 1.0h read as the integer 15360 (that+ * much is the ISA's rule, not a measurement: hakios-metal's dispatch+ * refuses a uint-typed kernel over a float format before it runs, and+ * that refusal is what the old typing hit on the phone with Apple's+ * MetalFaceMask_Clear, a texture2d<half, write>). With the store typed+ * by its value, hakios-metal's tex_fill_h.ll landed 2048 of 2048 exact+ * half bit patterns in an RGBA16Float image on 29 October (hakios+ * logs/012127-run.log). Signedness still comes from the name (.u/.s). */+ unsigned value_arg = arg;nir_def *value = air_get_ssa(c, in->operands[arg++]);- if (!coord || !value)+ if (!value)return false;+ nir_alu_type src_type = tex_dest_type(c, in->operands[value_arg]->type, n);uint64_t lod;if (!tex_flag(c, in, arg, &lod))return false;@@ -850,17 +2365,14 @@ texture_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *nir_imm_int(&c->b, 0), nir_pad_vector(&c->b, value, 4),nir_imm_int(&c->b, 0),.image_dim = t.dim, .image_array = t.is_array,- .src_type = dest_type);+ .src_type = src_type);return true;}/* sample, gather, read: a texture instruction, or an image load */- nir_def *coord = air_get_ssa(c, in->operands[arg++]);+ nir_def *coord = tex_coord_and_layer(c, in, &t, n, &arg, ncoord);if (!coord)return false;- if (coord->num_components != ncoord)- return air_error(c, "intrinsic: air.%s with a %u-component coordinate, wanting %u",- n->part[0], coord->num_components, ncoord);if (!strcmp(t.op, "read") && tex.is_image) {uint64_t extra;@@ -876,59 +2388,158 @@ texture_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *return true;}- nir_tex_src srcs[6];+ /* The value a depth texel is compared against, which Metal passes straight after+ * the coordinate AND AFTER THE ARRAY INDEX -- __metal_sample_compare_depth_2d_t+ * takes (tex, samp, coord, compare_value, ...) and the _2d_array form takes (tex,+ * samp, coord, array, compare_value, ...), so the layer comes first in exactly the+ * place tex_coord_and_layer has already consumed it. Read here and not in the+ * branches below because it is before the offset pair in the plain form and before+ * the gradients in the _grad one: one position, not two.+ *+ * At the AIR level there is also the aspect selector the depth branch above ate, as+ * there is in air.sample_depth_2d -- Apple's header has no such argument and the+ * declaration in texturedFragment2DDepth does:+ * { float, i8 } air.sample_depth_2d.f32(ptr as(1), ptr as(2), i32, <2 x float>,+ * i1, <2 x i32>, i1, float, float, i32)+ * which is where the `i32` between the sampler and the coordinate comes from, and+ * the only reason the compare form's shape could be read off the header at all.+ */+ nir_def *compare = NULL;+ if (t.compare) {+ if (in->num_operands <= arg)+ return air_error(c, "intrinsic: air.%s without the value its signature compares "+ "against", n->part[0]);+ if (!(compare = air_get_ssa(c, in->operands[arg++])))+ return false;+ if (compare->num_components != 1)+ return air_error(c, "intrinsic: air.%s compares against %u components, and a "+ "depth comparison takes one", n->part[0],+ compare->num_components);+ /* The texel and the value compared with it go into one register tuple on the+ * Adreno -- ir3 puts the comparison straight after the coordinates in src0 --+ * so a 16-bit comparison value beside 32-bit coordinates is a half register in a+ * full tuple. Metal's own signature is `float`, so this never fires on anything+ * Apple's compiler emits; it is here so that if something does, it is widened+ * and not mixed.+ */+ if (compare->bit_size != coord->bit_size)+ compare = nir_f2fN(&c->b, compare, coord->bit_size);+ }++ /* Seven and not six: a compare sample in the gradient form carries a coordinate, a+ * comparison value, an offset, two derivatives and a minimum level at once, which is+ * exactly six, and the next source anyone adds should not be the one that overflows+ * this silently.+ */+ nir_tex_src srcs[7];unsigned nsrc = 0;srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_coord,- .src = nir_src_for_ssa(- air_type_is_float(air_type_scalar(- in->operands[arg - 1]->type))- ? coord : nir_u2u32(&c->b, coord)) };+ .src = nir_src_for_ssa(coord) };+ if (compare)+ srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_comparator,+ .src = nir_src_for_ssa(compare) };nir_texop op;if (!strcmp(t.op, "sample")) {- uint64_t has_offset, is_level;- if (!tex_flag(c, in, arg, &has_offset))- return false;- arg++;- nir_def *offset = air_get_ssa(c, in->operands[arg++]);- if (!offset)- return false;- if (has_offset && !all_zero(offset))+ /* The offset pair, for the dimensionalities that have one. A cube does+ * not, and reading one there consumed the level flag and the level and+ * left the minimum level to be read as the flags word -- which is the+ * refusal 621 of Resident Evil 4's modules used to earn, "argument 5 is+ * not a constant", from a float that indeed is not an integer constant.+ *+ * In the gradient form this pair comes AFTER the gradients, so the+ * reading is a small state machine over the signature rather than one+ * fixed order: see the table at the head of this section.+ */+ uint64_t has_offset = 0, is_level = 0;+ nir_def *offset = NULL, *ddx = NULL, *ddy = NULL, *min_lod = NULL, *level = NULL;++ if (t.grad) {+ ddx = air_get_ssa(c, in->operands[arg++]);+ ddy = air_get_ssa(c, in->operands[arg++]);+ min_lod = air_get_ssa(c, in->operands[arg++]);+ if (!ddx || !ddy || !min_lod)+ return false;+ if (!t.no_offset) {+ if (!tex_flag(c, in, arg, &has_offset))+ return false;+ arg++;+ if (!(offset = air_get_ssa(c, in->operands[arg++])))+ return false;+ }+ } else {+ if (!t.no_offset) {+ if (!tex_flag(c, in, arg, &has_offset))+ return false;+ arg++;+ if (!(offset = air_get_ssa(c, in->operands[arg++])))+ return false;+ }+ if (!tex_flag(c, in, arg, &is_level))+ return false;+ arg++;+ level = air_get_ssa(c, in->operands[arg++]);+ min_lod = air_get_ssa(c, in->operands[arg++]);+ if (!level || !min_lod)+ return false;+ }++ if (offset && has_offset && !all_zero(offset))srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_offset,.src = nir_src_for_ssa(nir_u2u32(&c->b, offset)) };- if (!tex_flag(c, in, arg, &is_level))- return false;- arg++;- nir_def *level = air_get_ssa(c, in->operands[arg++]);- nir_def *min_lod = air_get_ssa(c, in->operands[arg++]);- if (!level || !min_lod)- return false;- bool level_is_zero = nir_src_is_const(nir_src_for_ssa(level)) &&- nir_src_as_float(nir_src_for_ssa(level)) == 0.0;- if (is_level) {- op = nir_texop_txl;- srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_lod,- .src = nir_src_for_ssa(level) };- } else if (!level_is_zero) {- op = nir_texop_txb;- srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_bias,- .src = nir_src_for_ssa(level) };++ if (t.grad) {+ /* An explicit pair of derivatives of the coordinate, which is what+ * Metal's sample(..., gradient2d(dPdx, dPdy)) is for: a shader that+ * knows better than the quad's own differences -- a ray-marched or+ * reprojected coordinate, or one sampled inside a branch the quad+ * does not take together. NIR has the operation (txd) and so does+ * the Adreno; ir3 emits it and softpipe's interpreter has TXD, so+ * unlike most of this family it can be both compiled and run here.+ *+ * The gradients carry the coordinate's components WITHOUT the array+ * layer, which is what NIR asks for too (nir_tex_instr_src_size:+ * coord_components minus is_array for ddx and ddy) and what AIR+ * passes -- <2 x float> for a 2d_array whose coordinate is also+ * <2 x float>. So nothing is added or dropped here, and that+ * agreement is the reason this needs no reshaping.+ */+ op = nir_texop_txd;+ srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_ddx,+ .src = nir_src_for_ssa(ddx) };+ srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_ddy,+ .src = nir_src_for_ssa(ddy) };} else {- op = nir_texop_tex;+ bool level_is_zero = nir_src_is_const(nir_src_for_ssa(level)) &&+ nir_src_as_float(nir_src_for_ssa(level)) == 0.0;+ if (is_level) {+ op = nir_texop_txl;+ srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_lod,+ .src = nir_src_for_ssa(level) };+ } else if (!level_is_zero) {+ op = nir_texop_txb;+ srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_bias,+ .src = nir_src_for_ssa(level) };+ } else {+ op = nir_texop_tex;+ }}+if (!(nir_src_is_const(nir_src_for_ssa(min_lod)) &&nir_src_as_float(nir_src_for_ssa(min_lod)) == 0.0))srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_min_lod,.src = nir_src_for_ssa(min_lod) };} else if (!strcmp(t.op, "gather")) {- uint64_t has_offset;- if (!tex_flag(c, in, arg, &has_offset))- return false;- arg++;- nir_def *offset = air_get_ssa(c, in->operands[arg++]);- if (!offset)- return false;- if (has_offset && !all_zero(offset))+ uint64_t has_offset = 0;+ nir_def *offset = NULL;+ if (!t.no_offset) {+ if (!tex_flag(c, in, arg, &has_offset))+ return false;+ arg++;+ if (!(offset = air_get_ssa(c, in->operands[arg++])))+ return false;+ }+ if (offset && has_offset && !all_zero(offset))srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_offset,.src = nir_src_for_ssa(nir_u2u32(&c->b, offset)) };op = nir_texop_tg4;@@ -952,7 +2563,15 @@ texture_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *tex_instr->op = op;tex_instr->sampler_dim = t.dim;tex_instr->is_array = t.is_array;- tex_instr->is_shadow = false;+ /* is_new_style_shadow because Metal's sample_compare returns ONE value, which the+ * declaration says too -- air.sample_compare_depth_2d.f32 returns { float, i8 }, not+ * { <4 x float>, i8 } the way a colour sample does. NIR's old-style shadow+ * replicates the comparison into four components and would leave the shader reading+ * component 0 of four where the signature has one, which is the same answer by+ * accident; this says what the operation is.+ */+ tex_instr->is_shadow = t.compare;+ tex_instr->is_new_style_shadow = t.compare;tex_instr->dest_type = dest_type;tex_instr->coord_components = ncoord;tex_instr->texture_index = tex.binding;@@ -1005,12 +2624,26 @@ air_translate_call(struct air_ctx *c, const struct air_inst *in, const char *namif (!split_name(name, &n))return air_error(c, "intrinsic: %s (a name this cannot take apart)", name);+ if (!strcmp(n.part[0], "is_function_constant_defined"))+ return fc_defined_op(c, in);if (n.num >= 2 && !strcmp(n.part[0], "wg") && !strcmp(n.part[1], "barrier"))- return barrier_op(c, in);+ return barrier_op(c, in, false);+ if (n.num >= 2 && !strcmp(n.part[0], "simdgroup") && !strcmp(n.part[1], "barrier"))+ return barrier_op(c, in, true);if (!strcmp(n.part[0], "atomic"))return atomic_op(c, in, &n);+ if (!strcmp(n.part[0], "discard_fragment"))+ return discard_op(c, in);bool handled = false;+ /* before math_op, which resolves every argument as a number: these take a+ * lane number or nothing at all, and air.simd_sum falls through to it+ */+ if (!simd_op(c, in, &n, &handled))+ return false;+ if (handled)+ return true;+if (!math_op(c, in, &n, &handled))return false;if (handled)@@ -1022,5 +2655,62 @@ air_translate_call(struct air_ctx *c, const struct air_inst *in, const char *namstrstr(op, "_lod_"))return texture_op(c, in, &n);+ /* The rest of Apple's pack family, named as absent rather than as undecided.+ *+ * air.pack.unorm.rgb565 is translated above, off AGXCompilerCore's lowering.+ * The same reading gave the whole of that binary's `formats` table, so the+ * field layout of air.pack.{s,u}norm.rgb10a2, air.pack.unorm.rgb10 and the+ * 4444/1555/5551 rows the driver's own blit shaders use is known too -- but+ * no module of either corpus calls one. The rest take a different path in+ * that binary and it has not been read: the {s,u}norm NxM family that+ * initializePackUnpackMap builds out of its own component-count and width+ * tables (air.pack.unorm4x8.v4f32 and its relatives) goes to buildPackNorm,+ * and air.pack.unorm4x8.srgb, air.pack.unorm.rg11b10f and+ * air.pack.unorm.rgb9e5 go to buildPackSRGB, buildPack111110 and+ * buildPack9995. So this is "not implemented", with the places to read+ * named, and not "cannot be decided".+ */+ /* air.pack.{u,s}norm{4x8,2x16}.v<N>f{16,32} and air.unpack the other way:+ * Metal's pack_float_to_unorm4x8 and its kin, which are GLSL's packUnorm4x8+ * and its kin -- clamp, scale, round to nearest, component 0 in the low+ * bits -- and so NIR's pack_unorm_4x8 and the rest, over 32-bit floats (a+ * half vector is widened first, exactly). VideoToolbox's scaler+ * (VTMTSComputeFunction, the live wallpaper's) writes its pixels so.+ */+ if ((!strcmp(op, "pack") || !strcmp(op, "unpack")) && n.num >= 2) {+ const char *f = n.part[1];+ const bool un = !strcmp(f, "unorm4x8") || !strcmp(f, "unorm2x16");+ const bool sn = !strcmp(f, "snorm4x8") || !strcmp(f, "snorm2x16");+ if (un || sn) {+ const bool x8 = strstr(f, "4x8") != NULL;+ if (!strcmp(op, "pack")) {+ nir_def *v = in->num_operands > 1 ? air_get_ssa(c, in->operands[1]) : NULL;+ if (!v)+ return false;+ if (v->bit_size != 32)+ v = nir_f2f32(&c->b, v);+ nir_def *r = x8 ? (un ? nir_pack_unorm_4x8(&c->b, v) : nir_pack_snorm_4x8(&c->b, v))+ : (un ? nir_pack_unorm_2x16(&c->b, v) : nir_pack_snorm_2x16(&c->b, v));+ air_set(c, &in->base, air_val_ssa(c, r));+ return true;+ }+ nir_def *w = in->num_operands > 1 ? air_get_ssa(c, in->operands[1]) : NULL;+ if (!w)+ return false;+ nir_def *r = x8 ? (un ? nir_unpack_unorm_4x8(&c->b, w) : nir_unpack_snorm_4x8(&c->b, w))+ : (un ? nir_unpack_unorm_2x16(&c->b, w) : nir_unpack_snorm_2x16(&c->b, w));+ const unsigned want = air_nir_bit_size(c, in->base.type);+ if (want == 16)+ r = nir_f2f16(&c->b, r);+ air_set(c, &in->base, air_val_ssa(c, r));+ return true;+ }+ }++ if (!strcmp(op, "pack"))+ return air_error(c, "intrinsic: %s -- of Apple's pack family only "+ "unorm.rgb565 is translated, because only it occurs; "+ "the rest are unread lowerings, not undecided ones", name);+return air_error(c, "intrinsic: %s", name);}diff --git a/tools/mesa/air/air_layout.c b/tools/mesa/air/air_layout.cindex 313d498..60e194c 100644--- a/tools/mesa/air/air_layout.c+++ b/tools/mesa/air/air_layout.c@@ -5,7 +5,7 @@/** Sizes and alignments from the module's datalayout string, by LLVM's rules- * (LangRef, "Data Layout"). Every AIR module of Darwin userland 16.5 has the same layout,+ * (LangRef, "Data Layout"). Every AIR module of iOS 16.5 has the same layout,* but it is read rather than assumed: GEP offsets are computed from it, and* a wrong offset is a wrong read.*/diff --git a/tools/mesa/air/air_metallib.c b/tools/mesa/air/air_metallib.cindex 05020b6..f94492e 100644--- a/tools/mesa/air/air_metallib.c+++ b/tools/mesa/air/air_metallib.c@@ -6,7 +6,7 @@/** The MTLB container of a .metallib: a header, a function list of tagged* fields, and one LLVM module per function. Worked out against the files of- * Darwin userland 16.5 (docs/metal.md, section 9.1; tools/metallib.py is the same in+ * iOS 16.5 (docs/metal.md, section 9.1; tools/metallib.py is the same in* Python).*/diff --git a/tools/mesa/air/air_module.c b/tools/mesa/air/air_module.cindex d908885..534c80a 100644--- a/tools/mesa/air/air_module.c+++ b/tools/mesa/air/air_module.c@@ -851,12 +851,33 @@ read_metadata_kind_block(struct reader *r)/* -- module records ----------------------------------------------------- */+/* Module version 1 (LLVM before 5) has no string table: a GLOBALVAR or FUNCTION+ * record is the version 2 one without its first two operands (the name's place in+ * the table), and the names come from the module's own symbol table instead+ * (read_module_vst). Apple's compiler service still writes version 1 for some of+ * the MSL it compiles at run time (sokol-samples' debugtext and drawex), so such a+ * record is read as the version 2 one with a name of no length, named later.+ */+static const struct air_bs_entry *+as_version2(struct reader *r, const struct air_bs_entry *e, struct air_bs_entry *copy)+{+ if (r->m->version >= 2)+ return e;+ uint64_t *ops = ralloc_array(r->m, uint64_t, e->num_ops + 2);+ ops[0] = ops[1] = 0;+ memcpy(ops + 2, e->ops, e->num_ops * sizeof(uint64_t));+ *copy = *e;+ copy->ops = ops;+ copy->num_ops = e->num_ops + 2;+ return copy;+}+static bool-read_globalvar(struct reader *r, const struct air_bs_entry *e)+read_globalvar(struct reader *r, const struct air_bs_entry *e1){struct air_module *m = r->m;- if (r->m->version < 2)- return fail(r, "module version %u: only bitcode with a string table is read", m->version);+ struct air_bs_entry copy;+ const struct air_bs_entry *e = as_version2(r, e1, ©);if (e->num_ops < 8)return fail(r, "GLOBALVAR too short");@@ -901,11 +922,11 @@ read_globalvar(struct reader *r, const struct air_bs_entry *e)}static bool-read_function_record(struct reader *r, const struct air_bs_entry *e)+read_function_record(struct reader *r, const struct air_bs_entry *e1){struct air_module *m = r->m;- if (m->version < 2)- return fail(r, "module version %u: only bitcode with a string table is read", m->version);+ struct air_bs_entry copy;+ const struct air_bs_entry *e = as_version2(r, e1, ©);if (e->num_ops < 8)return fail(r, "FUNCTION too short");@@ -1289,6 +1310,45 @@ read_instruction(struct reader *r, const struct air_bs_entry *e, struct air_blocreturn fail(r, "br of %u operands", e->num_ops);}break;+ case FUNC_CODE_INST_SWITCH: {+ /* [ type of the condition, condition, default block,+ * (case constant, block)* ]+ *+ * The condition is a RELATIVE value number and each case constant is an+ * ABSOLUTE one, because that is what the writer does -- pushValue for+ * the condition, getValueID for each case value. Reading them the same+ * way gives the wrong constant for every case in any function that has+ * more than a handful of values, which is every function here.+ *+ * The blocks are block numbers, as br's are, not value numbers.+ */+ if (e->num_ops < 3 || (e->num_ops - 3) % 2)+ return fail(r, "a switch of %u operands", e->num_ops);+ unsigned ty = take(&rc);+ /* LLVM 3.0 wrote switches over ranges, with the case count and a bit+ * width packed into the first operand under this magic instead of a type+ * id. Nothing Apple's compiler emits uses it, and reading it as a type+ * would silently take the wrong path, so it is named and refused.+ */+ if ((ty >> 16) == 0x4B5)+ return fail(r, "a switch in LLVM 3.0's range form (SWITCH_INST_MAGIC)");+ unsigned cond = take_value(r, &rc);+ unsigned deflt = take(&rc);+ unsigned n = (e->num_ops - 3) / 2;+ in = new_inst(r, AIR_OP_SWITCH, NULL, 1 + n);+ in->num_targets = 1 + n;+ in->target_ids = ralloc_array(m, unsigned, 1 + n);+ /* target 0 and operand 0 are the default edge and the condition; the+ * cases follow, so case k is operand k+1 and target k+1+ */+ in->target_ids[0] = deflt;+ in->operand_ids[0] = cond;+ for (unsigned k = 0; k < n; k++) {+ in->operand_ids[1 + k] = take(&rc);+ in->target_ids[1 + k] = take(&rc);+ }+ break;+ }case FUNC_CODE_INST_UNREACHABLE:in = new_inst(r, AIR_OP_UNREACHABLE, NULL, 0);break;@@ -1651,6 +1711,29 @@ read_function_block(struct reader *r)/* -- the module block --------------------------------------------------- */+/* A version 1 module's symbol table: ENTRY [value id, name...] for a global or a+ * declaration, FNENTRY [value id, offset of its body, name...] for a function with+ * a body. Version 2 keeps the same names in the string table and has no use for it.+ */+static bool+read_module_vst(struct reader *r)+{+ for (;;) {+ struct air_bs_entry e = air_bs_next(&r->bs);+ if (e.kind == AIR_BS_END_BLOCK)+ return true;+ if (e.kind != AIR_BS_RECORD)+ return check_bs(r) && fail(r, "module symbol table ends early");+ unsigned skip = e.id == VST_CODE_ENTRY ? 1 : e.id == VST_CODE_FNENTRY ? 2 : 0;+ if (!skip || e.num_ops < skip)+ continue;+ struct air_value *v = value_by_id(r, e.ops[0]);+ if (!v)+ return false;+ v->name = air_bs_ops_to_string(r->m, e.ops + skip, e.num_ops - skip);+ }+}+static boolread_module_block(struct reader *r){@@ -1692,9 +1775,18 @@ read_module_block(struct reader *r)air_bs_enter(&r->bs, e.id);ok = ok && read_function_block(r);break;+ case VALUE_SYMTAB_BLOCK_ID:+ /* a version 1 module's names (read_module_vst); version 2's are the string table's */+ if (m->version < 2) {+ air_bs_enter(&r->bs, e.id);+ ok = read_module_vst(r);+ } else {+ ok = air_bs_skip_block(&r->bs);+ }+ break;default:- /* attributes, symbol table, operand bundle tags, sync scopes,- * use lists: nothing the translation depends on+ /* attributes, operand bundle tags, sync scopes, use lists: nothing+ * the translation depends on*/ok = air_bs_skip_block(&r->bs);break;@@ -1870,12 +1962,14 @@ air_module_read(void *mem_ctx, const void *data, size_t size, const char **errorgoto out;}- /* the string table comes after the module, so names come last */- for (unsigned i = 0; i < m->num_globals; i++)+ /* the string table comes after the module, so names come last (a version 1+ * module named them from its symbol table already, read_module_vst)+ */+ for (unsigned i = 0; m->version >= 2 && i < m->num_globals; i++)if (!(m->globals[i]->base.name =strtab_name(&r, m->globals[i]->strtab_offset, m->globals[i]->strtab_size)))goto out;- for (unsigned i = 0; i < m->num_functions; i++)+ for (unsigned i = 0; m->version >= 2 && i < m->num_functions; i++)if (!(m->functions[i]->base.name =strtab_name(&r, m->functions[i]->strtab_offset, m->functions[i]->strtab_size)))goto out;diff --git a/tools/mesa/air/air_module.h b/tools/mesa/air/air_module.hindex b0aede9..7ea7c95 100644--- a/tools/mesa/air/air_module.h+++ b/tools/mesa/air/air_module.h@@ -6,7 +6,7 @@/** The part of LLVM IR that AIR uses, read from bitcode into plain C* structures. It is a reader, not an IR library: nothing here is modified- * after reading, and only what the 456 modules of Darwin 16.5's system+ * after reading, and only what the 456 modules of iOS 16.5's system* libraries contain (plus a little margin) is understood. Anything else is an* error that names the record, never a silent skip, because a skipped* instruction is a wrong shader.diff --git a/tools/mesa/air/air_print.c b/tools/mesa/air/air_print.cindex 5654a10..bc70239 100644--- a/tools/mesa/air/air_print.c+++ b/tools/mesa/air/air_print.c@@ -637,6 +637,20 @@ print_inst(struct printer *p, const struct air_inst *in)print_block_ref(p, in->targets[1]);}break;+ case AIR_OP_SWITCH:+ fputs("switch ", fp);+ print_typed(p, in->operands[0]);+ fputs(", ", fp);+ print_block_ref(p, in->targets[0]);+ fputs(" [", fp);+ for (unsigned k = 1; k < in->num_targets; k++) {+ fputs("\n ", fp);+ print_typed(p, in->operands[k]);+ fputs(", ", fp);+ print_block_ref(p, in->targets[k]);+ }+ fputs("\n ]", fp);+ break;case AIR_OP_UNREACHABLE:fputs("unreachable", fp);break;diff --git a/tools/mesa/air/air_private.h b/tools/mesa/air/air_private.hindex e8760dd..1ebb930 100644--- a/tools/mesa/air/air_private.h+++ b/tools/mesa/air/air_private.h@@ -29,6 +29,32 @@ enum air_ptr_class {AIR_PTR_FUNCTION_CONSTANT, /* a function constant: .binding is its index */AIR_PTR_TEXTURE, /* a texture handle; offset unused */AIR_PTR_SAMPLER, /* a sampler handle */++ /* An ARGUMENT BUFFER (`air.indirect_buffer`): .binding is the set's number+ * in air_ctx.argbufs and .offset the byte offset into it. Nothing is ever+ * loaded from such a pointer as a value -- a load through it RESOLVES to+ * the member that byte offset names (argbuf_load), which is the whole of+ * why argument buffers need no addresses here.+ */+ AIR_PTR_ARGBUF,++ /* A LOCAL, OR A READ-ONLY GLOBAL, THAT HOLDS A POINTER -- kept as a+ * translation-time slot and not as memory, because a pointer in this+ * frontend is a resolution and has no bytes to put anywhere. .binding is+ * the slot's number in air_ctx.slots, and a slot holds a whole air_val, so+ * what comes back out of one may itself be a texture or a sampler.+ *+ * Only a whole load or store at offset 0 reaches a slot. That is not a+ * simplification: the two shapes this exists for are `alloca ptr+ * addrspace(2)` followed by a store of a sampler argument, and+ * `constant %"struct.metal::sampler" { ptr addrspace(2) @__air_sampler_state }`+ * read through a getelementptr whose every index is zero -- and offset 0 of+ * a struct is offset 0 of its first member, transitively, which is the only+ * byte arithmetic a tree of values can answer. Anything else is refused by+ * name rather than resolved to whichever member the offset happened to land+ * on (air_to_nir.c, slot_at_zero).+ */+ AIR_PTR_SLOT,};struct air_ptr {@@ -44,6 +70,21 @@ enum air_val_kind {AIR_VAL_SSA, /* a scalar or vector */AIR_VAL_AGG, /* a struct or array, as a tree of values */AIR_VAL_PTR,++ /* A POINTER as a value rather than a resolution. air_ptr is *(where, byte+ * offset)* decided while translating, which is what lets this frontend emit+ * load_ubo at Metal's own index with no descriptor indirection; a pointer+ * CHOSEN at run time has no such resolution, so the choice itself is kept+ * and made where the pointer is used. A pointer STORED to memory is the+ * other half of the same problem and has no form here at all -- see+ * docs/air.md section 3 for what that costs.+ */++ /* `select i1 %c, ptr %a, ptr %b`: .def is the condition and .elems[0..1]+ * are the two pointers. A LOAD through it loads BOTH and selects the+ * values, which is the only form NIR has; that is why it leaves a note.+ */+ AIR_VAL_PTR_SELECT,};struct air_val {@@ -54,6 +95,43 @@ struct air_val {struct air_val **elems;};+/* One member of an argument buffer, as `air.struct_type_info` declares it and+ * the module's own struct type confirms it.+ *+ * Keyed on `offset` and not on the member number, which is what makes the+ * three shapes SPIRV-Cross emits one case: a buffer member is the pointer+ * itself, while a texture or sampler member is a one-field+ * `metal::texture2d` / `metal::sampler` wrapper whose field sits at byte 0 of+ * it -- so the extra `getelementptr` into the wrapper adds nothing to the+ * offset and resolves by the same lookup (docs/air.md section 8.2).+ */+struct air_argbuf_member {+ uint64_t offset; /* bytes from the start of the set: the KEY */+ enum air_resource_kind kind;+ unsigned metal_id; /* the member's own air.location_index: Metal's [[id(n)]] */+ unsigned index; /* the flat per-kind index this frontend allocated */++ /* A buffer member: 1 device, 2 constant. The metadata does not say -- not+ * one of the 82 048 members of Resident Evil 4's 7 857 argument buffers+ * carries `air.address_space` -- so this comes from the module's own struct+ * type, and every load through the member checks the pointer it loads+ * against it (argbuf_load). 0 for a texture or a sampler.+ */+ unsigned addrspace;+ bool read, write;+ const char *name, *type_name;+ const struct air_md *node; /* the member's own air.indirect_argument descriptor */+};++struct air_argbuf {+ unsigned metal_index; /* the SET's own Metal [[buffer(n)]] */+ unsigned size; /* air.buffer_size: eight bytes per member */+ unsigned param; /* the entry point's parameter the set arrives as */+ const char *name, *type_name;+ unsigned num_members;+ struct air_argbuf_member *members;+};+struct air_ctx {void *mem_ctx;const struct air_module *m;@@ -72,6 +150,19 @@ struct air_ctx {struct hash_table *phi_cursors;/* phi -> nir register decl */struct hash_table *phi_regs;+ /* Where this function's constants are built: the top of its entry block,+ * which dominates every use of them. air_get caches a constant the first+ * time it is asked for, so the def has to be somewhere every later block+ * can see; built at the cursor of the first use, a constant shared by a+ * branch and the block after it is a non-dominating use, and nir_repair_ssa+ * turns that into undef on the path that misses the branch.+ */+ nir_cursor const_cursor;+ /* set while a constant is being built there, so that the elements of an+ * aggregate constant -- which come through air_get too -- keep appending+ * after it instead of each restoring the cursor to where the aggregate+ * itself will go, which put a vec before the components it reads */+ bool in_const;/* A call of another function in the module becomes a nir_call, and* nir_inline_functions folds it in afterwards. These two carry the state@@ -85,6 +176,19 @@ struct air_ctx {struct hash_table *fns;nir_deref_instr *ret_deref;+ /* ... and, for a callee whose return type is a STRUCT, one out-parameter per+ * member instead of one for the whole value: a NIR function parameter+ * carries a number, and a struct is a tree of values in this translator and+ * never a NIR value at all. ret_fields is indexed by the AIR member number+ * and is NULL for a member that is not a number -- the `[8 x i8]` tail+ * padding that Metal's own vertex-output structs carry, which takes no+ * parameter because nothing may read or write it. num_ret_fields is+ * non-zero only while such a callee is being translated, and that is how+ * emit_return tells this case from the scalar one.+ */+ nir_deref_instr **ret_fields;+ unsigned num_ret_fields;+/* What each module global resolves to, decided once (plan_globals) and* turned into a value in whichever function needs it (materialise_globals).* The two are separate because the deciding half has side effects that must@@ -101,12 +205,53 @@ struct air_ctx {unsigned offset; /* bytes, when has_offset */bool has_offset;bool skip; /* llvm.* lists, and anything that failed */++ /* A PROGRAM-SCOPE VARIABLE THAT IS A VALUE, not memory. Set by+ * plan_global_folding for a scalar integer global that the module only+ * ever loads and stores WHOLE, whose single store is in its+ * air.static_init constructor, and whose address never escapes. `known`+ * and `bits` are then filled when that store is translated and the value+ * turns out to be a constant -- which it is when it came from a function+ * constant -- and every later load answers with the constant instead of+ * a load_scratch. The scratch slot is still allocated and still written:+ * the fold is an answer on top of memory that is there, not a promise+ * that nothing reads it (air_to_nir.c, AIR_OP_LOAD).+ */+ bool fold;+ bool known;+ uint64_t bits;+ unsigned bit_size;} *global_plan;+ /* The translation-time slots: locals and read-only globals that hold a+ * POINTER. Grown as allocas are met; the read-only globals reserve theirs+ * in plan_globals, before any body is translated, and materialise_globals+ * fills each one from the global's initialiser in whatever builder is+ * asking -- the same rule the offsets follow, and for the same reason.+ */+ struct air_slot {+ struct air_val *val;+ const struct air_type *type; /* what a whole load of it yields */+ const struct air_value *init; /* a read-only global's initialiser, or NULL */+ } *slots;+ unsigned num_slots;++ /* The reserved texture slot the null texture lives in, or -1 before anything+ * has asked for one. Allocated once per shader and shared by every+ * air.get_null_texture_* call in it, because a module with fifteen of them+ * (fixed_frag_lph_cph) wants one zero texture and not fifteen.+ */+ int null_texture_index;+/* what the entry point's parameters are */unsigned num_params;struct air_val **params;+ /* the argument buffers the entry point declares, in the order its input+ * metadata lists them, which is the order their members are numbered in */+ unsigned num_argbufs;+ struct air_argbuf *argbufs;+/* outputs, in the order of the returned struct's fields */unsigned num_outputs;struct {@@ -117,6 +262,7 @@ struct air_ctx {unsigned scratch_size;unsigned shared_size;+ unsigned tg_param_plus1; /* the [[threadgroup(0)]] argument's parameter + 1; 0 none *//* module-scope variables, by their air_global */struct hash_table *globals;@@ -139,7 +285,24 @@ nir_def *air_get_ssa(struct air_ctx *c, const struct air_value *v);unsigned air_nir_bit_size(struct air_ctx *c, const struct air_type *t);unsigned air_nir_components(const struct air_type *t);+/* Records that the shader was translated with a caveat: what it asked for and+ * what was done instead (air_shader_info.notes). Not an error -- translation+ * continues -- and not a warning about our own code: it is for a thing the+ * hardware or this frontend cannot do exactly, where doing something else is+ * still correct. Refusing is air_error; passing over it in silence is the one+ * option this is here to remove.+ */+void air_note(struct air_ctx *c, const char *fmt, ...) PRINTFLIKE(2, 3);+/* memory, by the pointer's class */+/* The reserved texture slot a null texture lives in, allocated on the first+ * air.get_null_texture_* of a shader and shared by the rest of them. Negative+ * when it does not fit, with the reason already in c->error. Across the file+ * boundary because the interface half decides it and the INTRINSIC half asks+ * (air_intrinsics.c).+ */+int air_null_texture_slot(struct air_ctx *c);+nir_def *air_load_ptr(struct air_ctx *c, struct air_ptr ptr, const struct air_type *type,unsigned align);bool air_store_ptr(struct air_ctx *c, struct air_ptr ptr, nir_def *value, unsigned align);diff --git a/tools/mesa/air/air_run.c b/tools/mesa/air/air_run.cindex 3007534..46af530 100644--- a/tools/mesa/air/air_run.c+++ b/tools/mesa/air/air_run.c@@ -19,9 +19,12 @@* (osmesa_create_gpu_screen), and nothing else about it changes.** usage: air_run FILE[.metallib] [FUNCTION]+ * air_run --library FILE.metallib FUNCTION one air.visible function, wrapped+ * air_run --stitch "f(in0); g(n0)" FILE.metallib a graph of them, stitched*/#include <stdio.h>+#include <ctype.h>#include <errno.h>#include <stdlib.h>#include <string.h>@@ -55,7 +58,7 @@#include <mach-o/dyld.h>#include <signal.h>/* <sys/ucontext.h>, not <ucontext.h>: the latter refuses to be included- * without _XOPEN_SOURCE on the Darwin SDK, and userland/src/inject.c takes the+ * without _XOPEN_SOURCE on the iOS SDK, and userland/src/inject.c takes the* same route for the same reason.*/#include <sys/ucontext.h>@@ -104,7 +107,7 @@ texel_value(unsigned slot, unsigned x, unsigned y, unsigned z, float out[4])* print the fault before dying. These two functions are userland/src/inject.c's* say_the_fault()/watch_for_faults() (branch sileo2, 6683521), with the slide* taken from this program's own image rather than from the shared cache, since- * the pc of interest is in air_run or in libOSMesa and not in Darwin.+ * the pc of interest is in air_run or in libOSMesa and not in iOS.*/static voidsay_the_fault(int signal_number, siginfo_t *info, void *context_in)@@ -156,6 +159,34 @@ static void watch_for_faults(void) {}/* set by --library: the fill below is then floats, for the reason given there */static bool library_fill = false;+/* set by --stitch: a function-stitching graph, as air_stitch.c reads it */+static const char *stitch_text = NULL;++/* set by --zero: every buffer the shader reads is zeroed instead of patterned.+ *+ * For the 14 kernels whose loops leave on a count or a sentinel read out of a+ * buffer, the pattern is not an input at all -- the loop never ends and the GPU+ * runs forever (docs/air.md section 5, tools/air/kstruct.py). Zeros are a+ * degenerate input but a *defined* one: a count of zero means the loop runs no+ * times, so the kernel reaches its end and the hardware answers. What that+ * proves is that the translated shader runs on the Adreno without hanging; it+ * proves nothing about what the kernel computes, and the harness says so rather+ * than printing the zeros as a result.+ */+static bool zero_fill = false;++/* set by --ones: every byte is 0xff.+ *+ * The other half of the same idea. A loop that counts up to a value from a+ * buffer ends at once when that value is zero; a loop that walks until it meets+ * a sentinel -- MetalFaceMask's span tables leave on -1 -- never meets it in a+ * buffer of zeros, and ends at once in a buffer of 0xff. Neither fill is an+ * input the kernel was written for, and between them they let most of these+ * kernels reach their end instead of running forever. Which of the two a kernel+ * terminates under is itself a fact about its shape.+ */+static bool ones_fill = false;+/* The argument buffer of a wrapped library function holds floats, not the byte* pattern.*@@ -187,9 +218,98 @@ library_byte(unsigned i)static uint8_tpattern_byte(unsigned i){+ if (zero_fill)+ return 0;+ if (ones_fill)+ return 0xff;return library_fill ? library_byte(i) : (i * 7 + 13) & 0xff;}+/* --buffer N@OFF=HEX puts those bytes into buffer N at that offset, over+ * whatever fill the rest of it keeps.+ *+ * It exists because the kernels that never finish do not need a whole buffer+ * chosen for them: they need a span table in one slot and a row index in+ * another, and the value their exit test compares against is a field or two+ * (tools/air/kinput.py works out which, and prints these arguments). A file is+ * one fill for every buffer, so it cannot say "this slot and not that one".+ * Offsets are decimal so that a job line stays readable; init.c splits a line+ * on blanks and knows no variables, so each kernel has to be one line.+ */+struct buffer_patch {+ unsigned index;+ unsigned off;+ unsigned len;+ uint8_t *bytes;+};+static struct buffer_patch patches[64];+static unsigned num_patches;++/* Parses "N@OFF=HEX". Returns false when the argument is not that shape, so+ * that --buffer keeps meaning a file name and a typo is not read as one.+ */+static bool+add_buffer_patch(const char *spec)+{+ char *end;+ unsigned long index = strtoul(spec, &end, 10);+ if (end == spec || *end != '@')+ return false;+ const char *p = end + 1;+ unsigned long off = strtoul(p, &end, 10);+ if (end == p || *end != '=')+ return false;++ const char *hex = end + 1;+ size_t n = strlen(hex);+ if (!n || n % 2)+ return false;+ for (size_t i = 0; i < n; i++)+ if (!isxdigit((unsigned char)hex[i]))+ return false;++ if (num_patches == ARRAY_SIZE(patches)) {+ fprintf(stderr, "air_run: more than %zu --buffer patches\n", ARRAY_SIZE(patches));+ exit(2);+ }+ struct buffer_patch *q = &patches[num_patches++];+ q->index = index;+ q->off = off;+ q->len = n / 2;+ q->bytes = malloc(q->len);+ for (unsigned i = 0; i < q->len; i++) {+ char byte[3] = { hex[2 * i], hex[2 * i + 1], 0 };+ q->bytes[i] = strtoul(byte, NULL, 16);+ }+ return true;+}++/* Writes the patches for one buffer and returns how many bytes they wrote.+ *+ * A patch that does not fit is refused out loud rather than clipped: these+ * bytes are what makes a loop end, and a kernel that silently did not get its+ * sentinel hangs the GPU, which the next boot pays for.+ */+static unsigned+apply_buffer_patches(unsigned index, uint8_t *bits, unsigned bytes)+{+ unsigned written = 0;+ for (unsigned i = 0; i < num_patches; i++) {+ const struct buffer_patch *q = &patches[i];+ if (q->index != index)+ continue;+ if (q->off + q->len > bytes) {+ printf("air_run: --buffer %u@%u= is %u bytes past the end of this %u-byte "+ "buffer and was NOT written\n", q->index, q->off,+ q->off + q->len - bytes, bytes);+ continue;+ }+ memcpy(bits + q->off, q->bytes, q->len);+ written += q->len;+ }+ return written;+}+/* Fill a buffer the shader is about to read.** AIR_RUN_BUFFER=FILE puts that file's bytes there instead of the pattern,@@ -205,11 +325,16 @@ pattern_byte(unsigned i)* rest as the pattern.*/static unsigned-fill_buffer(uint8_t *bits, unsigned bytes)+fill_buffer(unsigned index, uint8_t *bits, unsigned bytes){for (unsigned k = 0; k < bytes; k++)bits[k] = pattern_byte(k);+ unsigned patched = apply_buffer_patches(index, bits, bytes);+ if (patched)+ printf("air_run: buffer at index %u, %u of its %u bytes from --buffer %u@...\n",+ index, patched, bytes, index);+const char *path = getenv("AIR_RUN_BUFFER");if (!path || !*path)return 0;@@ -221,6 +346,10 @@ fill_buffer(uint8_t *bits, unsigned bytes)}size_t got = fread(bits, 1, bytes, f);fclose(f);+ /* the file is the coarser statement -- one fill for every buffer -- so the+ * per-slot patches go on top of it and not under it+ */+ apply_buffer_patches(index, bits, bytes);return got;}@@ -274,8 +403,42 @@ type_size_vec4(const struct glsl_type *type, bool bindless)}static void-finalize_for_gallium(nir_shader *nir)+finalize_for_gallium(nir_shader *nir, struct pipe_screen *screen){+ /* Does this screen's own backend lower I/O from the variables? softpipe's+ * does -- its shader state is built by nir_to_tgsi, which calls nir_lower_io+ * with this very mask and this very flag -- and ir3's does not.+ *+ * Gated on the screen's NAME, as the scratch block at the end of this+ * function is and for the same reason: there is no pipe cap that says "this+ * driver lowers I/O itself". PIPE_SHADER_CAP_SUPPORTED_IRS cannot stand in+ * for one, because freedreno advertises TGSI too (through tgsi_to_nir, which+ * is the other direction), and get_compiler_options returns a+ * nir_shader_compiler_options with no field for it. So the name is what+ * there is, and naming it is honest where inferring it from a cap that means+ * something else would not be.+ */+ const bool backend_lowers_io =+ screen->get_name && !strcmp(screen->get_name(screen), "softpipe");++ /* A fragment shader's [[position]]: the frontend hands it over as a system-value+ * variable, because that is the one form both of these drivers can be served from+ * (the long version is beside air.position in air_to_nir.c). freedreno sets+ * pipe_caps.fs_position_is_sysval and wants the intrinsic; softpipe does not set it+ * and reads (0,0,0,0) from the intrinsic, because nothing fills TGSI's system values+ * for a fragment shader. For such a screen the variable becomes an input varying+ * here -- and then needs a driver_location, which nothing else would give it.+ */+ if (nir->info.stage == MESA_SHADER_FRAGMENT && !screen->caps.fs_position_is_sysval) {+ static const struct nir_lower_sysvals_to_varyings_options pos_opts = {+ .frag_coord = true,+ };+ if (nir_lower_sysvals_to_varyings(nir, &pos_opts)) {+ nir_foreach_shader_in_variable(var, nir)+ if (var->data.location == VARYING_SLOT_POS)+ var->data.driver_location = nir->num_inputs++;+ }+ }NIR_PASS(_, nir, nir_lower_system_values);/* An options struct of all-false and not NULL, which are not the same thing* here and the difference is a trap. nir_lower_system_values *introduces*@@ -293,8 +456,116 @@ finalize_for_gallium(nir_shader *nir)*/static const nir_lower_compute_system_values_options cs_options = { 0 };NIR_PASS(_, nir, nir_lower_compute_system_values, &cs_options);- NIR_PASS(_, nir, nir_lower_io, nir_var_shader_in | nir_var_shader_out,- type_size_vec4, 0);+ /*+ * nir_lower_io_use_interpolated_input_intrinsics, and WITHOUT it every varying+ * was FLAT on the Adreno while softpipe interpolated it. Mesa's own comment on the+ * flag says exactly why: without it nir_lower_io "generates regular load_input+ * intrinsics and interpolation information must be inferred from the list of input+ * nir_variables" -- and the two consumers infer it from different places.+ * nir_to_tgsi declares each FS input from its variable's data.interpolation (SMOOTH,+ * as the frontend sets it), so softpipe was right by accident; ir3's setup_input+ * does `so->inputs[n].flat = !coord` and only builds `coord` for+ * load_interpolated_input, so a plain load_input is flat, full stop.+ *+ * What that cost: the one draw in this project whose varying actually varies --+ * Apple's texturedVertex2D handing a texcoord to texturedFragment2D* -- read the+ * provoking vertex's texcoord at every fragment on the phone, and the picture looked+ * VERTICALLY FLIPPED for three boots (docs/metal.md 10.26, 10.32). It is not flipped.+ * The quad's pixel (0,0) sits in the triangle whose last vertex carries uv (0,1), so+ * flat shading hands it texel (0,3) -- `100 203 303 400`, bit for bit the number the+ * device printed, and reproduced here on softpipe by forcing INTERP_MODE_FLAT.+ *+ * Why nothing caught it: the 255 fragment shaders compared pixel by pixel are drawn+ * over the generated vertex stage, which gives every vertex the SAME texcoord -- flat+ * and smooth are identical over a constant. And ir3_compiler, the tool the corpus's+ * "compiles through ir3" column comes from, passes this flag already+ * (tools/mesa/air-ir3.patch): the tool was lowering correctly while the two harnesses+ * that actually draw were not.+ *+ * ... and it is skipped entirely for a backend that lowers I/O itself, which+ * is what made this harness unable to draw a fragment shader that reads a+ * varying. nir_to_tgsi lowers I/O a SECOND time, from the variables, with+ * the same mask, the same flag and a type_size that is type_size_vec4 word+ * for word -- and it opens with+ * nir_remove_dead_variables(nir_var_shader_in). Lowering here first leaves a+ * shader whose inputs are intrinsics and whose input variables are held up+ * by nothing but the orphaned deref_var instructions nir_lower_io does not+ * delete; the scratch block below then runs nir_opt_dce, those go, ntt+ * deletes the variables, and ntt_setup_inputs sizes input_index_map by the+ * driver_locations of the variables that are left -- none. The+ * load_interpolated_input this pass already emitted then swizzles a+ * TGSI_FILE_NULL register and tgsi_ureg asserts, before any format, texture+ * or sampler is reached.+ *+ * That abort was read as softpipe's limit in two places (pixfmtcheck's 30+ * SAMPLED cases, metalcheck's draw sweep) and it is this file's. Counted:+ * 134 of the corpus's 400 fragment entry points have an input variable on+ * this screen -- 125 read a [[stage_in]] varying and nine read only+ * [[position]], which the block above turns into one -- and 87 of them are+ * QuartzCore's, the library every pixel of the screen is drawn with. Two of+ * the 134 drew before this and both drew the WRONG pixel: the register+ * ntt swizzled was whatever the heap held under a zero-length+ * input_index_map, so the abort was the loud outcome and not the only one.+ * air_metal.c runs the same passes without the dce and draws those shaders+ * through the same softpipe, which is what said the limit was not softpipe's.+ *+ * Not lowering is preferred over keeping the derefs alive past the dce+ * because it leaves ONE lowering of I/O in the pipeline instead of one and a+ * half: ntt also runs nir_lower_indirect_derefs(nir_var_shader_in) before it+ * lowers, which is a no-op on a shader whose derefs are already gone and is+ * there because ureg splits input declarations to elements.+ */+ if (!backend_lowers_io)+ NIR_PASS(_, nir, nir_lower_io, nir_var_shader_in | nir_var_shader_out,+ type_size_vec4, nir_lower_io_use_interpolated_input_intrinsics);++ /* SCRATCH, for a screen that has none.+ *+ * An `alloca` becomes load_scratch and store_scratch, which ir3 compiles+ * and nir_to_tgsi has no case for at all: softpipe printed+ * "Unknown intrinsic: @store_scratch" and then asserted inside ureg_swizzle+ * on the undefined register it had made nothing for.+ *+ * WHAT IT ACTUALLY UNLOCKS, measured and not assumed: tools/air/callptr.ll,+ * which is the whole reason it is here -- a shader with an alloca in it and+ * nothing else nir_to_tgsi lacks, so LLVM on this machine and this draw can+ * be compared and they agree on all four lanes. It does NOT make Apple's own+ * RenderBox modules draw here: all fourteen of the ones that hand a shape+ * function a `ptr` still refuse on softpipe, now for a reason that has+ * nothing to do with scratch (`u2u32` from 16 bits asserts `src_64` in+ * ntt_emit_alu, and these shaders are full of halves). The scratch is off+ * the list of reasons; the list is not empty.+ *+ * nir_lower_scratch_to_var puts the whole scratch space back into one local+ * variable, which nir_lower_vars_to_ssa then turns into ordinary values.+ * That is sound exactly where this harness runs it: one invocation per+ * fragment, no addresses taken across a call (nir_inline_functions has+ * already folded every callee in), and nothing else reading the space.+ *+ * Gated on the screen by NAME and not run unconditionally, because the+ * point of this harness is to put the SAME NIR through two drivers: doing+ * it for ir3 as well would mean the Adreno column measured a shader with+ * the scratch lowered away, which is not the shader the phone runs. There+ * is no pipe cap that says "this driver has no scratch", so the name is+ * what there is, and it is named here rather than guessed at from a cap+ * that means something else.+ *+ * The nir_opt_dce at the end of this block is the pass that used to orphan+ * every [[stage_in]] variable, and it is kept: with I/O left unlowered above+ * for exactly the screens this block runs on, a load_deref still references+ * each one, so there is nothing dead to remove. That is a coupling between+ * two gates that test the same screen for different reasons, and the thing+ * that would notice it coming apart is a test rather than a comment --+ * tools/air/pixfmtcheck.py's last case draws one of Apple's varying-reading+ * shaders and fails if the abort comes back.+ */+ if (screen->get_name && !strcmp(screen->get_name(screen), "softpipe")) {+ NIR_PASS(_, nir, nir_lower_scratch_to_var);+ NIR_PASS(_, nir, nir_split_var_copies);+ NIR_PASS(_, nir, nir_lower_var_copies);+ NIR_PASS(_, nir, nir_lower_vars_to_ssa);+ NIR_PASS(_, nir, nir_opt_dce);+ }}/* -- compute ---------------------------------------------------------------@@ -325,6 +596,17 @@ finalize_for_gallium(nir_shader *nir)*/#define GRID 4+/* The threadgroup this harness dispatches. 4x4x1 by default, because a grid with more than+ * one thread in two dimensions is what makes a wrong index show, and --block X,Y,Z because+ * some kernels are written for a particular one and say so in their own code: both fbs_pcg_*+ * kernels end with `if (tid == 0)` summing 128 entries of a threadgroup array, which is a+ * 128x1x1 reduction. Under 4x4x1 four threads have tid == 0 -- Metal's scalar+ * thread_position_in_threadgroup is the x component, not the flattened index -- so four of+ * them run the whole reduction over the four entries this shape writes, and the word they+ * add up is a property of the dispatch rather than of the kernel (docs/air.md section 5).+ */+static unsigned block[3] = { GRID, GRID, 1 };+#define MAX_IMG 16/* The shape of each image a kernel touches, read off the intrinsics the@@ -362,13 +644,40 @@ image_shapes(nir_shader *cs, enum pipe_format fmt[MAX_IMG], bool isf[MAX_IMG],unsigned b = nir_src_as_uint(in->src[0]);if (b >= MAX_IMG || seen[b])continue;+ /* The WIDTH as well as the base type.+ *+ * A 32-bit format for a 16-bit store is not a conversion, it is a+ * descriptor that disagrees with the instruction: ir3 emits+ * `stib.b.typed.2d.u16.4` from air.write_texture_2d.v4f16 and the+ * image it is given says 32 bits per component. MetalFaceMask_DrawSpans+ * is that case -- it declares texture2d<half, write> -- and it is the+ * kernel whose dispatch took a whole boot with it in+ * logs/070755-run.log, printing its bindings and then nothing at all.+ * 22 kernels of the corpus emit a narrow typed image store, so this is+ * not one shader's accident. GL has no 16-bit image store, which is why+ * neither Mesa nor this harness had met it; the draw path already picks+ * the narrow format when a fragment shader returns halves.+ */+ unsigned bits = nir_alu_type_get_type_size(t);+switch (nir_alu_type_get_base_type(t)) {case nir_type_int:- fmt[b] = PIPE_FORMAT_R32G32B32A32_SINT; isf[b] = false; break;+ fmt[b] = bits == 16 ? PIPE_FORMAT_R16G16B16A16_SINT+ : bits == 8 ? PIPE_FORMAT_R8G8B8A8_SINT+ : PIPE_FORMAT_R32G32B32A32_SINT;+ isf[b] = false;+ break;case nir_type_uint:- fmt[b] = PIPE_FORMAT_R32G32B32A32_UINT; isf[b] = false; break;+ fmt[b] = bits == 16 ? PIPE_FORMAT_R16G16B16A16_UINT+ : bits == 8 ? PIPE_FORMAT_R8G8B8A8_UINT+ : PIPE_FORMAT_R32G32B32A32_UINT;+ isf[b] = false;+ break;default:- fmt[b] = PIPE_FORMAT_R32G32B32A32_FLOAT; isf[b] = true; break;+ fmt[b] = bits == 16 ? PIPE_FORMAT_R16G16B16A16_FLOAT+ : PIPE_FORMAT_R32G32B32A32_FLOAT;+ isf[b] = true;+ break;}dim[b] = nir_intrinsic_image_dim(in);seen[b] = true;@@ -414,6 +723,452 @@ texture_shapes(nir_shader *cs, enum pipe_format fmt[MAX_IMG], bool isf[MAX_IMG],}}+/* Wait for the work to finish, and say so when it does not.+ *+ * This matters more than it looks. On 20 October a dispatch hung the GPU --+ * our own driver said "gpu: no answer: submission 2 is the last, the core has+ * done 1" -- and every dispatch for the rest of that boot was accepted,+ * returned, and did nothing. air_run did not wait on a fence, so it printed+ * the buffer it had filled itself as though the GPU had answered: a page of+ * numbers that were a statement about the harness and not about the shader.+ * 21 of the 47 kernels of one run were vacuous that way before anyone noticed+ * (docs/air.md section 5).+ *+ * The hang itself turned out to be this harness too: the kernels that did it+ * loop until a buffer value they were never given, and the fill is a byte+ * pattern. So the message below leads with that rather than with the GPU.+ *+ * So the fence is waited on with a timeout, and a timeout is reported as+ * loudly as a crash. A dispatch nobody waited for is not a measurement.+ */+static bool+finish_or_say_why(struct pipe_screen *screen, struct pipe_context *pipe, const char *what)+{+ struct pipe_fence_handle *fence = NULL;+ pipe->flush(pipe, &fence, 0);+ if (!fence) {+ printf("air_run: the driver gave no fence for the %s, so nothing here waited for "+ "the GPU and the numbers below may be the fill\n", what);+ return true;+ }+ bool done = screen->fence_finish(screen, NULL, fence, 2000000000ull); /* 2 s */+ screen->fence_reference(screen, &fence, NULL);+ if (!done)+ printf("air_run: THE GPU DID NOT FINISH the %s within two seconds. Everything below is "+ "what this harness wrote, not what the shader computed. The likeliest reason is "+ "this harness: a kernel whose buffers carry counts, indices or sentinels is being "+ "given (i * 7 + 13) & 0xff, and a loop that leaves on a value it was never given "+ "does not leave at all -- both kernels measured this way turned out to be exactly "+ "that (docs/air.md section 5). And one timeout wedges this GPU for the rest of the "+ "boot, so every later run in it is vacuous too.\n", what);+ return done;+}++/* What a shader's own sampler says, in one line. Printed rather than assumed:+ * the ten modules of the corpus that carry one have no reference to be checked+ * against, so the only thing a reader can verify is that the harness bound+ * what the word says.+ */+static void+print_own_sampler(unsigned slot, const struct air_sampler_state *a, bool compares)+{+ static const char *const addr[] = { "clamp_to_zero", "clamp_to_edge", "repeat",+ "mirrored_repeat", "clamp_to_border" };+ static const char *const filt[] = { "nearest", "linear", "bicubic" };+ static const char *const mip[] = { "none", "nearest", "linear" };+ /* Metal's compare_func, in its own order: `none` at 0 and `never` at 8, which is+ * the value Apple's default carries (docs/air.md 3.3).+ */+ static const char *const cmp[] = { "none", "less", "less_equal", "greater",+ "greater_equal", "equal", "not_equal", "always",+ "never" };+ printf("air_run: sampler %u is the shader's own: %s/%s filtering, mip %s, %s "+ "addressing, %s coordinates, compare_func::%s%s\n", slot,+ filt[a->mag_filter % ARRAY_SIZE(filt)], filt[a->min_filter % ARRAY_SIZE(filt)],+ mip[a->mip_filter % ARRAY_SIZE(mip)],+ addr[a->address[0] % ARRAY_SIZE(addr)],+ a->pixel_coord ? "pixel" : "normalized",+ a->compare_func < ARRAY_SIZE(cmp) ? cmp[a->compare_func] : "?",+ compares ? " APPLIED (a shadow sample reads this slot)"+ : " not read (nothing samples this slot with a comparison)");+}++/*+ * THE PACKED 16-BIT PROBE: one known word, both directions, on whatever screen this is.+ *+ * air_metal.c has to decide which pipe_format each of Metal's four packed 16-bit names+ * means, and that decision has two halves. Which Metal name goes with which Gallium name+ * is Apple's convention and only Apple's own compositor can settle it. Where the fields+ * of a given GALLIUM name actually land is a fact about the driver and the hardware, and+ * that is what this measures -- on the Adreno, which is the only place it matters and+ * the one place `air_metal_test --pixel-formats` cannot run, because that program is not+ * on the device and air_run is.+ *+ * TWO FLAGS AND NOT ONE, because a round trip measures nothing. Sampling a packed+ * texture and rendering into a packed target of the same format is unpack then pack with+ * the same layout, which is the identity whatever that layout is: it would catch a read+ * path disagreeing with a write path and could never catch both being wrong together.+ * So the two directions are separate runs with a known value at the end that is NOT+ * packed:+ *+ * --packed16 FORMAT 0xWORD every texel of the bound texture is that 16-bit word,+ * with one field set; the target is the ordinary float one,+ * so the four numbers printed say which CHANNEL the field+ * became. The SAMPLED direction.+ * --packed16-rt FORMAT the colour target is that packed format and the shader+ * supplies the colour, so the word printed says which FIELD+ * each channel became. The WRITTEN direction, and it wants+ * a module whose output is a constant -- four floats could+ * not show it, because they would have come back out+ * through the same unpacking the write used.+ *+ * FORMAT is one of the eight Gallium names the two readings of Metal's four can mean. A+ * name and not a number, so the job line says which layout it is asking about and a+ * reader needs nothing else.+ */+static const struct { const char *name; enum pipe_format fmt; } packed16_names[] = {+ { "R5G6B5_UNORM", PIPE_FORMAT_R5G6B5_UNORM },+ { "B5G6R5_UNORM", PIPE_FORMAT_B5G6R5_UNORM },+ { "A1R5G5B5_UNORM", PIPE_FORMAT_A1R5G5B5_UNORM },+ { "A1B5G5R5_UNORM", PIPE_FORMAT_A1B5G5R5_UNORM },+ { "R5G5B5A1_UNORM", PIPE_FORMAT_R5G5B5A1_UNORM },+ { "B5G5R5A1_UNORM", PIPE_FORMAT_B5G5R5A1_UNORM },+ { "A4R4G4B4_UNORM", PIPE_FORMAT_A4R4G4B4_UNORM },+ { "A4B4G4R4_UNORM", PIPE_FORMAT_A4B4G4R4_UNORM },+ /* the control, and the census is why it is here: the window server makes a+ * BGRA8 texture of exactly the shape it makes the packed one+ */+ { "B8G8R8A8_UNORM", PIPE_FORMAT_B8G8R8A8_UNORM },+};+static enum pipe_format packed16_fmt = PIPE_FORMAT_NONE; /* the sampled texture */+static enum pipe_format packed16_rt = PIPE_FORMAT_NONE; /* ... or the colour target */+static enum pipe_format miplevel_fmt = PIPE_FORMAT_NONE; /* ... or a mip pyramid */+/*+ * --clear: what a framebuffer fetch will read, PER RENDER TARGET.+ *+ * Softpipe's TGSI interpreter has no FBFETCH at all, so the only driver here that+ * can answer a fetch is the one on the phone -- and there, a fetch in a single+ * draw reads what the clear put down. Chosen numbers turn that into a+ * measurement: a shader that hands the fetch back must return the clear, and+ * zero is then distinguishable from a fetch that read nothing.+ *+ * Per target and not one colour for all of them, because the question that+ * matters is WHICH ATTACHMENT was read. The window server's own passes carry two+ * colour attachments and mask one of them off entirely; a fetch that read the+ * other one would be invisible if both held the same numbers.+ */+#define MAX_CLEARS 8+static bool halftex;+static union pipe_color_union clear_colour[MAX_CLEARS];+static unsigned clear_count;+static bool clear_given;+static const char *packed16_name;+static unsigned packed16_word;++/*+ * THE MIP PYRAMID, at the shape the window server actually asks for.+ *+ * The texture census of a whole boot (userland/src/redwoodmetal.m's texture line)+ * found exactly ONE packed texture in the entire system: MTLPixelFormatBGR5A1Unorm,+ * 2D, shaderRead|renderTarget, TEN MIP LEVELS, 776x24. Nothing else packed is ever+ * created -- not formats 40, 41 or 42, and SpringBoard asks for none at all. So a+ * probe on a single-level 4x4 texture is a probe of something the system does not+ * have, and a mip chain is where a two-byte format differs most from the+ * four-byte one everything else uses.+ *+ * AND THE SAME CENSUS CARRIES ITS OWN CONTROL: a format 80 (BGRA8Unorm) texture of+ * the IDENTICAL shape -- type 2, usage 0x5, mips 10, 776x24. One logical object in+ * two formats. Run through this probe, BGRA8 is the line that says whether a+ * 776x24 pyramid works at all on this GPU, so a failure confined to the packed+ * formats cannot be blamed on the shape, the usage, the level count or the+ * harness.+ *+ * THE LEVEL IS IN THE VALUE, which is this harness's rule everywhere else: level L+ * is filled with a texel whose four channels are the binary code of L + 1, each+ * channel either all-ones or all-zeros. Three things follow. A sample that read+ * the wrong level names the level it actually read, rather than merely+ * disagreeing. Every expectation is exactly 0.0 or exactly 1.0 whatever the field+ * widths are -- a 5-bit field at 9/31 would not be, and an expectation needing a+ * tolerance cannot tell a near miss from a hit. And L + 1 RATHER THAN L, so that+ * no level's code is zero: with L the first level would read 0 0 0 0, which is+ * also what an unwritten level and an untouched allocation read, and the one+ * outcome this is built to catch would have been indistinguishable from the+ * healthy one.+ */+/*+ * --halftex: a 256x1 RGBA16Float lookup table, written as RAW HALF BIT PATTERNS.+ *+ * CoreAnimation's gradient shader -- axial_shading_frag_lpf, 47 NIR instructions+ * and one air.sample_texture_2d -- does nothing but sample a table of this exact+ * shape and write the result, and the census found it: format 115, 2D, 256x1,+ * shaderRead only, so the 256 entries are built on the CPU and uploaded. That+ * upload path's half conversion has never been measured here.+ *+ * THE WORDS ARE CHOSEN AS INTEGERS AND WRITTEN AS INTEGERS, with no float-to-half+ * conversion anywhere in this program, because a test that samples a table it+ * also converted would pass with both halves wrong. The expectation is decoded+ * from the same integers by a different implementation of binary16 on the host,+ * so the two sides share nothing but the bit patterns.+ *+ * Texel i carries four patterns chosen to span what binary16 can be and nothing+ * it cannot:+ *+ * r (i << 7) & 0x7bff a NORMAL of rising exponent; masked so the exponent+ * never reaches 31, which would be an infinity or a NaN+ * g r | 0x8000 the same magnitude NEGATIVE, so a sign dropped shows+ * b i a DENORMAL for every i, and exactly zero at i = 0+ * a 0x3c00 exactly 1.0, so one channel is a constant a wrong+ * channel order cannot hide in+ */+#define HALFTEX_W 256++static void+halftex_texel(unsigned i, uint16_t out[4])+{+ out[0] = (uint16_t)((i << 7) & 0x7bff);+ out[1] = (uint16_t)(out[0] | 0x8000);+ out[2] = (uint16_t)i;+ out[3] = 0x3c00;+}++#define MIPTEX_W 776+#define MIPTEX_H 24+#define MIPTEX_LEVELS 10++static void+miplevel_texel(unsigned level, float out[4])+{+ for (unsigned c = 0; c < 4; c++)+ out[c] = ((level + 1) >> c) & 1 ? 1.0f : 0.0f;+}++static enum pipe_format+packed16_by_name(const char *want, const char **name)+{+ for (unsigned i = 0; i < ARRAY_SIZE(packed16_names); i++)+ if (!strcmp(want, packed16_names[i].name)) {+ *name = packed16_names[i].name;+ return packed16_names[i].fmt;+ }+ fprintf(stderr, "air_run: --packed16 wants one of these names:");+ for (unsigned i = 0; i < ARRAY_SIZE(packed16_names); i++)+ fprintf(stderr, " %s", packed16_names[i].name);+ fprintf(stderr, "\n");+ return PIPE_FORMAT_NONE;+}++/* The depth texel a shadow sample reads, which is the only pattern that fits.+ *+ * A depth format is normalised to [0, 1] and this harness's colour pattern starts at+ * 100, so the two cannot be the same rule. (x + TEX_SIZE*y) / 16 gives sixteen+ * distinct values over a 4x4 texture, every one exactly representable in binary32 and+ * every one a different answer -- so a comparison that read the wrong texel answers+ * differently, and a reference landing between two of them is exact on both sides.+ * tools/air/sampcheck.py computes its expectations from this same expression.+ */+static float+depth_texel_value(unsigned x, unsigned y)+{+ return (float)(x + TEX_SIZE * y) / 16.0f;+}++/* Which sampler slots a shadow texture instruction reads, out of the translated+ * shader itself.+ *+ * Metal decides AT THE SAMPLE CALL that a comparison happens: sample() and+ * sample_compare() are two instructions over one sampler, and the sampler word+ * only says WHICH comparison. Gallium decides it in the sampler state, through+ * pipe_sampler_state.compare_mode, and softpipe compares on every sample of a+ * sampler that has it set (sp_tex_sample.c: `if (compare_mode != NONE)+ * sample_compare(...)`). So switching it on for a slot a plain sample() reads+ * would turn every texel into a 0 or a 1, and leaving it off for a slot a+ * sample_compare() reads would return the texel instead of the comparison.+ * Exactly these slots, and the shader is the only thing that knows which.+ *+ * On the Adreno the same two lines mean something slightly different and the+ * answer is the same: a6xx has no compare-enable bit at all, only+ * A6XX_TEX_SAMP_1_COMPARE_FUNC, which fd6_sampler_state_create writes only when+ * compare_mode is set -- and the hardware reads it only for the shadow form of+ * the sample instruction, which ir3 selects from nir_tex_instr::is_shadow+ * (IR3_INSTR_S). So the field has to be programmed for these slots, and+ * programming it for the others would be harmless there and wrong here.+ */+static void+compare_sampled_slots(nir_shader *s, bool *slot, unsigned count)+{+ nir_foreach_function_impl(impl, s) {+ nir_foreach_block(block, impl) {+ nir_foreach_instr(instr, block) {+ if (instr->type != nir_instr_type_tex)+ continue;+ nir_tex_instr *t = nir_instr_as_tex(instr);+ if (t->is_shadow && t->sampler_index < count)+ slot[t->sampler_index] = true;+ }+ }+ }+}++/* A comparison through a sampler whose own word says `compare_func::none`,+ * refused by name and in the same words air_metal.c's bind_stage refuses it.+ *+ * MTLCompareFunction has eight values and no "not configured", so `none` --+ * value 0 of the shader word's nine -- has no API spelling and arrives at a+ * driver as Never, which is MTLSamplerDescriptor's own default and is right for+ * the plain sample that is all either can do. A COMPARISON is where the two+ * part company: AGX keeps its NONE and its NEVER apart (AGXCompilerCore's+ * _AGCCompareFunction), and what it does differently is not readable from here.+ * Answering such a sample as `never` would be a harness quietly agreeing with+ * itself where the library this is a reference for says no.+ */+static bool+compare_without_a_function(const struct air_shader_info *info,+ const bool *compare_slot, unsigned count,+ const char *name)+{+ for (unsigned i = 0; i < info->num_resources; i++) {+ const struct air_resource *r = &info->resources[i];+ if (r->kind != AIR_RESOURCE_SAMPLER || !r->inline_sampler)+ continue;+ if (r->index >= count || !compare_slot[r->index])+ continue;+ if (r->sampler.compare_func != 0)+ continue;+ fprintf(stderr, "air_run: %s samples sampler %u with a comparison and its own "+ "word says compare_func::none, which is not one of "+ "MTLCompareFunction's eight: a sampler that was not made to "+ "compare\n", name, r->index);+ return true;+ }+ return false;+}++/* The sampler state a shader declared for itself, as Gallium wants it.+ *+ * Metal's clamp_to_zero is a wrap mode and not a border colour: outside the+ * texture it reads transparent black, in every channel including alpha, and+ * the border colour is not configurable for it. Gallium has no such mode, so+ * it becomes CLAMP_TO_BORDER with a zero border -- which is the same thing and+ * is worth writing down, because the obvious reading of that enum value is+ * clamp_to_border with a default border, and one of Apple's own binaries+ * prints it that way (nir_air.h).+ *+ * Fields with no Gallium equivalent, or none this harness can honour, are left+ * at the default and named by the caller rather than silently dropped:+ * bicubic filtering and the reduction mode AGX itself does not read.+ *+ * The comparison function is honoured, and only for the slots a shadow+ * instruction reads -- `compares`, from compare_sampled_slots above. Its own+ * enumeration is not MTLCompareFunction's and not Gallium's: in the word 0 is+ * `none` and `never` is 8, which is the value Apple's own default carries, so+ * every `constexpr sampler` that names no compare_func arrives here with 8 and+ * must mean no comparison rather than an unknown one.+ */+static struct pipe_sampler_state+sampler_from_air(const struct air_sampler_state *a, bool compares)+{+ static const unsigned wrap[] = {+ [AIR_SAMPLER_CLAMP_TO_ZERO] = PIPE_TEX_WRAP_CLAMP_TO_BORDER,+ [AIR_SAMPLER_CLAMP_TO_EDGE] = PIPE_TEX_WRAP_CLAMP_TO_EDGE,+ [AIR_SAMPLER_REPEAT] = PIPE_TEX_WRAP_REPEAT,+ [AIR_SAMPLER_MIRRORED_REPEAT] = PIPE_TEX_WRAP_MIRROR_REPEAT,+ [AIR_SAMPLER_CLAMP_TO_BORDER] = PIPE_TEX_WRAP_CLAMP_TO_BORDER,+ };+ static const unsigned filt[] = {+ [AIR_SAMPLER_NEAREST] = PIPE_TEX_FILTER_NEAREST,+ [AIR_SAMPLER_LINEAR] = PIPE_TEX_FILTER_LINEAR,+ [AIR_SAMPLER_BICUBIC] = PIPE_TEX_FILTER_LINEAR,+ };+ static const unsigned mip[] = {+ [AIR_SAMPLER_MIP_NONE] = PIPE_TEX_MIPFILTER_NONE,+ [AIR_SAMPLER_MIP_NEAREST] = PIPE_TEX_MIPFILTER_NEAREST,+ [AIR_SAMPLER_MIP_LINEAR] = PIPE_TEX_MIPFILTER_LINEAR,+ };+ struct pipe_sampler_state st = {+ .wrap_s = wrap[a->address[0] % ARRAY_SIZE(wrap)],+ .wrap_t = wrap[a->address[1] % ARRAY_SIZE(wrap)],+ .wrap_r = wrap[a->address[2] % ARRAY_SIZE(wrap)],+ .mag_img_filter = filt[a->mag_filter % ARRAY_SIZE(filt)],+ .min_img_filter = filt[a->min_filter % ARRAY_SIZE(filt)],+ .min_mip_filter = mip[a->mip_filter % ARRAY_SIZE(mip)],+ .unnormalized_coords = a->pixel_coord,+ .max_anisotropy = a->max_anisotropy > 1 ? a->max_anisotropy : 0,+ .min_lod = _mesa_half_to_float(a->lod_min),+ .max_lod = _mesa_half_to_float(a->lod_max),+ };+ /* The comparison, where the shader asked for one. The table is the one in+ * air_metal.c's sampler_state_from_air -- MTLCompareFunction and+ * PIPE_FUNC_x run in the same order, so one table serves both -- and it is+ * written out here rather than cast because 2 `less_equal` would arrive as+ * PIPE_FUNC_EQUAL and 3 `greater` as PIPE_FUNC_LEQUAL, which is the+ * opposite comparison and still a picture.+ */+ if (compares) {+ static const unsigned func[] = {+ [0] = PIPE_FUNC_NEVER, /* none: no MTLCompareFunction has a name for it */+ [1] = PIPE_FUNC_LESS, /* less */+ [2] = PIPE_FUNC_LEQUAL, /* less_equal */+ [3] = PIPE_FUNC_GREATER, /* greater */+ [4] = PIPE_FUNC_GEQUAL, /* greater_equal */+ [5] = PIPE_FUNC_EQUAL, /* equal */+ [6] = PIPE_FUNC_NOTEQUAL, /* not_equal */+ [7] = PIPE_FUNC_ALWAYS, /* always */+ [8] = PIPE_FUNC_NEVER, /* never */+ };+ st.compare_mode = PIPE_TEX_COMPARE_R_TO_TEXTURE;+ st.compare_func = func[a->compare_func % ARRAY_SIZE(func)];+ }++ /* Metal's border_color: 0 transparent black, 1 opaque black, 2 opaque white.+ * clamp_to_zero does not read it -- it is transparent black by definition.+ */+ if (a->address[0] == AIR_SAMPLER_CLAMP_TO_BORDER ||+ a->address[1] == AIR_SAMPLER_CLAMP_TO_BORDER ||+ a->address[2] == AIR_SAMPLER_CLAMP_TO_BORDER) {+ float v = a->border_color == 2 ? 1.0f : 0.0f;+ st.border_color.f[0] = st.border_color.f[1] = st.border_color.f[2] = v;+ st.border_color.f[3] = a->border_color == 0 ? 0.0f : 1.0f;+ }+ return st;+}++/* The state for sampler slot `idx`: the shader's own if it declared one there,+ * and this harness's nearest/clamp-to-edge otherwise -- which is what the API+ * would have bound and what every measurement before inline samplers were+ * decoded was made with.+ */+static struct pipe_sampler_state+sampler_for_slot(const struct air_shader_info *info, unsigned idx, bool unnormalized,+ bool compares, const struct air_sampler_state **from_shader)+{+ for (unsigned i = 0; i < info->num_resources; i++) {+ const struct air_resource *r = &info->resources[i];+ if (r->kind == AIR_RESOURCE_SAMPLER && r->inline_sampler && r->index == idx) {+ if (from_shader)+ *from_shader = &r->sampler;+ return sampler_from_air(&r->sampler, compares);+ }+ }+ if (from_shader)+ *from_shader = NULL;+ /* The API's sampler, and a comparison only if the shader asked for one.+ * MTLSamplerDescriptor's own compareFunction defaults to Never, so this is+ * the API's default sampler said completely rather than left out.+ */+ return (struct pipe_sampler_state){+ .wrap_s = PIPE_TEX_WRAP_CLAMP_TO_EDGE,+ .wrap_t = PIPE_TEX_WRAP_CLAMP_TO_EDGE,+ .wrap_r = PIPE_TEX_WRAP_CLAMP_TO_EDGE,+ .min_img_filter = PIPE_TEX_FILTER_NEAREST,+ .mag_img_filter = PIPE_TEX_FILTER_NEAREST,+ .min_mip_filter = PIPE_TEX_MIPFILTER_NONE,+ .unnormalized_coords = unnormalized,+ .compare_mode = compares ? PIPE_TEX_COMPARE_R_TO_TEXTURE+ : PIPE_TEX_COMPARE_NONE,+ .compare_func = PIPE_FUNC_NEVER,+ };+}+static intdispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,nir_shader *cs, const char *name, const char *how)@@ -428,12 +1183,12 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,* filled with a distinctive pattern -- had it been zeroed, a kernel named* InitMinMax writing zeros would have been indistinguishable from this.*/- cs->info.workgroup_size[0] = GRID;- cs->info.workgroup_size[1] = GRID;- cs->info.workgroup_size[2] = 1;+ cs->info.workgroup_size[0] = block[0];+ cs->info.workgroup_size[1] = block[1];+ cs->info.workgroup_size[2] = block[2];cs->info.workgroup_size_variable = false;- finalize_for_gallium(cs);+ finalize_for_gallium(cs, screen);struct pipe_context *pipe = screen->context_create(screen, NULL, 0);if (!pipe) {fprintf(stderr, "air_run: no context for a kernel\n");@@ -444,6 +1199,14 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,/* every buffer the kernel declares; the device ones are what it writes */struct pipe_resource *bufs[32] = { 0 };+ /* The bytes this harness wrote, kept until the readback. The dump used to+ * recompute them from pattern_byte(), which --buffer N@OFF=HEX makes a lie:+ * a patched word looked changed by the kernel. Keeping the fill is also the+ * only way to say which words a kernel touched when it touches them past+ * the sixteen the dump prints -- and that is exactly what a prediction out+ * of tools/air/kinput.py has to be compared against.+ */+ uint8_t *before[32] = { 0 };unsigned sizes[32] = { 0 };bool writable[32] = { false };bool img_read[MAX_IMG] = { false }, img_write[MAX_IMG] = { false };@@ -457,12 +1220,28 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,if (r->index >= ARRAY_SIZE(bufs))continue;uint8_t *bits = calloc(1, bytes);- unsigned from_file = fill_buffer(bits, bytes);+ bool kept = false;+ unsigned from_file = fill_buffer(r->index, bits, bytes);if (from_file)printf("air_run: buffer at index %u, %u bytes, %u of them from ""AIR_RUN_BUFFER\n", r->index, bytes, from_file);if (r->kind == AIR_RESOURCE_CONSTANT_BUFFER) {+ /* A transform matrix filled with the byte pattern sends every sample+ * somewhere arbitrary, so what the shader computes says nothing about+ * what it is for. The identity is the one distinguished value a matrix+ * has, and the resource says it is one: air.arg_type_name is Apple's+ * own ("float4x4"), carried through from the module's metadata. Any+ * other constant buffer keeps the pattern -- this is the only place+ * the harness pretends to know what a buffer means, and it says so.+ */+ if (r->type_name && !strncmp(r->type_name, "float4x4", 8) && bytes >= 64) {+ float *m = (float *)bits;+ memset(m, 0, 64);+ m[0] = m[5] = m[10] = m[15] = 1.0f;+ printf("air_run: constant buffer %u is a %s, so it holds the identity and "+ "not the fill\n", r->index, r->type_name);+ }struct pipe_resource *b = pipe_buffer_create(screen, PIPE_BIND_CONSTANT_BUFFER,PIPE_USAGE_DEFAULT, bytes);pipe_buffer_write(pipe, b, 0, bytes, bits);@@ -474,6 +1253,8 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,PIPE_USAGE_STAGING, bytes);pipe_buffer_write(pipe, bufs[r->index], 0, bytes, bits);sizes[r->index] = bytes;+ before[r->index] = bits;+ kept = true;writable[r->index] = r->write;num_ssbo = MAX2(num_ssbo, r->index + 1);num_written += r->write ? 1 : 0;@@ -497,7 +1278,8 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,tex_used[r->index] = true;num_tex = MAX2(num_tex, r->index + 1);}- free(bits);+ if (!kept)+ free(bits);}unsigned num_img_written = 0;for (unsigned i = 0; i < num_img; i++)@@ -643,14 +1425,6 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,views[i] = pipe->create_sampler_view(pipe, texes[i], &vtempl);}if (num_tex) {- csampler = pipe->create_sampler_state(pipe, &(struct pipe_sampler_state){- .wrap_s = PIPE_TEX_WRAP_CLAMP_TO_EDGE,- .wrap_t = PIPE_TEX_WRAP_CLAMP_TO_EDGE,- .wrap_r = PIPE_TEX_WRAP_CLAMP_TO_EDGE,- .min_img_filter = PIPE_TEX_FILTER_NEAREST,- .mag_img_filter = PIPE_TEX_FILTER_NEAREST,- .min_mip_filter = PIPE_TEX_MIPFILTER_NONE,- .unnormalized_coords = 1 });/* one entry per texture, not the address of one pointer. bind_sampler_states* reads num_tex pointers from what it is given, so passing &csampler made* the driver take whatever followed that single variable on the stack and@@ -658,10 +1432,24 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,* ->seqno and died at a wild address. It only showed with two or more* textures, which is why 22 kernels bound images here without trouble and* four did not. The draw path below always did this correctly.+ *+ * One state per slot, and not one shared: a slot's sampler is whatever+ * that slot declares, and a shader may mix its own with the API's.*/void *csamplers[MAX_IMG] = { 0 };- for (unsigned i = 0; i < num_tex && i < MAX_IMG; i++)- csamplers[i] = csampler;+ bool compare_slot[MAX_IMG] = { 0 };+ compare_sampled_slots(cs, compare_slot, MAX_IMG);+ if (compare_without_a_function(info, compare_slot, MAX_IMG, name))+ return 1;+ for (unsigned i = 0; i < num_tex && i < MAX_IMG; i++) {+ const struct air_sampler_state *own = NULL;+ struct pipe_sampler_state st = sampler_for_slot(info, i, true,+ compare_slot[i], &own);+ csamplers[i] = pipe->create_sampler_state(pipe, &st);+ csampler = csamplers[i];+ if (own)+ print_own_sampler(i, own, compare_slot[i]);+ }pipe->bind_sampler_states(pipe, PIPE_SHADER_COMPUTE, 0, num_tex, csamplers);pipe->set_sampler_views(pipe, PIPE_SHADER_COMPUTE, 0, num_tex, 0, false, views);}@@ -678,19 +1466,23 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,/* more than one thread, and the whole grid, so that a wrong index shows */struct pipe_grid_info grid = {.work_dim = 3,- .block = { GRID, GRID, 1 },+ .block = { block[0], block[1], block[2] },.grid = { 1, 1, 1 },};- printf("air_run: dispatching %ux%ux1 threads in one workgroup; %u device buffer%s bound, "- "%u of them written, filled with (i * 7 + 13) & 0xff\n",- GRID, GRID, num_ssbo, num_ssbo == 1 ? "" : "s", num_written);+ printf("air_run: dispatching %ux%ux%u threads in one workgroup; %u device buffer%s bound, "+ "%u of them written, filled with %s\n",+ block[0], block[1], block[2], num_ssbo, num_ssbo == 1 ? "" : "s", num_written,+ zero_fill ? "ZEROS (--zero: a degenerate input, see docs/air.md section 5)"+ : ones_fill ? "0xff BYTES (--ones: a degenerate input)"+ : library_fill ? "the floats ((w * 13) % 29 - 14) / 4"+ : "(i * 7 + 13) & 0xff");if (num_img || num_tex)printf("air_run: %u image%s bound (%u written) and %u texture%s, %ux%u, texel (x,y,z) of ""slot n = (100+x+1000n, 200+y, 300+x+y+z, 400)\n", num_img,num_img == 1 ? "" : "s", num_img_written, num_tex, num_tex == 1 ? "" : "s",TEX_SIZE, TEX_SIZE);pipe->launch_grid(pipe, &grid);- pipe->flush(pipe, NULL, 0);+ bool finished = finish_or_say_why(screen, pipe, "dispatch");for (unsigned i = 0; i < num_ssbo; i++) {if (!writable[i])@@ -701,13 +1493,61 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,fprintf(stderr, "air_run: device buffer %u could not be read back\n", i);return 1;}- unsigned words = MIN2(sizes[i] / 4, (unsigned)(GRID * GRID));+ unsigned words = MIN2(sizes[i] / 4, block[0] * block[1] * block[2]);printf("air_run: device buffer %u after the dispatch, first %u words:\n", i, words);for (unsigned w = 0; w < words; w++)printf(" [%2u] %11u 0x%08x\n", w, p[w], p[w]);++ /* A buffer the shader writes that comes back bit-identical to the fill+ * is not a result, and the only thing worse than no number is a number+ * that looks like one.+ *+ * This is not hypothetical and not rare. On the phone, a boot survives+ * about two dozen dispatches that bind textures or images; after that+ * every dispatch still "succeeds", every readback still prints, and+ * nothing the GPU writes is ever visible again -- measured on+ * 20 October (logs/182513-run.log: writes land for the 13th, 15th and+ * 23rd kernel of the run and for nothing after it, while the same+ * kernels and four probe kernels all write correctly in a short run,+ * logs/182805-run.log). The same boot later refuses an allocation+ * outright with "bo_allocate: alloc failed". So the words below can be+ * a statement about how long the run had been going rather than about+ * the shader, and whoever reads them has to be told which.+ */+ /* and then every word of the whole buffer that differs from what went in,+ * capped so that a kernel which rewrites all 1024 of them does not push+ * the rest of a run out of the boot log (the loader keeps the first+ * 256 KB and the last 1 MB and drops the middle).+ */+ unsigned changed = 0, shown = 0;+ const uint32_t *was = (const uint32_t *)before[i];+ for (unsigned w = 0; was && w < sizes[i] / 4; w++) {+ if (p[w] == was[w])+ continue;+ changed++;+ if (shown < 24) {+ printf(" changed [%4u] 0x%08x -> 0x%08x\n", w, was[w], p[w]);+ shown++;+ }+ }+ if (was && changed)+ printf("air_run: device buffer %u: %u of its %u words differ from what this "+ "harness wrote%s\n", i, changed, sizes[i] / 4,+ shown < changed ? " (the first 24 are above)" : "");+ else if (was)+ printf("air_run: device buffer %u came back bit-identical to the fill, so this "+ "dispatch wrote nothing visible. Either the shader writes elsewhere, or "+ "the GPU has stopped making writes visible -- which it does after a few "+ "dozen dispatches in one boot (docs/air.md section 4).\n", i);pipe_buffer_unmap(pipe, xfer);}+ if (!finished) {+ fprintf(stderr, "air_run: %s: the dispatch did not finish, so no number here is "+ "evidence\n", name);+ return 1;+ }+/* and every image the kernel wrote: without this a dispatch that writes only* an image -- which is most of the corpus's kernels -- would have nothing to* read back, and air_run would have reported a successful run of something@@ -745,6 +1585,60 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,return 0;}+/* -- function constant values ---------------------------------------------+ *+ * --fc INDEX=VALUE supplies one function constant, and it exists because+ * nothing in this tree ever supplied one. air_module_function_constants()+ * registers every constant a module declares with its index and leaves+ * air_constant_value.bits at zero, so air.is_function_constant_defined folds+ * to true and every load yields zero -- for a constant whose global is an+ * `externally_initialized ... undef` in section "air.fc_initializer" there is+ * no default to fall back on, and zero is then not a default but a value this+ * compiler invented. 34 of RenderBox's 47 modules declare such a constant and+ * 25 of them derive their resource interface and their control flow from it in+ * an "air.static_init" constructor, so zero picks one arbitrary specialization+ * out of a state machine and the picture cannot say which.+ *+ * An index the module does not declare is refused rather than ignored: an+ * option that silently sets nothing would make every run agree with every+ * other one for the wrong reason.+ */+/* 64 and not 32, which is the number the corpus asks for rather than a round+ * one: fixed_frag_lph_cph and fixed_frag_lph_cpf declare 58 function constants+ * each, so the old array could not carry one pipeline's worth of them and the+ * refusal was "more than 32 function constants" -- on the module this matters+ * for most. The same bound is RW_MAX_CONSTANTS in air_metal.h, raised to 64+ * for the same reason.+ */+static struct {+ unsigned index;+ uint64_t bits;+} fc_set[64];+static unsigned num_fc_set;++static bool+apply_fc_values(struct air_constant_value *fc, unsigned num_fc)+{+ for (unsigned i = 0; i < num_fc_set; i++) {+ unsigned j;+ for (j = 0; j < num_fc; j++) {+ if (fc[j].index == fc_set[i].index) {+ fc[j].bits = fc_set[i].bits;+ fc[j].supplied = true;+ printf("air_run: function constant %u = 0x%llx\n", fc_set[i].index,+ (unsigned long long)fc_set[i].bits);+ break;+ }+ }+ if (j == num_fc) {+ fprintf(stderr, "air_run: this module declares no function constant %u"+ " (it declares %u)\n", fc_set[i].index, num_fc);+ return false;+ }+ }+ return true;+}+intmain(int argc, char **argv){@@ -762,7 +1656,7 @@ main(int argc, char **argv)*/bool want_library = false;/* --buffer FILE is AIR_RUN_BUFFER as an argument, and it exists because the- * device has no shell to set one with. Darwin 16.5's own root carries no sh at+ * device has no shell to set one with. iOS 16.5's own root carries no sh at* all -- /bin holds df and ps -- and the job list that runs this is read by* our init, which splits a line on blanks and has no notion of a variable* (userland/src/init.c, run_command). So a generated suite whose every@@ -775,12 +1669,108 @@ main(int argc, char **argv)setenv("IR3_SHADER_DEBUG", "disasm", 1);argv++;argc--;+ } else if (!strcmp(argv[1], "--zero")) {+ zero_fill = true;+ argv++;+ argc--;+ } else if (!strcmp(argv[1], "--ones")) {+ ones_fill = true;+ argv++;+ argc--;+ } else if (!strcmp(argv[1], "--packed16") && argc > 3) {+ packed16_fmt = packed16_by_name(argv[2], &packed16_name);+ if (packed16_fmt == PIPE_FORMAT_NONE)+ return 2;+ packed16_word = (unsigned)strtoul(argv[3], NULL, 0) & 0xffff;+ argv += 3;+ argc -= 3;+ } else if (!strcmp(argv[1], "--clear") && argc > 2) {+ /* R,G,B,A for target 0, then ':' and four more for target 1, and so on.+ * A colon and not a semicolon: these reach the phone as job lines, and+ * the init that reads them splits on blanks and promises nothing about+ * shell syntax -- a separator that a shell WOULD have meant something by+ * is one nobody can be sure about by reading the line.+ */+ const char *spec = argv[2];+ clear_count = 0;+ while (spec && *spec && clear_count < MAX_CLEARS) {+ float c[4] = { 0, 0, 0, 0 };+ if (sscanf(spec, "%f,%f,%f,%f", &c[0], &c[1], &c[2], &c[3]) != 4) {+ fprintf(stderr, "air_run: --clear wants R,G,B,A per target, ':' between "+ "targets\n");+ return 2;+ }+ for (unsigned i = 0; i < 4; i++)+ clear_colour[clear_count].f[i] = c[i];+ clear_count++;+ const char *sep = strchr(spec, ':');+ spec = sep ? sep + 1 : NULL;+ }+ clear_given = true;+ argv += 2;+ argc -= 2;+ } else if (!strcmp(argv[1], "--halftex")) {+ halftex = true;+ argv++;+ argc--;+ } else if (!strcmp(argv[1], "--miplevels") && argc > 2) {+ miplevel_fmt = packed16_by_name(argv[2], &packed16_name);+ if (miplevel_fmt == PIPE_FORMAT_NONE)+ return 2;+ argv += 2;+ argc -= 2;+ } else if (!strcmp(argv[1], "--packed16-rt") && argc > 2) {+ packed16_rt = packed16_by_name(argv[2], &packed16_name);+ if (packed16_rt == PIPE_FORMAT_NONE)+ return 2;+ argv += 2;+ argc -= 2;} else if (!strcmp(argv[1], "--library")) {want_library = library_fill = true;argv++;argc--;+ } else if (!strcmp(argv[1], "--stitch") && argc > 2) {+ /* A graph of air.visible functions, dispatched as one kernel. The+ * argument buffer is filled the wrapper's way for the same reason the+ * stitched entry point reads the wrapper's slots: a one-node graph+ * then has to write exactly what --library writes for that function.+ */+ stitch_text = argv[2];+ library_fill = true;+ argv += 2;+ argc -= 2;+ } else if (!strcmp(argv[1], "--block") && argc > 2) {+ unsigned x = 0, y = 0, z = 0;++ if (sscanf(argv[2], "%u,%u,%u", &x, &y, &z) != 3 || !x || !y || !z) {+ fprintf(stderr, "air_run: --block wants X,Y,Z and all three above zero\n");+ return 2;+ }+ block[0] = x, block[1] = y, block[2] = z;+ argv += 2;+ argc -= 2;+ } else if (!strcmp(argv[1], "--fc") && argc > 2) {+ unsigned idx = 0;+ uint64_t val = 0;++ if (sscanf(argv[2], "%u=0x%llx", &idx, (unsigned long long *)&val) != 2 &&+ sscanf(argv[2], "%u=%llu", &idx, (unsigned long long *)&val) != 2) {+ fprintf(stderr, "air_run: --fc wants INDEX=VALUE, decimal or 0x\n");+ return 2;+ }+ if (num_fc_set >= ARRAY_SIZE(fc_set)) {+ fprintf(stderr, "air_run: more than %u function constants\n",+ (unsigned)ARRAY_SIZE(fc_set));+ return 2;+ }+ fc_set[num_fc_set].index = idx;+ fc_set[num_fc_set].bits = val;+ num_fc_set++;+ argv += 2;+ argc -= 2;} else if (!strcmp(argv[1], "--buffer") && argc > 2) {- setenv("AIR_RUN_BUFFER", argv[2], 1);+ if (!add_buffer_patch(argv[2]))+ setenv("AIR_RUN_BUFFER", argv[2], 1);argv += 2;argc -= 2;} else {@@ -788,8 +1778,10 @@ main(int argc, char **argv)}}if (argc < 2) {- fprintf(stderr, "usage: air_run [--disasm] [--library] [--buffer FILE] "- "FILE[.metallib] [FUNCTION]\n");+ fprintf(stderr, "usage: air_run [--disasm] [--library] [--block X,Y,Z] "+ "[--buffer FILE | --buffer N@OFF=HEX] "+ "[--fc INDEX=VALUE] "+ "[--zero | --ones] FILE[.metallib] [FUNCTION]\n");return 2;}@@ -817,6 +1809,27 @@ main(int argc, char **argv)size_t bitcode_size = size;const char *want = argc > 2 ? argv[2] : NULL;const char *name = argv[1];++ /* A stitched graph is still handed to air_to_nir as one module plus the+ * graph, and that module is the OUTPUT node's: its own visible function is+ * one node like any other.+ */+ const struct air_stitch_graph *graph = NULL;+ struct air_stitch_function *stitch_fns = NULL;+ unsigned num_stitch_fns = 0;+ if (stitch_text) {+ char why[256] = { 0 };+ graph = air_stitch_parse(ctx, stitch_text, why, sizeof(why));+ if (graph)+ num_stitch_fns = air_stitch_functions_of(ctx, graph, data, size, &stitch_fns,+ why, sizeof(why));+ if (!graph || !num_stitch_fns) {+ fprintf(stderr, "air_run: --stitch: %s\n", why[0] ? why : "no functions");+ return 2;+ }+ want = graph->nodes[graph->output_node].function;+ }+if (size >= 4 && !memcmp(data, "MTLB", 4)) {struct air_metallib lib;const char *err;@@ -898,8 +1911,13 @@ main(int argc, char **argv)/* the shader */struct air_constant_value *fc;unsigned num_fc = air_module_function_constants(ctx, bitcode, bitcode_size, &fc);+ if (!apply_fc_values(fc, num_fc))+ return 1;struct air_to_nir_options air_options = { .constants = fc, .num_constants = num_fc,- .library_wrapper = want_library };+ .library_wrapper = want_library,+ .stitch = graph,+ .stitch_functions = stitch_fns,+ .num_stitch_functions = num_stitch_fns };struct air_shader_info info;const char *err;nir_shader *fs = air_to_nir(ctx, bitcode, bitcode_size, NULL, &air_options, options,@@ -923,6 +1941,10 @@ main(int argc, char **argv)enum pipe_format rt_format = PIPE_FORMAT_R32G32B32A32_FLOAT;bool rt_is_float = true;bool rt_is_half = false;+ /* --packed16 renders INTO the same format it sampled, which is the written half of+ * the probe: the pixel is packed back into 16 bits by the GPU and read out raw.+ */+ bool rt_is_packed16 = packed16_rt != PIPE_FORMAT_NONE;/* Which attachment to read back: the lowest one the shader writes, not* target 0. clearFragment04 writes FRAG_RESULT_DATA1 alone, and target 0* then holds the clear colour -- a correct result that looks exactly like@@ -977,6 +1999,11 @@ main(int argc, char **argv)rt_is_float = true;}}+ if (rt_is_packed16) {+ rt_format = packed16_rt;+ rt_is_float = false;+ rt_is_half = false;+ }if (read_rt == ~0u) {fprintf(stderr, "air_run: %s writes no colour attachment\n", name);return 1;@@ -1010,7 +2037,108 @@ main(int argc, char **argv)return 1;}- finalize_for_gallium(fs);+ /* A shader that discards cannot be run by this screen, and that is+ * softpipe's limit rather than the frontend's. air.discard_fragment becomes+ * NIR's `demote` (air_intrinsics.c says at length why it is demote and not+ * terminate); nir_to_tgsi sends it to TGSI_OPCODE_DEMOTE, which the+ * interpreter does not implement -- tgsi_exec.c has a case for KILL and+ * none for DEMOTE, so exec_instruction falls through to its `assert(0)` and+ * the whole run aborts at tgsi_exec.c:5943. An abort inside a check reads+ * like a crash in the thing being checked, so it is refused here by name,+ * the way the multisample textures below are.+ *+ * ir3 has both instructions and compiles these, so what is open is the+ * VALUE a discarding shader leaves in the framebuffer, and that is a+ * question for the phone.+ */+ /*+ * On SOFTPIPE only, and that qualification is the whole point of the comment above:+ * "ir3 has both instructions and compiles these, so what is open is the VALUE a+ * discarding shader leaves in the framebuffer, and that is a question for the phone."+ * The gate was written without it and asked only whether the module discards, so on+ * the phone -- where the question was supposed to be answered -- fifteen of+ * RenderBox's forty-seven modules were refused for a limitation of a screen that was+ * not in use. `mask_color_fragment` is one of them, and it is what draws a widget.+ *+ * A refusal that names the wrong wall reads as principled when it is merely absent,+ * which is the defect this file's own log line demonstrated: it printed "this screen's+ * TGSI interpreter" over a run whose every line said "on Adreno 7c+ Gen 3".+ */+ if (fs->info.fs.uses_discard && screen->get_name &&+ !strcmp(screen->get_name(screen), "softpipe")) {+ fprintf(stderr, "air_run: %s discards fragments, and THIS screen's TGSI "+ "interpreter has no DEMOTE case at all (tgsi_exec.c), so it is "+ "not run rather than run against the wrong thing. On the Adreno "+ "it runs: ir3 has the instruction\n", name);+ return 1;+ }++ /* And the rest of the family this interpreter does not have.+ *+ * `grep -c TGSI_OPCODE_VOTE_ALL tgsi_exec.c` is 0, and so are BALLOT,+ * READ_FIRST, READ_INVOC, READ_HELPER and DEMOTE: nir_to_tgsi emits all of+ * them and softpipe's exec_instruction implements none, so it falls through+ * to its own assert(0) -- an abort in the middle of a check, which reads+ * like a crash in the thing being checked rather than like a screen that+ * cannot do this. The quad swaps are worse again: nir_to_tgsi has no case+ * for them either, prints "Unknown intrinsic" and then asserts inside+ * ureg_swizzle on the register it never made.+ *+ * So a shader that asks for a cross-lane value is refused here by name. It+ * is not a frontend limit and not a hardware one -- ir3 has real+ * instructions for every one of these, and compiles them -- which is why+ * the refusal names the screen.+ */+ const char *no_lanes = NULL;+ nir_foreach_function_impl(impl, fs) {+ nir_foreach_block(block, impl) {+ nir_foreach_instr(instr, block) {+ if (instr->type != nir_instr_type_intrinsic)+ continue;+ nir_intrinsic_op op = nir_instr_as_intrinsic(instr)->intrinsic;+ switch (op) {+ case nir_intrinsic_vote_all:+ case nir_intrinsic_vote_any:+ case nir_intrinsic_vote_feq:+ case nir_intrinsic_vote_ieq:+ case nir_intrinsic_ballot:+ case nir_intrinsic_read_first_invocation:+ case nir_intrinsic_read_invocation:+ case nir_intrinsic_reduce:+ case nir_intrinsic_inclusive_scan:+ case nir_intrinsic_exclusive_scan:+ case nir_intrinsic_shuffle:+ case nir_intrinsic_shuffle_xor:+ case nir_intrinsic_shuffle_up:+ case nir_intrinsic_shuffle_down:+ case nir_intrinsic_quad_broadcast:+ case nir_intrinsic_quad_swap_horizontal:+ case nir_intrinsic_quad_swap_vertical:+ case nir_intrinsic_quad_swap_diagonal:+ case nir_intrinsic_is_helper_invocation:+ no_lanes = nir_intrinsic_infos[op].name;+ break;+ default:+ break;+ }+ if (no_lanes)+ break;+ }+ if (no_lanes)+ break;+ }+ if (no_lanes)+ break;+ }+ if (no_lanes) {+ fprintf(stderr, "air_run: %s reads another lane (NIR's %s), and this screen's "+ "TGSI interpreter implements none of the subgroup or quad "+ "opcodes nir_to_tgsi emits for them, so it is not run rather "+ "than aborting inside softpipe\n", name, no_lanes);+ return 1;+ }++ finalize_for_gallium(fs, screen);struct pipe_context *pipe = screen->context_create(screen, NULL, 0);@@ -1046,7 +2174,19 @@ main(int argc, char **argv)pipe->set_framebuffer_state(pipe, &fb);pipe->set_viewport_states(pipe, 0, 1, &(struct pipe_viewport_state){- .scale = { WIDTH / 2.0f, HEIGHT / 2.0f, 1.0f },+ /* The y scale is NEGATIVE, and that is Metal's rule rather than a convention picked+ * here. Metal's render-target origin is the top left and its NDC y points up, so clip+ * y = +1 belongs in memory row 0; a positive y scale puts clip y = -1 there, which is+ * OpenGL's drawable rule and what this code did until now (docs/metal.md 10.28). Mesa+ * negates y in exactly this place, for exactly this reason, when it draws to a window.+ *+ * It is landed only now because until the varying was fixed (10.32) nothing could see+ * it: every draw of the corpus either writes a constant or samples one fixed texel, and+ * over a constant an orientation is invisible. The sixteen-pixel grid of 10.32 is what+ * it is landed against -- it must turn from (100+x, 200+y, ...) into+ * (100+x, 200+(3-y), ...), the texture mirrored vertically, on both back ends.+ */+ .scale = { WIDTH / 2.0f, -(HEIGHT / 2.0f), 1.0f },.translate = { WIDTH / 2.0f, HEIGHT / 2.0f, 0.0f },});@@ -1063,21 +2203,51 @@ main(int argc, char **argv)});pipe->bind_rasterizer_state(pipe, rast);- /* every constant buffer the shader reads, filled with the pattern */+ /* Every buffer the shader reads, filled with the pattern.+ *+ * A `constant` buffer and a `device` buffer are TWO BINDINGS and not one.+ * The frontend translates a load from the first into load_ubo and from the+ * second into load_ssbo (docs/air.md section 3), and this path used to bind+ * both with set_constant_buffer -- so a draw whose shader reads a device+ * buffer read ZEROS, and read them silently: the pixel came back, four+ * numbers wide, with nothing anywhere saying a binding had been missed. The+ * compute path above has always bound the two separately; only the draw path+ * had them confused. Found by tools/air/callptr.ll, whose last two lanes are+ * a device buffer and whose first two are not, so the picture said which+ * binding was wrong rather than that something was.+ */+ struct pipe_shader_buffer fsb[32] = { 0 };+ unsigned num_fs_ssbo = 0;for (unsigned i = 0; i < info.num_resources; i++) {const struct air_resource *r = &info.resources[i];if (r->kind != AIR_RESOURCE_CONSTANT_BUFFER && r->kind != AIR_RESOURCE_DEVICE_BUFFER)continue;unsigned bytes = r->size ? r->size : 256;uint8_t *bits = malloc(bytes);- unsigned from_file = fill_buffer(bits, bytes);- struct pipe_resource *buf = pipe_buffer_create(screen, PIPE_BIND_CONSTANT_BUFFER,- PIPE_USAGE_DEFAULT, bytes);+ unsigned from_file = fill_buffer(r->index, bits, bytes);+ bool device = r->kind == AIR_RESOURCE_DEVICE_BUFFER;+ struct pipe_resource *buf =+ pipe_buffer_create(screen,+ device ? PIPE_BIND_SHADER_BUFFER : PIPE_BIND_CONSTANT_BUFFER,+ PIPE_USAGE_DEFAULT, bytes);pipe_buffer_write(pipe, buf, 0, bytes, bits);- pipe->set_constant_buffer(pipe, PIPE_SHADER_FRAGMENT, r->index, false,- &(struct pipe_constant_buffer){- .buffer = buf, .buffer_size = bytes,- });+ if (device) {+ if (r->index < ARRAY_SIZE(fsb)) {+ fsb[r->index].buffer = buf;+ fsb[r->index].buffer_size = bytes;+ num_fs_ssbo = MAX2(num_fs_ssbo, r->index + 1);+ } else {+ fprintf(stderr, "air_run: %s binds a device buffer at %u, past the %zu this "+ "path keeps room for\n", name, r->index, ARRAY_SIZE(fsb));+ free(bits);+ return 1;+ }+ } else {+ pipe->set_constant_buffer(pipe, PIPE_SHADER_FRAGMENT, r->index, false,+ &(struct pipe_constant_buffer){+ .buffer = buf, .buffer_size = bytes,+ });+ }if (from_file)printf("air_run: %s buffer at index %u, %u bytes, %u of them from ""AIR_RUN_BUFFER\n",@@ -1089,6 +2259,8 @@ main(int argc, char **argv)r->index, bytes);free(bits);}+ if (num_fs_ssbo)+ pipe->set_shader_buffers(pipe, PIPE_SHADER_FRAGMENT, 0, num_fs_ssbo, fsb, 0);/* every sampled texture the shader declares, and a sampler for it */unsigned num_tex = 0;@@ -1107,6 +2279,32 @@ main(int argc, char **argv)*/enum pipe_texture_target target = PIPE_TEXTURE_2D;enum glsl_sampler_dim dim = GLSL_SAMPLER_DIM_2D;+ bool is_array = false;+ /* A SHADOW SAMPLE WANTS A DEPTH TEXTURE, and binding a colour one is not a+ * harmless approximation: GL and Vulkan both leave a shadow lookup on a+ * non-depth texture undefined, and the Adreno says so by ignoring the+ * comparison. The first device run of these cases came back with the TEXEL+ * -- 101 where a comparison must answer 0 or 1, and for `always` and `never`+ * too, which no compare function and no clamp can explain -- against+ * r32g32b32a32_float bound for a Metal depth2d<float>, which is the one thing+ * in that run that was this harness's own doing (docs/air.md 3.3).+ *+ * So a slot a shadow instruction reads gets a real depth texture, and its+ * texels are the only pattern that fits: (x + TEX*y) / 16, which is sixteen+ * distinct values, every one exactly representable in binary32 and inside the+ * [0, 1] a depth format is normalised to. The colour pattern cannot be used --+ * 100 does not fit in a depth texel at all.+ */+ bool shadow = false;+ nir_foreach_function_impl(impl, fs) {+ nir_foreach_block(block, impl) {+ nir_foreach_instr(instr, block) {+ if (instr->type == nir_instr_type_tex &&+ nir_instr_as_tex(instr)->is_shadow)+ shadow = true;+ }+ }+ }nir_foreach_function_impl(impl, fs) {nir_foreach_block(block, impl) {nir_foreach_instr(instr, block) {@@ -1122,6 +2320,7 @@ main(int argc, char **argv)tex_is_float = false;}dim = t->sampler_dim;+ is_array = t->is_array;if (dim == GLSL_SAMPLER_DIM_3D)target = PIPE_TEXTURE_3D;else if (dim == GLSL_SAMPLER_DIM_MS)@@ -1131,77 +2330,245 @@ main(int argc, char **argv)}}have_format:;+ /* and a depth format where a comparison reads it, whatever the dest_type said:+ * a shadow sample's result is a comparison and not a texel, so the element type+ * the instruction carries says nothing about the texture's format.+ */+ if (halftex) {+ if (dim != GLSL_SAMPLER_DIM_2D || is_array) {+ fprintf(stderr, "air_run: --halftex binds a 2D table and %s samples "+ "something else\n", name);+ return 1;+ }+ tex_format = PIPE_FORMAT_R16G16B16A16_FLOAT;+ tex_is_float = true;+ }+ if (packed16_fmt != PIPE_FORMAT_NONE || miplevel_fmt != PIPE_FORMAT_NONE) {+ if (dim != GLSL_SAMPLER_DIM_2D || is_array) {+ fprintf(stderr, "air_run: --packed16 and --miplevels sample a 2D texture "+ "and %s is not one\n", name);+ return 1;+ }+ tex_format = packed16_fmt != PIPE_FORMAT_NONE ? packed16_fmt : miplevel_fmt;+ tex_is_float = true;+ }+ if (shadow) {+ if (dim != GLSL_SAMPLER_DIM_2D || is_array) {+ fprintf(stderr, "air_run: %s compares against a texture this harness makes no "+ "depth version of (dim %u%s); it is not run rather than run "+ "against a colour texture, which is what the Adreno answers "+ "with the texel\n", name, dim, is_array ? ", array" : "");+ return 1;+ }+ tex_format = PIPE_FORMAT_Z32_FLOAT;+ tex_is_float = true;+ }if (dim == GLSL_SAMPLER_DIM_MS) {fprintf(stderr, "air_run: %s samples a multisample texture; this harness binds ""none, so it is not run rather than run against the wrong thing\n",name);return 1;}- if (dim != GLSL_SAMPLER_DIM_2D && dim != GLSL_SAMPLER_DIM_3D) {+ if (dim != GLSL_SAMPLER_DIM_1D && dim != GLSL_SAMPLER_DIM_2D &&+ dim != GLSL_SAMPLER_DIM_3D) {fprintf(stderr, "air_run: %s samples a texture of a kind this harness does not ""bind (dim %u)\n", name, dim);return 1;}- unsigned depth = target == PIPE_TEXTURE_3D ? TEX_SIZE : 1;- unsigned tex_z = target == PIPE_TEXTURE_3D ? TEX_Y : 0;++ /* An ARRAY texture is a different resource and not a 2D one sampled with+ * a third coordinate, so the harness has to make one or the check would+ * be about nothing: a layer index handed to a 2D texture is ignored, and+ * every layer would answer with layer 0's texels. TEX_SIZE layers, filled+ * by the same rule as a 3D texture's slices -- texel (x, y, z) of slot n+ * is (100+x+1000n, 200+y, 300+x+y+z, 400) -- so a lowering that dropped+ * the layer, or put it in the wrong component, comes back with a number+ * from the wrong slice rather than a plausible one.+ *+ * Gallium addresses an array's layers through pipe_box.z and .depth, the+ * same fields a 3D texture's slices use, while the RESOURCE carries them+ * in array_size rather than depth0 -- which is why depth0 and the box's+ * depth part company here.+ */+ if (is_array)+ target = dim == GLSL_SAMPLER_DIM_1D ? PIPE_TEXTURE_1D_ARRAY+ : PIPE_TEXTURE_2D_ARRAY;+ else if (dim == GLSL_SAMPLER_DIM_1D)+ target = PIPE_TEXTURE_1D;+ unsigned height = dim == GLSL_SAMPLER_DIM_1D ? 1 : TEX_SIZE;+ unsigned slices = (target == PIPE_TEXTURE_3D || is_array) ? TEX_SIZE : 1;+ unsigned tex_z = slices > 1 ? TEX_Y : 0;struct pipe_resource ttempl = {.target = target,.format = tex_format,- .width0 = TEX_SIZE, .height0 = TEX_SIZE, .depth0 = depth, .array_size = 1,+ .width0 = TEX_SIZE, .height0 = height,+ .depth0 = is_array ? 1 : slices,+ .array_size = is_array ? slices : 1,.bind = PIPE_BIND_SAMPLER_VIEW,};- struct pipe_box box = { .width = TEX_SIZE, .height = TEX_SIZE, .depth = depth };+ if (halftex) {+ ttempl.width0 = HALFTEX_W;+ ttempl.height0 = 1;+ }+ if (miplevel_fmt != PIPE_FORMAT_NONE) {+ /* the shape the census found, not this harness's 4x4: a two-byte format+ * differs from a four-byte one most in how its levels are laid out, and+ * 776x24 is where the window server asks for it+ */+ ttempl.width0 = MIPTEX_W;+ ttempl.height0 = MIPTEX_H;+ ttempl.last_level = MIPTEX_LEVELS - 1;+ ttempl.bind |= PIPE_BIND_RENDER_TARGET;+ }+ struct pipe_box box = { .width = TEX_SIZE, .height = height, .depth = slices };struct pipe_resource *texes[PIPE_MAX_SAMPLERS] = { 0 };for (unsigned slot = 0; slot < num_tex && slot < PIPE_MAX_SAMPLERS; slot++) {float texels[TEX_SIZE * TEX_SIZE * TEX_SIZE * 4];int32_t itexels[TEX_SIZE * TEX_SIZE * TEX_SIZE * 4];- for (unsigned z = 0; z < depth; z++) {- for (unsigned y = 0; y < TEX_SIZE; y++) {+ texes[slot] = screen->resource_create(screen, &ttempl);+ if (halftex) {+ uint16_t words[HALFTEX_W * 4];+ for (unsigned i = 0; i < HALFTEX_W; i++)+ halftex_texel(i, &words[i * 4]);+ struct pipe_box hb = { .width = HALFTEX_W, .height = 1, .depth = 1 };+ pipe->texture_subdata(pipe, texes[slot], 0, PIPE_MAP_WRITE, &hb, words,+ HALFTEX_W * 4 * sizeof(uint16_t),+ HALFTEX_W * 4 * sizeof(uint16_t));+ printf("air_run: %ux1 r16g16b16a16_float table, texel i holding the half "+ "BIT PATTERNS ((i<<7)&0x7bff, that|0x8000, i, 0x3c00) written as "+ "integers -- no float-to-half conversion anywhere in this program\n",+ HALFTEX_W);+ continue;+ }+ if (miplevel_fmt != PIPE_FORMAT_NONE) {+ /* every level filled with the binary code of its own number, packed+ * through util_format so the harness never writes bits by hand+ */+ for (unsigned lv = 0; lv < MIPTEX_LEVELS; lv++) {+ unsigned w = MAX2(1u, MIPTEX_W >> lv), h = MAX2(1u, MIPTEX_H >> lv);+ unsigned texel = util_format_get_blocksize(tex_format);+ uint8_t *rows = calloc((size_t)w * h, texel);+ float v[4];+ miplevel_texel(lv, v);+ /* ONE texel packed and then repeated. util_format_pack_rgba reads w+ * TEXELS from its source, not one -- handing it a single colour and a+ * width of 776 reads 776 texels out of a four-float array, which is+ * how the first version of this segfaulted.+ */+ uint8_t one[16] = { 0 };+ util_format_pack_rgba(tex_format, one, v, 1);+ for (size_t k = 0; k < (size_t)w * h; k++)+ memcpy(rows + k * texel, one, texel);+ struct pipe_box lb = { .width = w, .height = h, .depth = 1 };+ pipe->texture_subdata(pipe, texes[slot], lv, PIPE_MAP_WRITE, &lb, rows,+ w * texel, (size_t)w * h * texel);+ free(rows);+ }+ printf("air_run: %ux%u %s pyramid, %u levels, level L filled with the "+ "binary code of L (r=bit0 g=bit1 b=bit2 a=bit3)\n", MIPTEX_W,+ MIPTEX_H, util_format_short_name(tex_format), MIPTEX_LEVELS);+ continue;+ }+ if (packed16_fmt != PIPE_FORMAT_NONE) {+ /* one 16-bit word in every texel, so no coordinate, filter or address+ * mode can change which channel answers+ */+ uint16_t words[TEX_SIZE * TEX_SIZE];+ for (unsigned k = 0; k < TEX_SIZE * TEX_SIZE; k++)+ words[k] = (uint16_t)packed16_word;+ pipe->texture_subdata(pipe, texes[slot], 0, PIPE_MAP_WRITE, &box, words,+ TEX_SIZE * sizeof(uint16_t),+ TEX_SIZE * TEX_SIZE * sizeof(uint16_t));+ continue;+ }+ if (shadow) {+ /* one float per texel, not four, and a depth row is a quarter as wide */+ for (unsigned y = 0; y < TEX_SIZE; y++)+ for (unsigned x = 0; x < TEX_SIZE; x++)+ texels[y * TEX_SIZE + x] = depth_texel_value(x, y);+ pipe->texture_subdata(pipe, texes[slot], 0, PIPE_MAP_WRITE, &box, texels,+ TEX_SIZE * sizeof(float),+ TEX_SIZE * TEX_SIZE * sizeof(float));+ continue;+ }+ for (unsigned z = 0; z < slices; z++) {+ for (unsigned y = 0; y < height; y++) {for (unsigned x = 0; x < TEX_SIZE; x++) {- unsigned i = ((z * TEX_SIZE + y) * TEX_SIZE + x) * 4;+ unsigned i = ((z * height + y) * TEX_SIZE + x) * 4;texel_value(slot, x, y, z, &texels[i]);for (unsigned c = 0; c < 4; c++)itexels[i + c] = (int32_t)texels[i + c];}}}- texes[slot] = screen->resource_create(screen, &ttempl);pipe->texture_subdata(pipe, texes[slot], 0, PIPE_MAP_WRITE, &box,tex_is_float ? (const void *)texels : (const void *)itexels,TEX_SIZE * 4 * sizeof(float),- TEX_SIZE * TEX_SIZE * 4 * sizeof(float));+ height * TEX_SIZE * 4 * sizeof(float));}- void *sampler = pipe->create_sampler_state(pipe, &(struct pipe_sampler_state){- .wrap_s = PIPE_TEX_WRAP_CLAMP_TO_EDGE,- .wrap_t = PIPE_TEX_WRAP_CLAMP_TO_EDGE,- .wrap_r = PIPE_TEX_WRAP_CLAMP_TO_EDGE,- .min_img_filter = PIPE_TEX_FILTER_NEAREST,- .mag_img_filter = PIPE_TEX_FILTER_NEAREST,- .min_mip_filter = PIPE_TEX_MIPFILTER_NONE,- });+ /* the API's sampler where the shader declares none, and its own where it+ * does -- per slot, since a shader may mix the two+ */struct pipe_sampler_view *views[PIPE_MAX_SHADER_SAMPLER_VIEWS] = { 0 };void *samplers[PIPE_MAX_SAMPLERS] = { 0 };+ bool compare_slot[PIPE_MAX_SAMPLERS] = { 0 };+ compare_sampled_slots(fs, compare_slot, PIPE_MAX_SAMPLERS);+ if (compare_without_a_function(&info, compare_slot, PIPE_MAX_SAMPLERS, name))+ return 1;for (unsigned i = 0; i < num_tex && i < PIPE_MAX_SAMPLERS; i++) {struct pipe_sampler_view vtempl = { 0 };u_sampler_view_default_template(&vtempl, texes[i], texes[i]->format);views[i] = pipe->create_sampler_view(pipe, texes[i], &vtempl);- samplers[i] = sampler;+ const struct air_sampler_state *own = NULL;+ struct pipe_sampler_state sst = sampler_for_slot(&info, i, false,+ compare_slot[i], &own);+ samplers[i] = pipe->create_sampler_state(pipe, &sst);+ if (own)+ print_own_sampler(i, own, compare_slot[i]);}pipe->set_sampler_views(pipe, PIPE_SHADER_FRAGMENT, 0, num_tex, 0, false, views);pipe->bind_sampler_states(pipe, PIPE_SHADER_FRAGMENT, 0, num_tex, samplers);float want[4];texel_value(0, TEX_X, TEX_Y, tex_z, want);- printf("air_run: %u texture%s bound: %ux%ux%u %s, texel (x,y,z) of slot n = "+ if (miplevel_fmt != PIPE_FORMAT_NONE)+ printf("air_run: %u texture%s bound: %ux%u %s, %u mip levels, level L holding "+ "the binary code of L+1 -- so the four numbers below NAME the level "+ "that answered, and 0 0 0 0 is no level at all\n", num_tex, num_tex == 1 ? "" : "s",+ MIPTEX_W, MIPTEX_H, util_format_short_name(tex_format), MIPTEX_LEVELS);+ else if (shadow)+ printf("air_run: %u DEPTH texture%s bound: %ux%u %s, texel (x,y) = (x + 4y)/16, "+ "because a shadow sample of a colour texture is undefined and the Adreno "+ "answers it with the texel\n", num_tex, num_tex == 1 ? "" : "s",+ TEX_SIZE, TEX_SIZE, util_format_short_name(tex_format));+ else+ printf("air_run: %u texture%s bound: %ux%ux%u %s%s, texel (x,y,z) of slot n = ""(100+x+1000n, 200+y, 300+x+y+z, 400), nearest, clamped\n",- num_tex, num_tex == 1 ? "" : "s", TEX_SIZE, TEX_SIZE, depth,- util_format_short_name(tex_format));- printf("air_run: expected %.9g %.9g %.9g %.9g (the centre of texel %u,%u,%u sampled "- "once)\n", (double)want[0], (double)want[1], (double)want[2], (double)want[3],- TEX_X, TEX_Y, tex_z);+ num_tex, num_tex == 1 ? "" : "s", TEX_SIZE, height, slices,+ util_format_short_name(tex_format),+ is_array ? " array (z is the layer)" : "");+ if (!shadow && miplevel_fmt == PIPE_FORMAT_NONE &&+ packed16_fmt == PIPE_FORMAT_NONE)+ printf("air_run: expected %.9g %.9g %.9g %.9g (the centre of texel %u,%u,%u "+ "sampled once)\n", (double)want[0], (double)want[1], (double)want[2],+ (double)want[3], TEX_X, TEX_Y, tex_z);+ /* A comparison's result is not a texel, so no "expected" line is printed for one:+ * the first device run of these cases printed the colour pattern's texel there and+ * the GPU then answered WITH that texel, which read as agreement until the numbers+ * were looked at. The line said what the comparison reads and the device said what+ * it returned, and they were the same sentence.+ */+ for (unsigned i = 0; i < num_tex && i < PIPE_MAX_SAMPLERS; i++)+ if (compare_slot[i]) {+ printf("air_run: sampler %u COMPARES against depth texel (%u,%u) = %.9g, so "+ "the result is a 0, a 1 or a filtered fraction of them and never a "+ "texel\n", i, TEX_X, TEX_Y,+ (double)depth_texel_value(TEX_X, TEX_Y));+ break;+ }}/* the quad */@@ -1222,7 +2589,7 @@ main(int argc, char **argv)pipe->bind_vertex_elements_state(pipe, velem);nir_shader *vs = quad_vertex_shader(options, info.num_varyings);- finalize_for_gallium(vs);+ finalize_for_gallium(vs, screen);void *vs_cso = pipe->create_vs_state(pipe, &(struct pipe_shader_state){.type = PIPE_SHADER_IR_NIR, .ir.nir = vs,});@@ -1232,8 +2599,27 @@ main(int argc, char **argv)});pipe->bind_fs_state(pipe, fs_cso);+ /* The clear colour is the PREVIOUS CONTENT of the attachment, and that is the+ * only thing on this machine a framebuffer fetch can read: softpipe's TGSI+ * interpreter has no FBFETCH at all, so the only driver here that can answer is+ * the one on the phone, and there a fetch in a single draw reads what the clear+ * put down. --clear turns the one number a fetch could return into a number+ * CHOSEN, so a shader that returns the fetch unchanged is distinguishable from+ * one that returns zero, which is what a fetch that read nothing returns.+ */union pipe_color_union clear = { .ui = { 0, 0, 0, 0 } };- pipe->clear(pipe, PIPE_CLEAR_COLOR, NULL, &clear, 0.0, 0);+ if (!clear_given) {+ pipe->clear(pipe, PIPE_CLEAR_COLOR, NULL, &clear, 0.0, 0);+ } else {+ for (unsigned i = 0; i < num_rt; i++) {+ const union pipe_color_union *c =+ &clear_colour[MIN2(i, clear_count - 1)];+ pipe->clear_render_target(pipe, fb.cbufs[i], c, 0, 0, WIDTH, HEIGHT, false);+ printf("air_run: render target %u cleared to %.9g %.9g %.9g %.9g, which is "+ "what a framebuffer fetch of it in this draw will read\n", i,+ (double)c->f[0], (double)c->f[1], (double)c->f[2], (double)c->f[3]);+ }+ }struct pipe_draw_info draw = {.mode = MESA_PRIM_TRIANGLE_STRIP, .index_bounds_valid = true,@@ -1241,7 +2627,11 @@ main(int argc, char **argv)};struct pipe_draw_start_count_bias sc = { .start = 0, .count = 4 };pipe->draw_vbo(pipe, &draw, 0, NULL, &sc, 1);- pipe->flush(pipe, NULL, 0);+ if (!finish_or_say_why(screen, pipe, "draw")) {+ fprintf(stderr, "air_run: %s: the draw did not finish, so no pixel here is "+ "evidence\n", name);+ return 1;+ }/* the pixel */struct pipe_transfer *xfer;@@ -1261,6 +2651,18 @@ main(int argc, char **argv)want_disasm ? ", ir3's instructions are above" : "");printf("air_run: the pixel at (0, 0) of render target %u came back as:\n",read_rt);+ if (rt_is_packed16) {+ printf("air_run: packed16-rt %s: the shader's colour came back as the word "+ "0x%04x\n", packed16_name, ((uint16_t *)pix)[0]);+ pipe->texture_unmap(pipe, xfer);+ printf("air_run: drew %ux%u into %s, %u render target%s\n", WIDTH, HEIGHT,+ util_format_short_name(rt_format), num_rt, num_rt == 1 ? "" : "s");+ pipe->destroy(pipe);+ screen->destroy(screen);+ ralloc_free(ctx);+ glsl_type_singleton_decref();+ return 0;+ }for (unsigned c = 0; c < 4; c++) {if (rt_is_half)printf(" %.9g\n", (double)_mesa_half_to_float(((uint16_t *)pix)[c]));diff --git a/tools/mesa/air/air_stitch.c b/tools/mesa/air/air_stitch.cnew file mode 100644index 0000000..e723ed7--- /dev/null+++ b/tools/mesa/air/air_stitch.c@@ -0,0 +1,224 @@+/*+ * Copyright © 2026 Darwin on redwood contributors+ * SPDX-License-Identifier: MIT+ */++/*+ * A function-stitching graph, typed as text, and the functions it names.+ *+ * The graph itself is Metal's: `air_stitch_graph` in nir_air.h is+ * MTLFunctionStitchingGraph field for field. What is here is the one thing+ * Metal has no equivalent of -- a way to write a graph on a command line --+ * and it is in the frontend's library rather than in one tool because every+ * tool that touches stitching needs it: air2nir folds a graph, air_run+ * dispatches one, and the Metal library's test will build the same graphs+ * through the objective API, where they have to be the same graphs or the+ * comparison is between two different things.+ *+ * "_pf_invertRed(in0); _pf_redContour(n0); _pf_textureDiff(n0, n1)"+ *+ * One node per `;', in dependency order, numbered by position. An argument is+ * `inN' (the stitched function's own argument N), `nN' (node N's result),+ * `texN' / `sampN' / `bufN' (a texture, a sampler or a constant buffer bound at+ * index N), or `gridpos'. The output node is the last one unless the text ends+ * with `=> nN'.+ */++#include <stdarg.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>++#include "nir_air.h"+#include "util/macros.h"+#include "util/ralloc.h"++/* Every refusal goes into the caller's buffer rather than to stderr: this is+ * library code, and a tool that wants the sentence on one line with its own+ * prefix cannot get it back out of stderr.+ */+static void PRINTFLIKE(3, 4)+say(char *error, size_t size, const char *fmt, ...)+{+ if (!error || !size)+ return;+ va_list ap;+ va_start(ap, fmt);+ vsnprintf(error, size, fmt, ap);+ va_end(ap);+}++static char *+trim(char *s)+{+ while (*s == ' ' || *s == '\t')+ s++;+ char *e = s + strlen(s);+ while (e > s && (e[-1] == ' ' || e[-1] == '\t'))+ *--e = '\0';+ return s;+}++static bool+parse_arg(const char *tok, struct air_stitch_arg *out)+{+ static const struct { const char *prefix; enum air_stitch_arg_kind kind; } kinds[] = {+ /* "in" before "n", or `in0' would read as node 0 of a prefix "n" */+ { "in", AIR_STITCH_INPUT },+ { "tex", AIR_STITCH_TEXTURE },+ { "samp", AIR_STITCH_SAMPLER },+ { "buf", AIR_STITCH_BUFFER },+ { "n", AIR_STITCH_NODE },+ };+ if (!strcmp(tok, "gridpos")) {+ *out = (struct air_stitch_arg){ .kind = AIR_STITCH_GRID_POSITION };+ return true;+ }+ for (unsigned k = 0; k < ARRAY_SIZE(kinds); k++) {+ size_t len = strlen(kinds[k].prefix);+ if (strncmp(tok, kinds[k].prefix, len))+ continue;+ char *end = NULL;+ unsigned long v = strtoul(tok + len, &end, 10);+ if (!end || *end || end == tok + len)+ continue;+ *out = (struct air_stitch_arg){ .kind = kinds[k].kind, .index = (unsigned)v };+ return true;+ }+ return false;+}++/* "f(a, b)" -> one node.+ *+ * strtok_r and not strtok, here and in the caller: strtok keeps one buffer for+ * the whole program, so splitting a node's arguments inside the loop that+ * splits the graph ENDED that loop -- a two-node graph parsed as one node and+ * the shader stored node 0's result. That is exactly what a stitched call that+ * had not been wired would look like, and it was the parser.+ */+static bool+parse_node(void *ctx, char *text, struct air_stitch_node *node,+ char *error, size_t size)+{+ text = trim(text);+ char *open = strchr(text, '(');+ char *close = strrchr(text, ')');+ if (!open || !close || close < open) {+ say(error, size, "`%s' is not `function(arguments)'", text);+ return false;+ }+ *open = *close = '\0';+ node->function = ralloc_strdup(ctx, trim(text));++ char *args = open + 1;+ unsigned n = 0;+ bool empty = true;+ for (char *p = args; *p; p++) {+ n += *p == ',';+ empty &= *p == ' ' || *p == '\t';+ }+ n = empty ? 0 : n + 1;++ struct air_stitch_arg *out = rzalloc_array(ctx, struct air_stitch_arg, MAX2(n, 1));+ unsigned idx = 0;+ char *save = NULL;+ for (char *tok = strtok_r(args, ",", &save); tok; tok = strtok_r(NULL, ",", &save)) {+ tok = trim(tok);+ if (idx >= n || !parse_arg(tok, &out[idx++])) {+ say(error, size, "`%s' is not inN, nN, texN, sampN, bufN or gridpos", tok);+ return false;+ }+ }+ node->num_args = idx;+ node->args = out;+ return true;+}++const struct air_stitch_graph *+air_stitch_parse(void *mem_ctx, const char *text, char *error, size_t size)+{+ char *copy = ralloc_strdup(mem_ctx, text);+ struct air_stitch_graph *g = rzalloc(mem_ctx, struct air_stitch_graph);+ g->name = "stitched";+ g->output_node = ~0u;++ char *arrow = strstr(copy, "=>");+ if (arrow) {+ *arrow = '\0';+ char *tok = trim(arrow + 2);+ char *e = tok + strlen(tok);+ while (e > tok && e[-1] == ';')+ *--e = '\0';+ struct air_stitch_arg which;+ if (!parse_arg(trim(tok), &which) || which.kind != AIR_STITCH_NODE) {+ say(error, size, "`=> %s' is not `=> nN'", tok);+ return NULL;+ }+ g->output_node = which.index;+ }++ unsigned count = 1;+ for (const char *p = copy; *p; p++)+ count += *p == ';';+ struct air_stitch_node *nodes = rzalloc_array(mem_ctx, struct air_stitch_node, count);+ unsigned n = 0;+ char *save = NULL;+ for (char *tok = strtok_r(copy, ";", &save); tok; tok = strtok_r(NULL, ";", &save)) {+ if (!*trim(tok))+ continue;+ if (n >= count || !parse_node(mem_ctx, tok, &nodes[n++], error, size))+ return NULL;+ }+ if (!n) {+ say(error, size, "no nodes");+ return NULL;+ }+ g->num_nodes = n;+ g->nodes = nodes;+ if (g->output_node == ~0u)+ g->output_node = n - 1;+ if (g->output_node >= n) {+ say(error, size, "the output node is n%u of %u nodes", g->output_node, n);+ return NULL;+ }+ return g;+}++unsigned+air_stitch_functions_of(void *mem_ctx, const struct air_stitch_graph *g,+ const void *data, size_t size,+ struct air_stitch_function **out, char *error, size_t esize)+{+ struct air_metallib lib;+ const char *why = NULL;+ if (!air_metallib_parse(mem_ctx, data, size, &lib, &why)) {+ say(error, esize, "the container: %s", why ? why : "?");+ return 0;+ }++ struct air_stitch_function *fns =+ rzalloc_array(mem_ctx, struct air_stitch_function, MAX2(g->num_nodes, 1));+ unsigned num = 0;+ for (unsigned i = 0; i < g->num_nodes; i++) {+ const char *name = g->nodes[i].function;+ bool have = false;+ for (unsigned k = 0; k < num; k++)+ have |= !strcmp(fns[k].name, name);+ if (have)+ continue; /* two nodes calling one function need it once */++ const struct air_metallib_function *found = NULL;+ for (unsigned k = 0; k < lib.num_functions; k++)+ if (!strcmp(lib.functions[k].name, name))+ found = &lib.functions[k];+ if (!found) {+ say(error, esize, "%s is not a function of this library", name);+ return 0;+ }+ fns[num++] = (struct air_stitch_function){+ found->name, found->bitcode, found->bitcode_size,+ };+ }+ *out = fns;+ return num;+}diff --git a/tools/mesa/air/air_to_nir.c b/tools/mesa/air/air_to_nir.cindex 26478c5..11a8db3 100644--- a/tools/mesa/air/air_to_nir.c+++ b/tools/mesa/air/air_to_nir.c@@ -16,6 +16,7 @@#include <stdio.h>#include <string.h>+#include "nir_constant_expressions.h"#include "util/hash_table.h"#include "util/u_math.h"@@ -25,12 +26,348 @@ air_error(struct air_ctx *c, const char *fmt, ...)if (!c->error[0]) {va_list ap;va_start(ap, fmt);- vsnprintf(c->error, sizeof(c->error), fmt, ap);+ int n = vsnprintf(c->error, sizeof(c->error), fmt, ap);va_end(ap);++ /* A REFUSAL THAT WAS CUT SAYS SO.+ *+ * This buffer is 256 bytes (air_private.h) and vsnprintf stops there in+ * silence, so a longer sentence used to arrive mid-word with nothing to+ * show for the rest -- which is this project's cardinal defect wearing a+ * disguise: a message that reads as complete and is not. It is not+ * hypothetical. The tile-shading refusal was written long enough to name+ * both of its walls and every report printed it ending at+ * "-featureProfile", so the half that said which wall decides was the+ * half that vanished, and nothing anywhere said a word was missing.+ *+ * The marker carries the length vsnprintf WANTED, because that is the+ * number a reader can act on: it says how much is gone and therefore+ * whether the sentence should be shortened or the buffer grown. Widening+ * the buffer is the other fix and is not this one -- air_ctx is in a+ * header every object of the frontend includes, and a marker costs+ * nothing to anyone who is not already over the limit.+ */+ if (n >= (int)sizeof(c->error)) {+ char *tail = c->error + sizeof(c->error) - 20;+ snprintf(tail, 20, " ...[cut, %d]", n);+ }}return false;}+void+air_note(struct air_ctx *c, const char *fmt, ...)+{+ struct air_shader_info *info = c->info;+ va_list ap;+ va_start(ap, fmt);+ char *text = ralloc_vasprintf(c->mem_ctx, fmt, ap);+ va_end(ap);+ /* the same note twice says nothing twice: two resources in one group, or a+ * callee translated for two call sites, would each add one */+ for (unsigned i = 0; i < info->num_notes; i++)+ if (!strcmp(info->notes[i], text))+ return;+ info->notes = reralloc(c->mem_ctx, info->notes, const char *, info->num_notes + 1);+ info->notes[info->num_notes++] = text;+}++/* -- constants the builder did not fold ---------------------------------- */++/* Is this def a constant expression, and what is its value?+ *+ * nir_builder does NOT fold: nir_build_alu inserts the instruction whatever+ * its sources are (nir_builder.c), so `icmp eq i8 %c, 0` with %c an immediate+ * is an ALU instruction here and nir_src_is_const says no. That matters+ * because the two decisions this is for -- which edge of a branch exists, and+ * which side of a `select` between two pointers is taken -- are made while+ * translating, long before nir_opt_constant_folding runs, and a branch left in+ * is a phi of pointers left unresolved.+ *+ * THE SEMANTICS ARE NIR'S OWN. Every operation is evaluated by+ * nir_eval_const_opcode with this shader's float controls, which is the table+ * and the mode nir_opt_constant_folding itself uses, so nothing here can+ * disagree with the folder that runs after. What this adds to try_fold_alu is+ * only recursion: that one requires every source to be a load_const, which a+ * chain of an immediate and two operations never is.+ */+static bool+def_const(nir_def *def, nir_const_value out[NIR_MAX_VEC_COMPONENTS],+ unsigned depth, unsigned float_controls)+{+ if (!def || depth == 0 || def->num_components > NIR_MAX_VEC_COMPONENTS)+ return false;++ if (def->parent_instr->type == nir_instr_type_load_const) {+ nir_load_const_instr *lc = nir_instr_as_load_const(def->parent_instr);+ for (unsigned i = 0; i < def->num_components; i++)+ out[i] = lc->value[i];+ return true;+ }+ if (def->parent_instr->type != nir_instr_type_alu)+ return false;++ nir_alu_instr *alu = nir_instr_as_alu(def->parent_instr);+ const nir_op_info *info = &nir_op_infos[alu->op];+ nir_const_value src[NIR_ALU_MAX_INPUTS][NIR_MAX_VEC_COMPONENTS];+ nir_const_value *srcs[NIR_ALU_MAX_INPUTS];++ /* the unsized-type rule, copied from try_fold_alu rather than reasoned+ * about: an op whose output type carries no size takes it from the first+ * source that carries none either+ */+ unsigned bit_size = 0;+ if (!nir_alu_type_get_type_size(info->output_type))+ bit_size = alu->def.bit_size;++ for (unsigned i = 0; i < info->num_inputs; i++) {+ nir_const_value v[NIR_MAX_VEC_COMPONENTS];+ if (!def_const(alu->src[i].src.ssa, v, depth - 1, float_controls))+ return false;+ if (bit_size == 0 &&+ !nir_alu_type_get_type_size(info->input_types[i]))+ bit_size = alu->src[i].src.ssa->bit_size;+ for (unsigned j = 0; j < nir_ssa_alu_instr_src_components(alu, i); j++)+ src[i][j] = v[alu->src[i].swizzle[j]];+ srcs[i] = src[i];+ }+ if (bit_size == 0)+ bit_size = 32;++ memset(out, 0, sizeof(nir_const_value) * NIR_MAX_VEC_COMPONENTS);+ nir_eval_const_opcode(alu->op, out, def->num_components, bit_size, srcs,+ float_controls);+ return true;+}++/* A one-component condition that is decided already. 16 is the recursion+ * limit and not a judgement about shaders: the chains this is for are three+ * instructions long (a load folded to an immediate, an icmp, the branch), and+ * a limit is here so that a pathological module costs a refusal to fold and+ * not a stack.+ */+/* The same question for an integer, which a switch asks of its selector and of+ * every one of its case values. */+static bool+uint_const(struct air_ctx *c, nir_def *def, uint64_t *val)+{+ nir_const_value v[NIR_MAX_VEC_COMPONENTS];+ if (!def || def->num_components != 1)+ return false;+ if (!def_const(def, v, 16, c->nir->info.float_controls_execution_mode))+ return false;+ *val = nir_const_value_as_uint(v[0], def->bit_size);+ return true;+}++static bool+cond_const(struct air_ctx *c, nir_def *def, bool *val)+{+ nir_const_value v[NIR_MAX_VEC_COMPONENTS];+ if (!def || def->num_components != 1)+ return false;+ if (!def_const(def, v, 16, c->nir->info.float_controls_execution_mode))+ return false;+ *val = nir_const_value_as_bool(v[0], def->bit_size);+ return true;+}++/* Two things the module-scope half of this file decides and the INSTRUCTION half+ * needs: plan_global_folding's verdict for one global, and the name and+ * required-ness of a function constant by index. Both are declared here because+ * air_load_ptr and AIR_OP_{LOAD,STORE} come first in this file and the planner+ * last, and a missing prototype in C is an implicit int that compiles with a+ * warning and returns a pointer truncated to one.+ */+static struct air_global_plan *foldable_global_plan(struct air_ctx *c,+ const struct air_value *v);+static const char *function_constant_decl(struct air_ctx *c, unsigned index,+ bool *required);++/* -- translation-time slots ---------------------------------------------- */++/* Does this type hold a pointer anywhere inside it? A slot exists for exactly+ * the values that memory cannot hold, and that is what makes one necessary.+ */+static bool+type_has_pointer(const struct air_type *t, unsigned depth)+{+ if (!t || depth == 0)+ return false;+ switch (t->kind) {+ case AIR_TYPE_POINTER:+ return true;+ case AIR_TYPE_ARRAY:+ case AIR_TYPE_VECTOR:+ return type_has_pointer(t->elem, depth - 1);+ case AIR_TYPE_STRUCT:+ for (unsigned i = 0; i < t->num_members; i++)+ if (type_has_pointer(t->members[i], depth - 1))+ return true;+ return false;+ default:+ return false;+ }+}++static unsigned+new_slot(struct air_ctx *c, const struct air_type *type, const struct air_value *init)+{+ c->slots = reralloc(c->mem_ctx, c->slots, struct air_slot, c->num_slots + 1);+ c->slots[c->num_slots] = (struct air_slot) { .type = type, .init = init };+ return c->num_slots++;+}++/* The value at byte offset 0 of a slot, for a load that wants `want`.+ *+ * Offset 0 of a struct or an array is offset 0 of its first element, and again+ * of that element's first element, so descending is the whole of the+ * arithmetic -- but only while the caller is asking for something that is not+ * itself the aggregate. A load of the aggregate gets the aggregate.+ */+static struct air_val *+slot_at_zero(struct air_val *v, const struct air_type *want)+{+ bool want_agg = want && (want->kind == AIR_TYPE_STRUCT ||+ want->kind == AIR_TYPE_ARRAY);+ while (!want_agg && v && v->kind == AIR_VAL_AGG && v->num_elems)+ v = v->elems[0];+ return v;+}++/* A pointer that IS a slot's base: class SLOT and offset zero. A slot holds a+ * tree of values and not bytes, so an offset into one is not something to+ * resolve -- `gep_into` may well have produced one (a getelementptr past the+ * first member), and the honest answer to that is the refusal at the load.+ */+static bool+slot_base(struct air_ctx *c, const struct air_val *v, unsigned *index)+{+ if (!v || v->kind != AIR_VAL_PTR || v->ptr.class != AIR_PTR_SLOT)+ return false;+ if (v->ptr.offset &&+ (!nir_src_is_const(nir_src_for_ssa(v->ptr.offset)) ||+ nir_src_as_uint(nir_src_for_ssa(v->ptr.offset)) != 0))+ return false;+ if (v->ptr.binding >= c->num_slots)+ return false;+ *index = v->ptr.binding;+ return true;+}++static struct air_val *air_val_agg(struct air_ctx *c, unsigned n);++/* A pointer into a slot at a byte offset the translator knows: class SLOT, a+ * binding in range, and an offset that is constant or absent.+ */+static bool+slot_at_const(struct air_ctx *c, const struct air_val *v, unsigned *index,+ uint64_t *off)+{+ if (!v || v->kind != AIR_VAL_PTR || v->ptr.class != AIR_PTR_SLOT)+ return false;+ if (v->ptr.binding >= c->num_slots)+ return false;+ if (v->ptr.offset) {+ if (!nir_src_is_const(nir_src_for_ssa(v->ptr.offset)))+ return false;+ *off = nir_src_as_uint(nir_src_for_ssa(v->ptr.offset));+ } else {+ *off = 0;+ }+ *index = v->ptr.binding;+ return true;+}++/* THE CELL OF A SLOT'S TREE THAT A BYTE OFFSET AND A TYPE NAME TOGETHER.+ *+ * A slot holds a whole air_val because a pointer in this frontend is a+ * resolution and not bytes (air_private.h, AIR_PTR_SLOT), and until now only+ * offset zero could reach one: a `getelementptr` past the first member+ * produced a byte offset, and a tree of values has no bytes for it to mean.+ * That is still true of a byte offset ALONE, and it is why this takes a type+ * as well. Offset 4 of `{ {ptr}, {ptr}, ptr, <2 x float> }` is the second+ * member, the second member's first member, and nothing else -- but only+ * because the store says it is storing a pointer. The same offset with+ * `<2 x float>` would land nowhere and this returns NULL.+ *+ * So the rule is: descend while the type at hand is not the type asked for,+ * taking at each level the one member or element whose byte range contains the+ * remaining offset, and refuse unless the descent ends exactly on the asked+ * type at offset zero. Types are interned per module, so `==` is the whole+ * comparison. A partial overlap -- half of a pointer, two members at once --+ * never matches and is refused by name at the caller, which is the behaviour+ * this replaces and not a behaviour it adds.+ *+ * What the four `disparity_refinement_*` of CIPortraitBlurStitchableV3 look+ * like, which is what this exists for: CoreImage hands a stitched function a+ * `coreimage::Sampler` flattened into four arguments, and the function's first+ * act is to `alloca` the struct back and store each argument into its field.+ * Three of those fields are a texture, a sampler and a constant buffer -- the+ * things a slot exists to hold -- and the fourth is a float2, which a slot can+ * hold too and could not reach.+ *+ * Levels that nothing has written yet are made here rather than refused, for+ * the reason agg_slot gives: an aggregate whose every field is NONE *is* the+ * undef of that type, so building one changes no value.+ */+static struct air_val **+slot_cell(struct air_ctx *c, unsigned s, uint64_t off, const struct air_type *want)+{+ struct air_val **cell = &c->slots[s].val;+ const struct air_type *t = c->slots[s].type;++ /* A struct nested 32 deep is not a thing Metal's compiler emits; the bound+ * is here so that a cyclic or malformed type cannot spin.+ */+ for (unsigned depth = 0; depth < 32; depth++) {+ if (t && t == want && off == 0)+ return cell;+ if (!t)+ return NULL;++ unsigned index, count;+ const struct air_type *et;+ if (t->kind == AIR_TYPE_STRUCT) {+ count = t->num_members;+ index = count;+ uint64_t base = 0;+ for (unsigned i = 0; i < count; i++) {+ uint64_t mo = air_struct_member_offset(c->m, t, i);+ uint64_t ms = air_type_alloc_size(c->m, t->members[i]);+ if (off >= mo && off < mo + ms) {+ index = i;+ base = mo;+ break;+ }+ }+ if (index == count)+ return NULL;+ et = t->members[index];+ off -= base;+ } else if (t->kind == AIR_TYPE_ARRAY) {+ uint64_t es = air_type_alloc_size(c->m, t->elem);+ if (!es || off / es >= t->count)+ return NULL;+ count = t->count > UINT_MAX ? UINT_MAX : (unsigned)t->count;+ index = (unsigned)(off / es);+ et = t->elem;+ off -= (uint64_t)index * es;+ } else {+ return NULL; /* a scalar, a vector or a pointer: no further in */+ }++ if (!*cell)+ *cell = air_val_agg(c, count);+ if (!*cell || (*cell)->kind != AIR_VAL_AGG || index >= (*cell)->num_elems)+ return NULL;+ cell = &(*cell)->elems[index];+ t = et;+ }+ return NULL;+}++/* -- values ------------------------------------------------------------- */struct air_val *@@ -51,6 +388,53 @@ air_val_ptr(struct air_ctx *c, struct air_ptr ptr)return v;}+static struct air_val *+air_val_ptr_select(struct air_ctx *c, nir_def *cond,+ struct air_val *a, struct air_val *b)+{+ struct air_val *v = rzalloc(c->mem_ctx, struct air_val);+ v->kind = AIR_VAL_PTR_SELECT;+ v->def = cond;+ v->num_elems = 2;+ v->elems = rzalloc_array(c->mem_ctx, struct air_val *, 2);+ v->elems[0] = a;+ v->elems[1] = b;+ return v;+}++/* Is this value a pointer in any of the three senses? */+static bool+val_is_pointerish(const struct air_val *v)+{+ return v && (v->kind == AIR_VAL_PTR || v->kind == AIR_VAL_PTR_SELECT);+}++/* Two resolutions that are the same resolution.+ *+ * This is what lets a `phi` or a `select` between pointers be answered without+ * choosing anything at run time, and it is needed rather than merely tidy:+ * LLVM duplicates a `getelementptr` onto two paths and then phis the copies+ * back together, so 207 of the pointer phis in Resident Evil 4's library have+ * two operands that name ONE member of ONE argument buffer. A pointer whose+ * byte offset is not a constant is compared by the def itself -- the same def+ * is the same offset, and two different defs cannot be proved equal here.+ */+static bool+same_ptr(const struct air_ptr *a, const struct air_ptr *b)+{+ if (a->class != b->class || a->binding != b->binding || a->is_image != b->is_image)+ return false;+ if (a->offset == b->offset)+ return true;+ if (!a->offset || !b->offset)+ return false;+ if (!nir_src_is_const(nir_src_for_ssa(a->offset)) ||+ !nir_src_is_const(nir_src_for_ssa(b->offset)))+ return false;+ return nir_src_as_uint(nir_src_for_ssa(a->offset)) ==+ nir_src_as_uint(nir_src_for_ssa(b->offset));+}+static struct air_val *air_val_agg(struct air_ctx *c, unsigned n){@@ -67,11 +451,129 @@ air_set(struct air_ctx *c, const struct air_value *v, struct air_val *val)_mesa_hash_table_insert(c->values, v, val);}+/* An integer of a width NIR does not have, and how wide it is.+ *+ * Metal's own compiler makes them. A function constant holding a 3-bit field+ * is narrowed with `trunc i32 %x to i3` and then switched on, which is how+ * four of RenderBox's shaders pick a blend mode; `i2` occurs the same way.+ * NIR has 1, 8, 16, 32 and 64 bits and nothing between, so such a value is+ * carried here in the next width up with the low N bits significant and+ * everything above them ZERO -- canonical form.+ *+ * Canonical form makes exactly five things exact, and that is the whole list:+ * a truncation INTO the type (mask the low N bits), a truncation or a+ * zero-extension OUT of it, an unsigned conversion to float, an unsigned or+ * equality comparison, and a switch -- which is equalities. A signed+ * comparison, a sign extension, an addition, a multiplication and a shift all+ * read or write the bits above N and would be wrong. narrow_int_gate refuses+ * everything that is not on the list, by width and by opcode, because a wrong+ * answer that nothing says is wrong is the one failure this frontend must not+ * have.+ */+static unsigned+narrow_int_bits(const struct air_type *t)+{+ if (!t)+ return 0;+ const struct air_type *s = air_type_scalar(t);+ if (!s || s->kind != AIR_TYPE_INT)+ return 0;+ unsigned b = s->bits;+ if (!b || b > 64 || b == 1 || b == 8 || b == 16 || b == 32 || b == 64)+ return 0;+ return b;+}++/* The NIR width such a value is carried in: 32 bits, or 64 for the widths+ * above it.+ *+ * NOT the next width up, which was the first choice and is worse for two+ * reasons measured rather than guessed. An `i3` carried in eight bits makes+ * `trunc i32 %x to i3` a `u2u8` followed by a mask, and nir_to_tgsi has no+ * u2u8 at all -- softpipe aborts on it ("Unknown NIR opcode: u2u8"), which+ * removes the one reference on this machine that RUNS the shader rather than+ * folding it. At 32 bits the truncation of a 32-bit word is the mask alone,+ * which is fewer instructions as well as portable. Canonical form is exactly+ * as exact in 32 bits as in 8: the low N bits significant, the rest zero.+ */+static unsigned+narrow_promoted_bits(unsigned n)+{+ return n <= 32 ? 32 : 64;+}++static uint64_t+narrow_mask(unsigned n)+{+ return n >= 64 ? ~(uint64_t)0 : ((uint64_t)1 << n) - 1;+}++/* The width a value of this type is CARRIED in, promoting a width NIR has no+ * register for instead of refusing it.+ *+ * Only for the places narrow_int_gate governs the uses of -- a constant, a+ * truncation into the type, a phi's register. air_nir_bit_size stays the+ * default everywhere else and stays a refusal, because a type that needs a+ * register whose upper bits MEAN something has no NIR width and promoting it+ * would be wrong rather than inconvenient.+ */+static unsigned+air_narrow_bits(struct air_ctx *c, const struct air_type *t)+{+ unsigned n = narrow_int_bits(t);+ return n ? narrow_promoted_bits(n) : air_nir_bit_size(c, t);+}+unsignedair_nir_bit_size(struct air_ctx *c, const struct air_type *t){unsigned bits = air_type_bit_size(air_type_scalar(t));- /* NIR has no i1 in SSA: booleans are 1-bit defs, which it does have */+ /* NIR has no i1 in SSA: booleans are 1-bit defs, which it does have.+ *+ * It has no 2- or 4-bit integer at all, though, and that is not a+ * hypothetical: an HLSL bit field reaches AIR as `i2` or `i4` through+ * SPIRV-Cross, and every NIR builder handed one ASSERTS instead of+ * refusing -- "Invalid Boolean size", "Invalid bit size" and "Invalid nir+ * alu bit size" between them aborted 495 of Resident Evil 4's modules the+ * first time the argument-buffer interface let them get as far as their+ * bodies. An abort is not a refusal: it says nothing a report can count and+ * it takes the run with it. So the width is named here, once, and 32 is+ * returned so that the instruction under construction stays well formed+ * while the error just set discards the shader.+ */+ if (bits != 1 && bits != 8 && bits != 16 && bits != 32 && bits != 64) {+ /* Zero is a different fact wearing the same symptom, and saying so in one+ * message made 79 modules read as carrying a 0-bit integer when what they+ * carry is a clip-distance ARRAY as a shader output: air_type_bit_size has+ * no answer for an aggregate, and this used to return it unexamined, so+ * `[4 x float]` became a one-component float output and four clip planes+ * became one with nothing in any log. A width NIR cannot hold and a type+ * that has no single width are both refusals, and they are not the same+ * refusal.+ */+ if (!bits)+ air_error(c, "instruction: %s where a number belongs, which has no single "+ "NIR width",+ t->kind == AIR_TYPE_ARRAY ? "an array" :+ t->kind == AIR_TYPE_STRUCT ? "a struct" :+ t->kind == AIR_TYPE_POINTER ? "a pointer" : "a value of no size");+ else+ /* Reaching HERE, and not narrow_int_gate, means the width turned up+ * somewhere that needs a register of its own -- a function parameter,+ * a shader output, a phi, a value stored to memory -- rather than in+ * an instruction whose low N bits are all it reads. Canonical form+ * (narrow_int_bits) does not help there: the bits above N would have+ * to mean something, and NIR has no width in which they do. So the+ * sentence names both halves, because "a width NIR does not have" on+ * its own sent the reader looking for a missing case in the builder+ * when what is missing is a representation.+ */+ air_error(c, "instruction: a %u-bit integer where NIR needs a register of its "+ "own, a width it has no register for; it is carried in the next "+ "width up only where nothing reads above bit %u",+ bits, bits - 1);+ return 32;+ }return bits;}@@ -81,6 +583,35 @@ air_nir_components(const struct air_type *t)return t->kind == AIR_TYPE_VECTOR ? t->count : 1;}+/* The glsl type a NIR function parameter needs to carry a value of this AIR+ * type, matching what air_nir_bit_size and air_nir_components say the def will+ * be. NIR tracks a value's real type by use, so the only thing that has to be+ * right here is the shape.+ */+static const struct glsl_type *+param_glsl_type(struct air_ctx *c, const struct air_type *t)+{+ unsigned bits = air_nir_bit_size(c, t);+ return glsl_vector_type(bits == 16 ? GLSL_TYPE_FLOAT16 :+ bits == 64 ? GLSL_TYPE_UINT64 : GLSL_TYPE_UINT,+ air_nir_components(t));+}++/* Whether a struct member is a number this frontend can hand through a+ * function parameter -- a scalar or a vector of one. Everything else takes no+ * parameter: an array, a nested struct, a pointer. In practice the one that+ * occurs is the tail padding Metal's compiler puts on a vertex-output struct+ * (`[8 x i8]`), which the shader never writes and the caller never reads.+ */+static bool+ret_member_is_number(const struct air_type *m)+{+ if (m->kind != AIR_TYPE_VECTOR && m->kind != AIR_TYPE_INT && !air_type_is_float(m))+ return false;+ const struct air_type *sc = air_type_scalar(m);+ return sc->kind == AIR_TYPE_INT || air_type_is_float(sc);+}+static struct air_val *air_constant_val(struct air_ctx *c, const struct air_constant *k);struct air_val *@@ -91,7 +622,32 @@ air_get(struct air_ctx *c, const struct air_value *v)return e->data;const struct air_constant *k = air_value_as_constant(v);if (k) {+ /* A constant is built once and then reused from wherever it is asked+ * for, so it must be somewhere that dominates all of those places. A+ * constant has no operands, so the top of the entry block always will.+ *+ * Built at the cursor of its first use instead, a constant that LLVM+ * shares between a conditional block and a block after it is a use that+ * its def does not dominate -- and nothing complains, because+ * nir_lower_goto_ifs runs nir_repair_ssa, which repairs such a use into+ * a phi whose other edge is undef. That is how `phi i16 [ %30, %27 ],+ * [ -1, %22 ]` in MetalFaceMask_AddSeedPoint became `phi b3: %92,+ * b4: %0 (undefined)`: the -1 was the same constant as the one the+ * compare after the branch uses, so the store on the entry edge pointed+ * at a def inside the branch (docs/air.md section 5).+ */+ bool outer = !c->in_const;+ nir_cursor use = c->b.cursor;+ if (outer) {+ c->in_const = true;+ c->b.cursor = c->const_cursor;+ }struct air_val *val = air_constant_val(c, k);+ if (outer) {+ c->const_cursor = c->b.cursor;+ c->b.cursor = use;+ c->in_const = false;+ }if (val)air_set(c, v, val);return val;@@ -113,7 +669,8 @@ air_get_ssa(struct air_ctx *c, const struct air_value *v)return NULL;if (val->kind != AIR_VAL_SSA) {air_error(c, "instruction: %s where a number belongs",- val->kind == AIR_VAL_PTR ? "a pointer" : "an aggregate");+ val->kind == AIR_VAL_PTR ? "a pointer" :+ val->kind == AIR_VAL_PTR_SELECT ? "a chosen pointer" : "an aggregate");return NULL;}return val->def;@@ -127,6 +684,28 @@ scalar_constant(struct air_ctx *c, const struct air_type *t, uint64_t bits)unsigned n = air_type_bit_size(t);if (t->kind == AIR_TYPE_INT && t->bits == 1)return nir_imm_bool(&c->b, bits & 1);+ unsigned narrow = narrow_int_bits(t);+ if (narrow) {+ /* Canonical form (narrow_int_bits): the low N bits, zero above them.+ *+ * The mask here is the SECOND guard and not the first. The reader+ * already truncates an integer constant to its type's width+ * (truncate_to in air_module.c), so `i3 -4` arrives as 4 and not as a+ * sign-extended 0xfffc; removing this mask and rebuilding changes no+ * number in tools/air/callargs.ll, which was checked rather than+ * assumed. What makes the switch's case live is the mask on the+ * TRUNCATION into the type, in translate_cast, and that one is+ * load-bearing: without it `trunc i32 %w to i3` keeps eight bits, no+ * case matches and the default is taken.+ *+ * It stays because the invariant belongs to both halves: a constant of+ * this type must be canonical however it was built, and AIR_CONST_DATA+ * carries its words from the record without passing through+ * truncate_to.+ */+ return nir_imm_intN_t(&c->b, bits & narrow_mask(narrow),+ narrow_promoted_bits(narrow));+ }if (air_type_is_float(t)) {switch (n) {case 16: return nir_imm_intN_t(&c->b, bits, 16);@@ -177,7 +756,7 @@ air_constant_val(struct air_ctx *c, const struct air_constant *k)}return v;}- return air_val_ssa(c, nir_imm_zero(&c->b, air_nir_components(t), air_nir_bit_size(c, t)));+ return air_val_ssa(c, nir_imm_zero(&c->b, air_nir_components(t), air_narrow_bits(c, t)));case AIR_CONST_UNDEF:case AIR_CONST_POISON:@@ -191,7 +770,7 @@ air_constant_val(struct air_ctx *c, const struct air_constant *k)struct air_ptr p = { .class = AIR_PTR_NONE, .pointee = t->elem };return air_val_ptr(c, p);}- return air_val_ssa(c, nir_undef(&c->b, air_nir_components(t), air_nir_bit_size(c, t)));+ return air_val_ssa(c, nir_undef(&c->b, air_nir_components(t), air_narrow_bits(c, t)));case AIR_CONST_AGGREGATE: {if (t->kind == AIR_TYPE_VECTOR) {@@ -235,6 +814,56 @@ air_constant_val(struct air_ctx *c, const struct air_constant *k)if (k->expr_op == AIR_OP_CAST && k->expr_sub == AIR_CAST_BITCAST &&k->base.type->kind == AIR_TYPE_POINTER)return air_get(c, k->operands[0]);+ /* ... and a constant getelementptr, which the GL-on-Metal vertex shaders+ * use to name the first element of a module array+ * (`getelementptr inbounds ([2 x <4 x float>], ptr @g, i64 0, i64 0)`).+ * It is the instruction form with every index a constant, so the whole+ * walk is done here in integers and the result is the base pointer with+ * a constant byte offset and a narrowed pointee.+ */+ if (k->expr_op == AIR_OP_GEP) {+ if (k->num_operands < 2 || !k->src_type)+ return air_error(c, "instruction: a constant getelementptr with no base "+ "or no source type"), NULL;+ struct air_val *base = air_get(c, k->operands[0]);+ if (!base)+ return NULL;+ if (base->kind != AIR_VAL_PTR)+ return air_error(c, "instruction: a constant getelementptr on something "+ "that is not a pointer"), NULL;+ struct air_ptr p = base->ptr;+ const struct air_type *t = k->src_type;+ uint64_t off = 0, idx = 0;+ if (!air_const_uint(k->operands[1], &idx))+ return air_error(c, "instruction: a constant getelementptr whose first "+ "index is not a constant"), NULL;+ off += idx * air_type_alloc_size(c->m, t);+ for (unsigned i = 2; i < k->num_operands; i++) {+ if (!air_const_uint(k->operands[i], &idx))+ return air_error(c, "instruction: a constant getelementptr with an index "+ "that is not a constant"), NULL;+ if (t->kind == AIR_TYPE_STRUCT) {+ if (idx >= t->num_members)+ return air_error(c, "instruction: a constant getelementptr to struct "+ "field %u of %u", (unsigned)idx,+ t->num_members), NULL;+ off += air_struct_member_offset(c->m, t, idx);+ t = t->members[idx];+ } else if (t->kind == AIR_TYPE_ARRAY || t->kind == AIR_TYPE_VECTOR) {+ off += idx * (t->kind == AIR_TYPE_ARRAY+ ? air_type_alloc_size(c->m, t->elem)+ : MAX2(air_type_bit_size(t->elem) / 8, 1));+ t = t->elem;+ } else {+ return air_error(c, "instruction: a constant getelementptr into a "+ "scalar"), NULL;+ }+ }+ p.pointee = t;+ if (off)+ p = air_ptr_add(c, p, nir_imm_int(&c->b, off));+ return air_val_ptr(c, p);+ }return air_error(c, "instruction: constant expression %u", k->expr_op), NULL;}return air_error(c, "instruction: constant kind %u", k->ckind), NULL;@@ -247,7 +876,40 @@ air_ptr_add(struct air_ctx *c, struct air_ptr ptr, nir_def *bytes){if (!bytes)return ptr;- ptr.offset = ptr.offset ? nir_iadd(&c->b, ptr.offset, bytes) : bytes;+ if (!ptr.offset) {+ ptr.offset = bytes;+ return ptr;+ }+ /* Two constant offsets become one constant, and that is a correctness fix+ * rather than a tidiness.+ *+ * A cached constant is built in the const region at the top of the entry+ * block (air_get), while a module-scope variable's byte offset is built+ * where materialise_globals runs, which is BELOW that region. A constant+ * `getelementptr` into a threadgroup array -- `load i32, ptr addrspace(3)+ * getelementptr([16 x i32], ptr @tbl, i64 0, i64 9)`, which is how+ * SPIRV-Cross reads a fixed slot of one -- is a constant whose value was+ * therefore `iadd %globaloffset, 36` sitting above the def of+ * %globaloffset: a use its def does not dominate, which nir_validate+ * rejects with "src->_parent & SRC_TAG_SEEN". 115 of Resident Evil 4's+ * modules are that shape, and every one of them ABORTED the translator the+ * first time the argument-buffer interface let their bodies be reached.+ *+ * Folding the pair removes the reference: a global's offset is always a+ * constant here, so the whole chain is one immediate built in the const+ * region with no operand at all. The latent ordering hazard it stands on is+ * written down in docs/air.md rather than left to be rediscovered.+ */+ if (nir_src_is_const(nir_src_for_ssa(ptr.offset)) &&+ nir_src_is_const(nir_src_for_ssa(bytes)) &&+ ptr.offset->bit_size == 32 && bytes->bit_size == 32 &&+ ptr.offset->num_components == 1 && bytes->num_components == 1) {+ uint32_t sum = (uint32_t)nir_src_as_uint(nir_src_for_ssa(ptr.offset)) ++ (uint32_t)nir_src_as_uint(nir_src_for_ssa(bytes));+ ptr.offset = nir_imm_int(&c->b, sum);+ return ptr;+ }+ ptr.offset = nir_iadd(&c->b, ptr.offset, bytes);return ptr;}@@ -301,14 +963,122 @@ air_load_ptr(struct air_ctx *c, struct air_ptr ptr, const struct air_type *type,v = &c->options->constants[i];if (!v)return air_error(c, "interface: function constant %u has no value", ptr.binding), NULL;- if (comps != 1)- return air_error(c, "interface: a function constant of %u components", comps), NULL;- res = nir_imm_intN_t(&c->b, v->bits, as_bool ? 8 : bits);+ /* READ, AND NOBODY SUPPLIED IT. air_module_function_constants registers+ * every constant a module DECLARES, with bits = 0 and supplied = false+ * (nir_air.h), so a load here answers zero whether or not a pipeline ever+ * chose one -- and this frontend now SPECIALISES on that zero+ * (plan_global_folding), which turns it from a number in an expression+ * into which blocks exist.+ *+ * air_metal.c says this on a refusal, with a count and two names; this is+ * the other half, and it is here rather than there because it lands in+ * the corpus report and so becomes countable instead of anecdotal. One+ * note per constant per module (air_note drops a repeat), so the report's+ * row for a name is the number of modules that read it unsupplied.+ *+ * Required-ness is in the sentence because it is what decides whether the+ * zero is a defect or a legal default: a declaration ending `i1 true` is+ * one Metal obliges the pipeline to supply, and `i1 false` is one the+ * module is supposed to ask about with air.is_function_constant_defined+ * first. The second is ordinary; the first means a pipeline left out+ * something it had to give.+ */+ if (!v->supplied) {+ bool required = false;+ const char *fcname = function_constant_decl(c, ptr.binding, &required);+ /* SHORT ON PURPOSE. 70 of these appear over the corpus, one per+ * constant, and the row that matters is the name beside the number of+ * modules -- a census. Repeating what it means 70 times would bury+ * exactly that, so the consequence is said once in docs/air.md 3 and+ * what is here is the fact.+ */+ air_note(c, "the function constant %s (index %u, %s) is read and no pipeline "+ "supplied a value, so it was taken as zero",+ fcname ? fcname : "(unnamed)", ptr.binding,+ required ? "i1 true, which Metal obliges a pipeline to supply"+ : "i1 false, so absent is legal");+ }+ if (comps == 1) {+ res = nir_imm_intN_t(&c->b, v->bits, as_bool ? 8 : bits);+ break;+ }+ /* A VECTOR function constant. Metal allows one and the GL-on-Metal+ * fragment shaders use two: `uchar3` opcode triples for the texture+ * combiners (`fc_color_opcodes0/1`, declared `<3 x i8>`).+ *+ * air_constant_value carries one 64-bit word per constant, so the+ * components have to be fields of it, and the packing is this frontend's+ * own interface with the runtime rather than something read out of+ * Apple's: component i sits at bit i*width, which is the word a memcpy+ * of the value would produce on this little-endian target. NOTHING HERE+ * MEASURES THAT -- every harness in tools/air supplies zero for every+ * constant, so all components are zero whatever the packing, and the+ * first thing that would tell a wrong packing from a right one is a+ * pipeline that sets one of these two constants.+ */+ if (comps * bits > 64)+ return air_error(c, "interface: a function constant of %u components of %u bits, "+ "which does not fit the one word air_constant_value carries",+ comps, bits), NULL;+ nir_def *fields[NIR_MAX_VEC_COMPONENTS];+ for (unsigned i = 0; i < comps; i++) {+ uint64_t mask = bits >= 64 ? ~0ull : (1ull << bits) - 1;+ fields[i] = nir_imm_intN_t(&c->b, (v->bits >> (i * bits)) & mask,+ as_bool ? 8 : bits);+ }+ res = nir_vec(&c->b, fields, comps);break;}+ case AIR_PTR_ARGBUF:+ /* argbuf_load answers a load through an argument buffer before it gets+ * here, so this is a load that resolution could not make: through a+ * pointer CHOSEN at run time between two sets, which load_through_choice+ * would read from both of. There is nothing to read either way -- the+ * bytes in an argument buffer are the runtime's 8-byte handles.+ */+ return air_error(c, "instruction: a load of bytes out of an argument buffer"), NULL;+ case AIR_PTR_SLOT:+ /* AIR_OP_LOAD answers a whole load of a slot before it gets here, so+ * this is one that could not be answered: through a CHOSEN pointer, one+ * of whose sides is a slot (load_through_choice), or at an offset into+ * one. There is nothing to read either way -- a slot is a tree of values.+ */+ return air_error(c, "instruction: a load of bytes out of a local that holds a "+ "pointer"), NULL;+ case AIR_PTR_NONE:+ /* A LOAD THROUGH A LITERAL `null`, which is undefined behaviour in the+ * source and therefore has exactly one honest translation: undef. Not a+ * value invented for it -- a declaration that there is no value, which+ * is what the source said.+ *+ * AIR_PTR_NONE arises from nothing else. air_constant_val builds it only+ * for AIR_CONST_NULL of a pointer type, so the module literally wrote+ * `null` here; a binding this frontend failed to resolve is a different+ * class and still refuses.+ *+ * Where it comes from is Metal's own contract rather than a mistake.+ * fixed_frag_lph_cph has `%124 = select i1 %123, ptr addrspace(1) null,+ * ptr addrspace(1) %14` on `fc_has_asg_77_weights`, and passes %124 to+ * its asg77 filter -- so when no weights are supplied the pointer IS+ * null, and the path that dereferences it is reachable only through a+ * switch on a FragmentShaderConfig field that comes from a bound buffer.+ * The application is what must not select that filter without supplying+ * weights; this frontend cannot know that it did not, and refusing the+ * module for a path the application is forbidden to take would refuse+ * the shader QuartzCore asks this driver for more than any other.+ *+ * It leaves a note, because a shader that reads undef is a fact a reader+ * of the report should be able to count.+ */+ air_note(c, "a load through a literal `null` pointer is undef: the module has a "+ "path that dereferences null, Metal's contract is that the pipeline "+ "must not take it (in the GL-on-Metal uber shaders it is a filter "+ "whose weights buffer was not supplied), and undef is what the source "+ "says rather than a value chosen here");+ return nir_undef(&c->b, comps, bits);default:return air_error(c, "instruction: a load through a %s pointer",- ptr.class == AIR_PTR_NONE ? "null" : "texture or sampler"), NULL;+ "texture or sampler"), NULL;}return as_bool ? nir_ine_imm(&c->b, res, 0) : res;}@@ -340,6 +1110,17 @@ air_store_ptr(struct air_ctx *c, struct air_ptr ptr, nir_def *value, unsigned alcase AIR_PTR_UBO:case AIR_PTR_CONSTANT_DATA:return air_error(c, "instruction: a store into constant memory");+ case AIR_PTR_ARGBUF:+ /* Writing a descriptor from inside the shader. The handle it would write+ * is the runtime's, and this frontend has resolved the set's members to+ * bindings rather than to addresses, so there is nothing to write.+ */+ return air_error(c, "instruction: a store into an argument buffer");+ case AIR_PTR_SLOT:+ /* as in air_load_ptr: a whole store into a slot is answered by+ * AIR_OP_STORE, so this one is at an offset or through a chosen pointer */+ return air_error(c, "instruction: a store of bytes into a local that holds a "+ "pointer");default:return air_error(c, "instruction: a store through a %s pointer",ptr.class == AIR_PTR_NONE ? "null" : "texture or sampler");@@ -384,6 +1165,20 @@ md_key_int(const struct air_md *node, const char *key, uint64_t *out)return i >= 0 && md_int(node, i + 1, out);}+/* the operand after the key, when it is a nested node: the shape Apple uses for+ * a key whose value is itself a list, as `air.function_constant` is+ * (`!{ptr @__metal_implicit_fc_pred_6, !"bool", !"vtx_attr_has_texcoords"}`)+ */+static const struct air_md *+md_key_node(const struct air_md *node, const char *key)+{+ int i = md_key(node, key);+ if (i < 0 || (unsigned)(i + 1) >= node->num_ops || !node->ops[i + 1] ||+ node->ops[i + 1]->kind != AIR_MD_NODE)+ return NULL;+ return node->ops[i + 1];+}+static const char *md_key_str(const struct air_md *node, const char *key){@@ -502,6 +1297,7 @@ enum air_sysval {AIR_SV_THREAD_POS_GRID, AIR_SV_THREAD_POS_TG, AIR_SV_TG_POS_GRID,AIR_SV_THREADS_PER_TG, AIR_SV_TGS_PER_GRID, AIR_SV_THREAD_INDEX_TG,AIR_SV_THREADS_PER_GRID, AIR_SV_SIMD_LANE, AIR_SV_SIMD_INDEX, AIR_SV_SIMD_WIDTH,+ AIR_SV_SIMDS_PER_TG,};static const struct { const char *key; enum air_sysval sv; } sysvals[] = {@@ -523,6 +1319,11 @@ static const struct { const char *key; enum air_sysval sv; } sysvals[] = {{ "air.thread_index_in_simdgroup", AIR_SV_SIMD_LANE },{ "air.simdgroup_index_in_threadgroup", AIR_SV_SIMD_INDEX },{ "air.threads_per_simdgroup", AIR_SV_SIMD_WIDTH },+ /* [[thread_execution_width]]: Metal's older name for the same number, the+ * width of the SIMD-group the kernel runs in (Resident Evil 4's mainCS and+ * CS_MiniClusterCompaction take it as an input) */+ { "air.thread_execution_width", AIR_SV_SIMD_WIDTH },+ { "air.simdgroups_per_threadgroup", AIR_SV_SIMDS_PER_TG },};static nir_def *@@ -550,6 +1351,12 @@ load_sysval(struct air_ctx *c, enum air_sysval sv)case AIR_SV_SIMD_LANE: return nir_load_subgroup_invocation(&c->b);case AIR_SV_SIMD_INDEX: return nir_load_subgroup_id(&c->b);case AIR_SV_SIMD_WIDTH: return nir_load_subgroup_size(&c->b);+ /* ir3 has this one and computes it rather than reading a register:+ * ir3_nir_lower_subgroup_id_cs turns load_num_subgroups into+ * 1 + ((threads_per_threadgroup - 1) >> the SIMD-width shift), which folds+ * to a constant whenever the workgroup size is one (ir3_nir.c).+ */+ case AIR_SV_SIMDS_PER_TG: return nir_load_num_subgroups(&c->b);}return NULL;}@@ -571,19 +1378,90 @@ fit(struct air_ctx *c, nir_def *def, const struct air_type *t)return def;}-static void+/* Returns the resource it added, so that a caller with more to say about it --+ * an argument-buffer member, which carries the Metal (set, id) it came from --+ * can fill the rest in; NULL when the node was refused.+ */+static struct air_resource *add_resource(struct air_ctx *c, enum air_resource_kind kind, unsigned index, unsigned size,bool read, bool write, unsigned param, const struct air_md *node){struct air_shader_info *info = c->info;info->resources = reralloc(c->mem_ctx, info->resources, struct air_resource,info->num_resources + 1);- info->resources[info->num_resources++] = (struct air_resource) {+ /* Metal's `[[raster_order_group(n)]]`. Until this was read it was DROPPED IN+ * SILENCE, and that is worth naming rather than just fixing: the branches of+ * translate_input match on the resource's kind and return, so the loop that+ * refuses an `air.*` key nobody handled never runs over a node whose kind is+ * recognised. Every key in such a node is therefore invisible unless+ * something looks for it by name -- which is how a shader could ask for+ * ordering between overlapping fragments and get none, with nothing in any+ * log saying so. `inplace_copy_{lpf,lph}` of QuartzCore's library ask for it+ * on the texture they write.+ */+ const char *name = md_key_str(node, "air.arg_name");+ uint64_t group = 0;+ bool has_group = md_key_int(node, "air.raster_order_group", &group);+ if (has_group) {+ if (group >= 32) {+ /* the bitmask has 32 bits; a group number past them would be recorded+ * as a different group, which is worse than not being recorded */+ air_error(c, "interface: raster order group %u", (unsigned)group);+ return NULL;+ }+ info->raster_order_groups |= 1u << group;+ /* What the hardware can do is not what was asked. The ask is per+ * resource; a6xx's GRAS_SC_CNTL.SINGLE_PRIM_MODE is per draw, and+ * freedreno drives it from pipe_blend_state.blend_coherent+ * (fd6_emit.cc, build_prim_mode). Setting that bit orders the whole draw,+ * which is MORE than the shader asked for -- safe, and slower. There is+ * no way to give exactly the ask and no way to give less, so the shader+ * is translated and the difference is said out loud here instead of being+ * left to a document.+ */+ air_note(c, "raster order group %u on %s %u (%s): a6xx orders a whole draw or "+ "nothing (GRAS_SC_CNTL.SINGLE_PRIM_MODE), never one resource, so the "+ "runtime must set pipe_blend_state.blend_coherent and the ordering "+ "given is coarser than the shader asked for",+ (unsigned)group,+ kind == AIR_RESOURCE_IMAGE ? "image" :+ kind == AIR_RESOURCE_TEXTURE ? "texture" :+ kind == AIR_RESOURCE_SAMPLER ? "sampler" : "buffer",+ index, name ? name : "unnamed");+ }++ info->resources[info->num_resources] = (struct air_resource) {.kind = kind, .index = index, .size = size, .read = read, .write = write,.param = param,- .name = md_key_str(node, "air.arg_name"),+ .name = name,.type_name = md_key_str(node, "air.arg_type_name"),+ .raster_order_group = has_group ? (int)group : -1,+ /* a direct argument: Metal names it by .index and there is no set */+ .argbuf_set = ~0u, .argbuf_id = ~0u,};+ return &info->resources[info->num_resources++];+}++/* A BOOLEAN varying is an integer varying.+ *+ * No hardware interpolates one bit, and make_io_var gives such an output+ * GLSL_TYPE_INT, which is 32 bits wide. Metal spells a `bool2` vertex output+ * as `<2 x i1>` in the returned struct -- RenderBox's stroke shaders pass the+ * two line caps that way, and the fragment stage takes them back as+ * `<2 x i1>` and zero-extends them -- so the store converts to the variable's+ * width and the load converts back.+ *+ * Both halves or neither: storing a 1-bit def into a 32-bit variable is what+ * nir_validate aborts on ("src->ssa->bit_size & bit_sizes"), and reading the+ * integer back as though it were already a boolean worked only because the+ * integer happens to hold 0 or 1 -- an agreement by accident, through a+ * `u2u8` that would have been a `b2i8`.+ */+static bool+io_is_boolean(const struct air_type *t)+{+ const struct air_type *s = air_type_scalar(t);+ return s->kind == AIR_TYPE_INT && s->bits == 1;}static nir_variable *@@ -622,23 +1500,689 @@ add_varying(struct air_ctx *c, const char *name, unsigned slot, unsigned comps,};}-static bool-translate_input(struct air_ctx *c, const struct air_md *node, unsigned *next_varying)+/* Metal's `[[stage_in]]` / `[[attribute(n)]]`: a vertex attribute, read out of+ * a vertex buffer by the fetch hardware rather than computed by the shader.+ *+ * AIR HAS NO STAGE-IN STRUCT. Apple's compiler flattens it: each member is an+ * ordinary parameter of the entry point carrying its own node, and the struct+ * type does not appear in the module at all. QuartzCore's std_vert0_lpf is the+ * shape, read out of the real library and not inferred:+ *+ * define <{ <4 x float>, <4 x float>, <2 x float> }> @std_vert0_lpf(+ * <4 x float> %0, <2 x float> %1, <4 x float> %2, ptr addrspace(2) %3, ...)+ * !19 = !{!20, !21, !22, !23, !24}+ * !20 = !{i32 0, !"air.vertex_input", !"air.location_index", i32 0, i32 1,+ * !"air.arg_type_name", !"float4", !"air.arg_name", !"position"}+ * !21 = !{i32 1, !"air.vertex_input", !"air.location_index", i32 1, i32 1,+ * !"air.arg_type_name", !"float2", !"air.arg_name", !"texcoord0",+ * !"air.arg_unused"}+ * !22 = !{i32 2, !"air.vertex_input", !"air.location_index", i32 3, i32 1,+ * !"air.arg_type_name", !"float4", !"air.arg_name", !"color"}+ *+ * Three things in that are worth saying out loud, because each one would be+ * guessed wrong:+ *+ * - the index is Metal's `[[attribute(n)]]` and NOT a position in the list.+ * These three are 0, 1 and 3, so a translator that numbered them in order+ * would feed `color` from attribute 2 -- a wrong vertex, drawn, with+ * nothing to say so.+ * - `air.arg_unused` occurs ON an attribute, and eight of QuartzCore's+ * fourteen such shaders have one. Nothing reads it, so no NIR input is made:+ * an input costs a VFD fetch slot and makes the pipeline demand a binding+ * that the application's MTLVertexDescriptor may legally not have for an+ * attribute the shader does not read.+ * - the format, offset, stride and buffer of the attribute are NOWHERE in+ * the module. They are the MTLVertexDescriptor's, which is API state. What+ * this builds is the shader's half of the match, recorded in+ * air_shader_info.vertex_attributes for the runtime to check against it.+ *+ * The NIR input is always 32 bits wide and the value narrowed afterwards, which+ * is a decision with a measurement behind it rather than a convention copied+ * from GL: ir3 is perfectly willing to put a 16-bit vertex input in the half+ * register file -- a `half4` attribute compiles to `@in(hr0.x)` and a+ * `cov.f16f32` -- but `ir3_shader_variant.inputs[]` has no half flag (only+ * `outputs[]` does), prints that input as `r0.x`, and+ * `fd6_program.cc:emit_vfd_dest` programs VFD_DEST_CNTL_INSTR_REGID straight+ * out of that field. The fetch would write the full register while the shader+ * read the half one. 32 bits in and a narrowing instruction keeps this in the+ * regime both freedreno drivers use, and the VFD converts the format either way.+ */+static void+add_vertex_attribute(struct air_ctx *c, const struct air_md *node, unsigned index,+ unsigned comps, unsigned bits, bool is_float, bool used,+ unsigned driver_location){- uint64_t param;- if (!md_int(node, 0, ¶m))- return air_error(c, "interface: an input without a parameter number");- if (param >= c->fn->num_args)- return air_error(c, "interface: input for parameter %u of %u", (unsigned)param,- c->fn->num_args);- const struct air_type *t = c->fn->args[param].type;+ struct air_shader_info *info = c->info;+ const char *tn = md_key_str(node, "air.arg_type_name");- /* resources */- if (md_key(node, "air.buffer") >= 0) {- uint64_t as = 0, index = 0, size = 0;- md_key_int(node, "air.address_space", &as);- md_key_int(node, "air.location_index", &index);- md_key_int(node, "air.buffer_size", &size);+ /* `int4` and `uint4` are the same `<4 x i32>`, so the sign is in Apple's+ * reflection string and in nothing the translator can see. `packed_uchar4`+ * is the same question with a prefix.+ */+ const char *base = tn;+ if (base && !strncmp(base, "packed_", 7))+ base += 7;+ enum air_attribute_base_type bt =+ is_float ? AIR_ATTRIB_FLOAT :+ (base && base[0] == 'u') ? AIR_ATTRIB_UINT : AIR_ATTRIB_INT;++ const struct air_md *fc = md_key_node(node, "air.function_constant");++ info->vertex_attributes = reralloc(c->mem_ctx, info->vertex_attributes,+ struct air_vertex_attribute,+ info->num_vertex_attributes + 1);+ info->vertex_attributes[info->num_vertex_attributes++] =+ (struct air_vertex_attribute) {+ .index = index,+ .location = VERT_ATTRIB_GENERIC0 + index,+ .driver_location = driver_location,+ .components = comps,+ .bit_size = bits,+ .base_type = bt,+ .name = md_key_str(node, "air.arg_name"),+ .type_name = tn,+ .used = used,+ /* operand 2 of the predicate node is its name; 0 is the global and 1+ * its type (`!"bool"`) */+ .function_constant = md_str(fc, 2),+ };+ if (used)+ info->vertex_attributes_read |= 1u << index;+}++static bool+translate_vertex_attribute(struct air_ctx *c, const struct air_md *node, unsigned param,+ const struct air_type *t)+{+ const char *tn = md_key_str(node, "air.arg_type_name");+ const char *name = md_key_str(node, "air.arg_name");++ int li = md_key(node, "air.location_index");+ uint64_t index = 0, span = 1;+ if (li < 0 || !md_int(node, li + 1, &index))+ return air_error(c, "interface: a vertex input (%s) with no air.location_index",+ name ? name : "unnamed");++ /* The second integer after the key. What it MEANS is inferred -- Apple's own+ * form for an array of resources, where it is the number of consecutive+ * indices the argument takes -- and nothing in reach can exercise another+ * value: all 974 air.location_index nodes of the five system libraries and+ * all 4 764 on Resident Evil 4's vertex inputs carry 1. An attribute+ * spanning several indices (a matrix member of a stage-in struct is the case+ * that would) is therefore refused by name rather than silently read from+ * the first of them.+ */+ if (md_int(node, li + 2, &span) && span != 1)+ return air_error(c, "interface: a vertex input (%s) spanning %u attribute indices",+ name ? name : "unnamed", (unsigned)span);++ /* Metal has 31 attribute slots; NIR's gl_vert_attrib has 16 generic ones,+ * and VERT_ATTRIB_GENERIC0 + 16 is VERT_ATTRIB_EDGEFLAG. Unchecked, index+ * 16 translated onto the edge flag and index 30 past the end of the enum --+ * both compiled through ir3 without a word, which is the failure this+ * refusal exists to replace.+ */+ if (index >= MAX_VERTEX_GENERIC_ATTRIBS)+ return air_error(c, "interface: vertex attribute %u (%s), and NIR has %u generic "+ "vertex attributes",+ (unsigned)index, name ? name : "unnamed",+ MAX_VERTEX_GENERIC_ATTRIBS);++ /* Two members at one index would make two NIR inputs at one location, two+ * VFD fetches into the same register, and a pipeline whose element array is+ * longer than the attribute set it describes. Metal forbids it; refusing is+ * what keeps a malformed module from being drawn.+ */+ for (unsigned i = 0; i < c->info->num_vertex_attributes; i++)+ if (c->info->vertex_attributes[i].index == index)+ return air_error(c, "interface: two vertex inputs at attribute %u (%s and %s)",+ (unsigned)index,+ c->info->vertex_attributes[i].name+ ? c->info->vertex_attributes[i].name : "unnamed",+ name ? name : "unnamed");++ if (!ret_member_is_number(t))+ return air_error(c, "interface: a vertex input (%s) of type %s, which is neither a "+ "scalar nor a vector",+ name ? name : "unnamed", tn ? tn : "?");++ unsigned comps = air_nir_components(t);+ unsigned bits = air_nir_bit_size(c, t);+ bool is_float = air_type_is_float(air_type_scalar(t));+ /* A vertex attribute is one fetch of at most four components; `double` has+ * no place in MSL at all, and an `i1` or an `i64` attribute has no+ * MTLVertexFormat to come from.+ */+ if (comps > 4 || (is_float ? (bits != 16 && bits != 32)+ : (bits != 8 && bits != 16 && bits != 32)))+ return air_error(c, "interface: a vertex input (%s) of type %s",+ name ? name : "unnamed", tn ? tn : "?");++ if (md_key(node, "air.arg_unused") >= 0) {+ add_vertex_attribute(c, node, index, comps, bits, is_float, false, ~0u);+ c->params[param] = air_val_ssa(c, nir_undef(&c->b, comps, bits));+ return true;+ }++ unsigned location = VERT_ATTRIB_GENERIC0 + index;+ const struct glsl_type *gt = is_float ? glsl_vec_type(comps)+ : glsl_vector_type(GLSL_TYPE_INT, comps);+ nir_variable *var = nir_variable_create(c->nir, nir_var_shader_in, gt,+ name ? name : "attr");+ var->data.location = location;+ unsigned driver_location = c->nir->num_inputs++;+ var->data.driver_location = driver_location;+ /* An interpolation mode is a fragment-stage word and INTERP_MODE_SMOOTH on a+ * vertex attribute would be a claim about something that does not happen.+ */+ var->data.interpolation = INTERP_MODE_NONE;++ /* nir_shader_gather_info does not run on this path (ir3 calls it only for a+ * binning variant), so what it would have set is set here, as for the+ * fragment position above. A runtime matching an MTLVertexDescriptor can+ * read either this or air_shader_info.vertex_attributes_read.+ */+ c->nir->info.inputs_read |= BITFIELD64_BIT(location);++ nir_def *v = nir_load_var(&c->b, var);+ if (bits != 32) {+ /* The fetch delivered a full register; the shader asked for a narrower+ * type. For an integer this is a truncation and the same instruction for+ * either sign -- which of the two the HARDWARE must do when it widens an+ * 8- or 16-bit component to 32 bits is the format's business, and the+ * format is the MTLVertexDescriptor's.+ */+ v = is_float ? nir_f2f16(&c->b, v) : nir_u2uN(&c->b, v, bits);+ }+ add_vertex_attribute(c, node, index, comps, bits, is_float, true, driver_location);++ /* An attribute the module reads only when a function-constant predicate is+ * true is still a fetch here, and that is a caveat rather than an error: the+ * parameter exists whatever the predicate says, so the input is built and+ * vertex_attributes_read gets its bit. A runtime that required a binding for+ * every bit of that mask would refuse a perfectly legal pipeline -- one+ * whose function constants say the attribute is absent and whose+ * MTLVertexDescriptor therefore configures nothing at that index. Which it+ * is cannot be decided here, because the value of the predicate is the+ * pipeline's: these globals are written by the module's air.static_init+ * constructor from the constants the API supplies. So the condition goes in+ * the log at translation time rather than only in a document.+ */+ const struct air_md *fc = md_key_node(node, "air.function_constant");+ if (fc) {+ const char *pred = md_str(fc, 2);+ air_note(c, "vertex attribute %u (%s) is gated by the function-constant predicate "+ "%s: the parameter exists either way, so the attribute is fetched and "+ "its bit is in vertex_attributes_read, and a pipeline whose constants "+ "make the predicate false may legally configure nothing at that index "+ "-- the runtime must read air_vertex_attribute.function_constant before "+ "demanding a binding",+ (unsigned)index, name ? name : "unnamed", pred ? pred : "(unnamed)");+ }++ c->params[param] = air_val_ssa(c, v);+ return true;+}++/* -- argument buffers ---------------------------------------------------- */++/* The module's own declaration of a named struct type, by that name.+ *+ * This is the SECOND statement of what an argument buffer holds. Apple's+ * metadata is the first; it names every member's kind, index and access but+ * not its address space -- not one of the 82 048 members of Resident Evil 4's+ * 7 857 argument buffers carries `air.address_space` -- and a constant buffer+ * bound as a device buffer is a shader reading the wrong descriptor with+ * nothing to say so. The struct type says it, member by member, and the two+ * descriptions are checked against each other before anything is bound: the+ * member count, every member's byte offset and every member's size. They+ * agree on all 7 857 sets of that library, which is what makes using it a+ * reading rather than a guess.+ */+static const struct air_type *+named_struct(const struct air_module *m, const char *name)+{+ for (unsigned i = 0; i < m->num_types; i++)+ if (m->types[i].kind == AIR_TYPE_STRUCT && !m->types[i].opaque &&+ m->types[i].name && !strcmp(m->types[i].name, name))+ return &m->types[i];+ return NULL;+}++/* The `air.*` key of a descriptor this frontend does not bind, for the refusal+ * to name -- `air.command_buffer` and `air.render_pipeline_state` are the two+ * in reach, both in the one kernel of Resident Evil 4 that encodes draws on+ * the GPU. The housekeeping keys every descriptor carries are skipped so that+ * the message names the kind and not the first word in the node.+ */+static const char *+descriptor_kind_key(const struct air_md *desc)+{+ static const char *const plumbing[] = {+ "air.location_index", "air.arg_name", "air.arg_type_name", "air.arg_type_size",+ "air.arg_type_align_size", "air.struct_type_info", "air.address_space",+ "air.buffer_size", "air.read", "air.write", "air.read_write", "air.sample",+ "air.arg_unused", "air.raster_order_group", "air.indirect_argument",+ };+ for (unsigned i = 0; i < (desc ? desc->num_ops : 0); i++) {+ const char *s = md_str(desc, i);+ if (!s || strncmp(s, "air.", 4))+ continue;+ bool known = false;+ for (unsigned k = 0; k < ARRAY_SIZE(plumbing); k++)+ known |= !strcmp(s, plumbing[k]);+ if (!known)+ return s;+ }+ return NULL;+}++/* `air.struct_type_info` of an argument buffer, into a member table.+ *+ * The shape, read off Apple's own output: five fixed fields -- byte offset,+ * size, a flag, the type name, the member name -- and then the key+ * `air.indirect_argument` and a node, which is an ORDINARY resource+ * descriptor. It differs from a direct `air.buffer` / `air.texture` /+ * `air.sampler` argument in one operand, the leading integer, which is the+ * member's number in the set where a direct argument's is its parameter+ * number. So everything needed to bind it is in the vocabulary+ * translate_input has read since the beginning.+ *+ * A member whose key comes BEFORE the five fields is how Apple writes one+ * whose own type is a struct (`!"air.struct_type_info", !N, i32 off, ...`),+ * which in an argument buffer is inline constant data rather than a resource;+ * that falls into the first refusal below, as it must -- the runtime's half of+ * this feature records descriptors against (buffer, offset) and has no bytes+ * to record for a member that is not one.+ */+static bool+argbuf_members(struct air_ctx *c, const struct air_md *sti, const struct air_type *st,+ struct air_argbuf *ab)+{+ unsigned n = 0;+ for (unsigned i = 0; i < sti->num_ops; n++) {+ uint64_t off = 0, msize = 0, flag = 0;+ const char *mtype = md_str(sti, i + 3), *mname = md_str(sti, i + 4);+ if (!md_int(sti, i, &off) || !md_int(sti, i + 1, &msize) ||+ !md_int(sti, i + 2, &flag) || !mtype || !mname) {+ const char *key = md_str(sti, i);+ return air_error(c, "interface: member %u of the argument buffer %s is "+ "described by %s and not by five fields and a descriptor",+ n, ab->name ? ab->name : "unnamed",+ key ? key : "something this translator cannot read");+ }++ const char *key = md_str(sti, i + 5);+ const struct air_md *desc = NULL;+ if (key && !strcmp(key, "air.indirect_argument") && i + 6 < sti->num_ops &&+ sti->ops[i + 6] && sti->ops[i + 6]->kind == AIR_MD_NODE)+ desc = sti->ops[i + 6];+ if (!desc)+ return air_error(c, "interface: member %s of the argument buffer %s is followed "+ "by %s and not by an air.indirect_argument descriptor",+ mname, ab->name ? ab->name : "unnamed", key ? key : "nothing");+ i += 7;++ /* A set inside a set. The member's descriptor would have a member table+ * of its own, reached through a pointer LOADED out of this one and then+ * walked -- which is the general pointer model and not this one.+ */+ if (md_key(desc, "air.indirect_buffer") >= 0)+ return air_error(c, "interface: member %s of the argument buffer %s is itself an "+ "argument buffer", mname, ab->name ? ab->name : "unnamed");++ bool is_buffer = md_key(desc, "air.buffer") >= 0;+ bool is_texture = md_key(desc, "air.texture") >= 0;+ bool is_sampler = md_key(desc, "air.sampler") >= 0;+ if (is_buffer + is_texture + is_sampler != 1) {+ const char *what = descriptor_kind_key(desc);+ return air_error(c, "interface: member %s of the argument buffer %s is %s, and "+ "this frontend binds a buffer, a texture or a sampler",+ mname, ab->name ? ab->name : "unnamed",+ is_buffer + is_texture + is_sampler > 1+ ? "more than one kind at once"+ : what ? what : "a kind this translator does not know");+ }++ int mli = md_key(desc, "air.location_index");+ uint64_t mid = 0, mspan = 1;+ if (mli < 0 || !md_int(desc, mli + 1, &mid))+ return air_error(c, "interface: member %s of the argument buffer %s has no "+ "air.location_index", mname,+ ab->name ? ab->name : "unnamed");+ /* An ARRAY of descriptors in one member: the shader could then pick one+ * at run time, which no resolution can do. Every one of Resident Evil+ * 4's 82 048 members spans exactly one index, so this is a guard on a+ * shape not yet seen rather than a gap with modules behind it.+ */+ if (md_int(desc, mli + 2, &mspan) && mspan != 1)+ return air_error(c, "interface: member %s of the argument buffer %s spans %u "+ "descriptor indices", mname,+ ab->name ? ab->name : "unnamed", (unsigned)mspan);++ /* against the module's own struct type */+ if (n >= st->num_members)+ return air_error(c, "interface: the argument buffer %s declares more members "+ "than the %u its struct type has",+ ab->name ? ab->name : "unnamed", st->num_members);+ const struct air_type *mt = st->members[n];+ uint64_t tyoff = air_struct_member_offset(c->m, st, n);+ uint64_t tysize = air_type_alloc_size(c->m, mt);+ if (tyoff != off || tysize != msize)+ return air_error(c, "interface: member %s of the argument buffer %s is %u bytes "+ "at byte %u, and the module's struct type puts %u bytes "+ "there at %u", mname, ab->name ? ab->name : "unnamed",+ (unsigned)msize, (unsigned)off, (unsigned)tysize,+ (unsigned)tyoff);++ enum air_resource_kind rk;+ unsigned addrspace;+ bool read = md_key(desc, "air.read") >= 0 || md_key(desc, "air.read_write") >= 0 ||+ md_key(desc, "air.sample") >= 0;+ bool write = md_key(desc, "air.write") >= 0 || md_key(desc, "air.read_write") >= 0;++ if (is_buffer) {+ /* The member IS the pointer, and its address space is the only thing+ * that says whether the descriptor is a constant buffer or a device+ * one. A frontend that reused the direct-argument path unchanged+ * would read air.address_space, find none, and refuse at "a buffer in+ * address space 0" -- which is the better of the two ways to be+ * wrong.+ */+ if (mt->kind != AIR_TYPE_POINTER || (mt->addrspace != 1 && mt->addrspace != 2))+ return air_error(c, "interface: the buffer member %s of the argument buffer "+ "%s is not a device or constant pointer in the module's "+ "struct type", mname, ab->name ? ab->name : "unnamed");+ addrspace = mt->addrspace;+ rk = mt->addrspace == 1 ? AIR_RESOURCE_DEVICE_BUFFER : AIR_RESOURCE_CONSTANT_BUFFER;+ } else {+ /* A texture or a sampler arrives as the one-field `metal::texture2d`+ * / `metal::sampler` wrapper the CoreImage stitched functions take+ * (section 7.8), here as a struct member instead of a parameter and+ * with the handle LOADED rather than passed. Keyed on Apple's own+ * name, as the library wrapper is: "a struct with one pointer in it"+ * would quietly accept anything shaped alike.+ */+ if (mt->kind != AIR_TYPE_STRUCT || !mt->name ||+ strncmp(mt->name, "struct.metal::", 14) || mt->num_members != 1 ||+ mt->members[0]->kind != AIR_TYPE_POINTER)+ return air_error(c, "interface: the %s member %s of the argument buffer %s is "+ "not one of Apple's one-field metal:: wrappers in the "+ "module's struct type", is_sampler ? "sampler" : "texture",+ mname, ab->name ? ab->name : "unnamed");+ addrspace = mt->members[0]->addrspace;+ rk = is_sampler ? AIR_RESOURCE_SAMPLER+ : write ? AIR_RESOURCE_IMAGE : AIR_RESOURCE_TEXTURE;+ }++ ab->members = reralloc(c->mem_ctx, ab->members, struct air_argbuf_member, n + 1);+ ab->members[n] = (struct air_argbuf_member) {+ .offset = off, .kind = rk, .metal_id = (unsigned)mid, .index = ~0u,+ .addrspace = addrspace, .read = read, .write = write,+ .name = mname, .type_name = mtype, .node = desc,+ };+ ab->num_members = n + 1;+ }++ if (n != st->num_members)+ return air_error(c, "interface: the argument buffer %s declares %u members and its "+ "struct type has %u", ab->name ? ab->name : "unnamed", n,+ st->num_members);+ return true;+}++/* Metal's argument buffer: `air.indirect_buffer`, one struct of descriptors+ * bound at one buffer index, which is how a Vulkan descriptor set reaches+ * Metal through SPIRV-Cross and how an MSL `[[id(n)]]` struct reaches it from+ * Metal itself. 7 620 of Resident Evil 4's 7 879 modules have one and 0 of+ * iOS 16.5's 671 do, so this is the one interface word that is invisible from+ * Apple's own libraries and is 97% of a game (docs/air.md section 8.2).+ *+ * The parameter does not become a buffer to read: it becomes a pointer of its+ * own class, and the member table built here is what a load through it+ * resolves against (argbuf_load). No address, no descriptor indirection, and+ * nothing of the `32bit_index_offset` model the rest of the frontend is built+ * on changes.+ */+static bool+translate_argument_buffer(struct air_ctx *c, const struct air_md *node, unsigned param)+{+ const char *name = md_key_str(node, "air.arg_name");+ const char *tn = md_key_str(node, "air.arg_type_name");++ int li = md_key(node, "air.location_index");+ uint64_t set_index = 0, span = 1, size = 0;+ if (li < 0 || !md_int(node, li + 1, &set_index))+ return air_error(c, "interface: an argument buffer (%s) with no air.location_index",+ name ? name : "unnamed");+ if (md_int(node, li + 2, &span) && span != 1)+ return air_error(c, "interface: an argument buffer (%s) spanning %u buffer indices",+ name ? name : "unnamed", (unsigned)span);+ md_key_int(node, "air.buffer_size", &size);++ const struct air_md *sti = md_key_node(node, "air.struct_type_info");+ if (!sti)+ return air_error(c, "interface: an argument buffer (%s) with no "+ "air.struct_type_info, so nothing says what is in it",+ name ? name : "unnamed");++ const struct air_type *st = NULL;+ if (tn) {+ char *sname = ralloc_asprintf(c->mem_ctx, "struct.%s", tn);+ st = named_struct(c->m, sname);+ }+ if (!st)+ return air_error(c, "interface: an argument buffer (%s) whose struct type %s the "+ "module does not declare, so its members' address spaces "+ "cannot be read", name ? name : "unnamed", tn ? tn : "(unnamed)");++ /* Two sets at one Metal buffer index would have one record at draw time and+ * two sets of members wanting it. */+ for (unsigned i = 0; i < c->num_argbufs; i++)+ if (c->argbufs[i].metal_index == set_index)+ return air_error(c, "interface: two argument buffers at buffer index %u (%s and "+ "%s)", (unsigned)set_index,+ c->argbufs[i].name ? c->argbufs[i].name : "unnamed",+ name ? name : "unnamed");++ c->argbufs = reralloc(c->mem_ctx, c->argbufs, struct air_argbuf, c->num_argbufs + 1);+ struct air_argbuf *ab = &c->argbufs[c->num_argbufs];+ *ab = (struct air_argbuf) {+ .metal_index = (unsigned)set_index, .size = (unsigned)size, .param = param,+ .name = name, .type_name = tn,+ };+ if (!argbuf_members(c, sti, st, ab))+ return false;+ /* eight bytes per member is what MTLArgumentEncoder writes and what the+ * runtime's encodedLength has to answer; a set whose declared size is not+ * that is a layout this frontend has not seen */+ if (size != 8ull * ab->num_members)+ return air_error(c, "interface: the argument buffer %s is %u bytes with %u members, "+ "and this frontend knows the eight-bytes-per-member layout "+ "only", name ? name : "unnamed", (unsigned)size,+ ab->num_members);++ struct air_ptr p = { .class = AIR_PTR_ARGBUF, .binding = c->num_argbufs++ };+ c->params[param] = air_val_ptr(c, p);+ return true;+}++/* The flat per-kind index of every member of every argument buffer, and one+ * air_resource each.+ *+ * Why this is a decision and not a transcription: a member's own+ * `air.location_index` is Metal's `[[id(n)]]`, which is per set, so set 0's+ * member 0 and set 1's member 0 both say 0 and 237 of Resident Evil 4's+ * modules have two sets. One counter per kind in (set, member) order gives+ * each member a binding of its own, and `argbuf_set` / `argbuf_id` on the+ * resource keep the Metal name the application uses -- the runtime binds by+ * the first and the application names by the second.+ *+ * The counters start above the highest index a DIRECT argument of that kind+ * already took, so a member and an argument cannot land in one Gallium slot.+ * None of Resident Evil 4's 7 620 modules mixes the two -- those shaders have+ * no direct resources at all -- so this is the part of the allocation nothing+ * in reach exercises, and it is here because the collision it prevents would+ * be a draw over the wrong resource rather than an error.+ *+ * Called after every input is translated, so that the direct arguments are all+ * known, and before plan_globals, so that a shader's own `constexpr sampler`+ * still lands after the samplers the API binds.+ */+static void+allocate_argbuf_indices(struct air_ctx *c)+{+ unsigned next_buffer = 0, next_texture = 0, next_sampler = 0;+ for (unsigned i = 0; i < c->info->num_resources; i++) {+ const struct air_resource *r = &c->info->resources[i];+ switch (r->kind) {+ case AIR_RESOURCE_CONSTANT_BUFFER:+ case AIR_RESOURCE_DEVICE_BUFFER:+ next_buffer = MAX2(next_buffer, r->index + 1);+ break;+ case AIR_RESOURCE_TEXTURE:+ case AIR_RESOURCE_IMAGE:+ next_texture = MAX2(next_texture, r->index + 1);+ break;+ case AIR_RESOURCE_SAMPLER:+ next_sampler = MAX2(next_sampler, r->index + 1);+ break;+ default:+ break;+ }+ }++ for (unsigned s = 0; s < c->num_argbufs; s++) {+ struct air_argbuf *ab = &c->argbufs[s];+ for (unsigned k = 0; k < ab->num_members; k++) {+ struct air_argbuf_member *m = &ab->members[k];+ switch (m->kind) {+ case AIR_RESOURCE_CONSTANT_BUFFER:+ case AIR_RESOURCE_DEVICE_BUFFER:+ m->index = next_buffer++;+ break;+ case AIR_RESOURCE_TEXTURE:+ case AIR_RESOURCE_IMAGE:+ m->index = next_texture++;+ break;+ default:+ m->index = next_sampler++;+ break;+ }+ struct air_resource *r = add_resource(c, m->kind, m->index, 0, m->read, m->write,+ ab->param, m->node);+ if (!r)+ return; /* add_resource has said why */+ r->argbuf_set = ab->metal_index;+ r->argbuf_id = m->metal_id;+ r->argbuf_offset = (unsigned)m->offset;+ }+ }+}++/* A load through an argument-buffer pointer RESOLVES instead of loading: the+ * byte offset names a member, and the member's descriptor names a binding.+ *+ * Keying on the byte offset rather than on the member number is what makes the+ * three shapes SPIRV-Cross emits one case. A buffer member is the pointer+ * itself; a texture or sampler member is a one-field wrapper, and the+ * `getelementptr` into that wrapper adds zero bytes -- so the texture shape+ * resolves by the same lookup as the buffer shape, with nothing special about+ * it and no second path to keep right.+ */+static struct air_val *+argbuf_load(struct air_ctx *c, struct air_ptr ptr, const struct air_type *type)+{+ const struct air_argbuf *ab = &c->argbufs[ptr.binding];+ const char *abname = ab->name ? ab->name : "unnamed";++ uint64_t off = 0;+ if (ptr.offset) {+ if (!nir_src_is_const(nir_src_for_ssa(ptr.offset))) {+ air_error(c, "instruction: a load through the argument buffer %s at an offset "+ "that is not constant, which is a descriptor chosen at run time",+ abname);+ return NULL;+ }+ off = nir_src_as_uint(nir_src_for_ssa(ptr.offset));+ }++ const struct air_argbuf_member *m = NULL;+ for (unsigned i = 0; i < ab->num_members; i++)+ if (ab->members[i].offset == off)+ m = &ab->members[i];+ if (!m) {+ air_error(c, "instruction: a load from byte %u of the argument buffer %s, which "+ "declares no member there", (unsigned)off, abname);+ return NULL;+ }++ /* A number read out of the set rather than a descriptor: Metal's+ * `constantDataAtIndex:`. Every member of this set is a resource -- that is+ * what argbuf_members checked -- so this is a load of a descriptor's bytes,+ * which are the runtime's 8-byte handle and mean nothing to a driver that+ * binds by index.+ */+ if (type->kind != AIR_TYPE_POINTER) {+ air_error(c, "instruction: a load of a value out of the argument buffer %s, whose "+ "member %s at byte %u is a resource and has no bytes to read",+ abname, m->name ? m->name : "unnamed", (unsigned)off);+ return NULL;+ }++ /* The address space of the pointer the shader LOADS, against the one the+ * module's struct type declares for that member. The two come from+ * different records -- the load instruction's own result type and the+ * module's type table -- and either one alone would bind a constant buffer+ * as a device buffer without a word if it were wrong.+ */+ if (type->addrspace != m->addrspace) {+ air_error(c, "instruction: a pointer in address space %u loaded from member %s of "+ "the argument buffer %s, which the module's struct type puts in "+ "address space %u", type->addrspace, m->name ? m->name : "unnamed",+ abname, m->addrspace);+ return NULL;+ }++ struct air_ptr p = { .binding = m->index, .pointee = type->elem };+ switch (m->kind) {+ case AIR_RESOURCE_CONSTANT_BUFFER: p.class = AIR_PTR_UBO; break;+ case AIR_RESOURCE_DEVICE_BUFFER: p.class = AIR_PTR_SSBO; break;+ case AIR_RESOURCE_SAMPLER: p.class = AIR_PTR_SAMPLER; break;+ case AIR_RESOURCE_IMAGE:+ p.class = AIR_PTR_TEXTURE;+ p.is_image = true;+ break;+ default:+ p.class = AIR_PTR_TEXTURE;+ break;+ }+ return air_val_ptr(c, p);+}++static bool+translate_input(struct air_ctx *c, const struct air_md *node, unsigned *next_varying)+{+ uint64_t param;+ if (!md_int(node, 0, ¶m))+ return air_error(c, "interface: an input without a parameter number");+ if (param >= c->fn->num_args)+ return air_error(c, "interface: input for parameter %u of %u", (unsigned)param,+ c->fn->num_args);+ const struct air_type *t = c->fn->args[param].type;++ /* resources */+ if (md_key(node, "air.buffer") >= 0) {+ uint64_t as = 0, index = 0, size = 0;+ /* AIR before 2.4 (MapKit's Clear::clear_vertex, Metal 2.3 for iOS 14)+ * writes no air.address_space on a buffer: the parameter's own pointer+ * type is then the only statement of it, and it is the same fact -- the+ * key, where present, restates the address space of that pointer.+ */+ if (!md_key_int(node, "air.address_space", &as) && t->kind == AIR_TYPE_POINTER)+ as = t->addrspace;+ md_key_int(node, "air.location_index", &index);+ md_key_int(node, "air.buffer_size", &size);bool read = md_key(node, "air.read") >= 0 || md_key(node, "air.read_write") >= 0;bool write = md_key(node, "air.write") >= 0 || md_key(node, "air.read_write") >= 0;@@ -650,13 +2194,18 @@ translate_input(struct air_ctx *c, const struct air_md *node, unsigned *next_varcase 3:p.class = AIR_PTR_SHARED;rk = AIR_RESOURCE_THREADGROUP_BUFFER;- /* a threadgroup argument is sized at dispatch; its base is where the- * previous ones end, which only the runtime knows. One is all the- * corpus uses, so its base is zero and anything else is refused.+ /* a threadgroup argument is sized at dispatch, so it can only go+ * after everything whose size is known here: the module's static+ * threadgroup arrays, which plan_globals lays out from offset 0. Its+ * base is set once they are (below plan_globals). At base 0 it would+ * share its bytes with the first of them. One is all the corpus uses;+ * a second would start where the first's runtime length ends, which+ * only the runtime knows, and is refused.*/if (index != 0)return air_error(c, "interface: a second threadgroup buffer (index %u)",(unsigned)index);+ c->tg_param_plus1 = param + 1;break;default:return air_error(c, "interface: a buffer in address space %u", (unsigned)as);@@ -666,6 +2215,9 @@ translate_input(struct air_ctx *c, const struct air_md *node, unsigned *next_varreturn true;}+ if (md_key(node, "air.indirect_buffer") >= 0)+ return translate_argument_buffer(c, node, param);+if (md_key(node, "air.texture") >= 0) {uint64_t index = 0;md_key_int(node, "air.location_index", &index);@@ -695,6 +2247,65 @@ translate_input(struct air_ctx *c, const struct air_md *node, unsigned *next_varreturn true;}+ /* A TILE SHADER's imageblock. Named here rather than left to the catch-all+ * at the end of this function, because that one printed "interface: input+ * air.imageblock" -- the key it met, which reads like a decision and was+ * only a gap. There are two walls behind it and the near one is NOT the+ * Adreno's, so the sentence has to say which is which or the next reader+ * spends a day on the wrong one.+ *+ * THE NEAR WALL: nothing on this device can ask for these modules.+ * QuartzCore reaches its four tile_downsample_* pipelines from exactly two+ * places, and both ask one question first -- the same bit either way, bit 23+ * of the capability word at ctx+0xa3c:+ *+ * CA::OGL::BlurState::tile_downsample 0x1883d3578 get(property 37), and+ * returns 0 on a zero+ * CA::OGL::MetalContext::warmup_shaders 0x188476350 ldrb [ctx, #0xa3e]+ * 0x188476354 tbz w8, #7 -> skip+ *+ * `get` (0x1882e8a78) is that bit and a CA default; the bit is set by+ * CA::OGL::new_metal_context (0x18847fbfc) from -[MTLDevice+ * supportsTileShaders]; and Metal answers that as+ * -[_MTLDevice supportsTileShaders] (0x181aa897c) ->+ * -[_MTLDeviceFeatureQueries familySupportsTileShaders] (0x181b216b0), whose+ * whole body is `supportsFamily:` with w2 = 0x3ec -- MTLGPUFamilyApple4.+ * -[_MTLDevice initGPUFamilySupport] (0x181a1be1c) builds that list from+ * -featureProfile through a jump table at 0x181b5bc7c: profile 0 claims+ * Apple1, profile 1 adds Apple2, and Apple4 first appears at profile 5+ * (0x181a1c424). This port answers 0 or 1 (userland/src/redwoodmetal.m,+ * boot-arg metal_profile=), and the phone itself has printed+ * supportsFamily:Apple4 = 0.+ *+ * Apple4 is not a later generation's feature, which is the part worth+ * saying out loud: it is the iPhone X's OWN family, and Apple's A11 driver+ * answers profile 5. These four shaders run on this very phone under iOS.+ * The only reason nothing here can ask for them is that this port claims+ * less than its hardware is.+ *+ * THE FAR WALL is the Adreno's, and it is why raising the profile would buy+ * a refusal rather than a shader: Mesa 25.0.7 has no tile memory anywhere in+ * src/freedreno, and the one instruction that reads the tile buffer is+ * fragment-only by assertion (ir3_compiler_nir.c, nir_texop_txf_ms_fb).+ * docs/air.md section 3 has what the DRIVER would have to do instead -- break+ * the pass, resolve the attachment, dispatch an ordinary kernel -- which is+ * the round trip tile shading exists to avoid, and which belongs to whoever+ * writes the pass splitting and not to this function.+ *+ * Only the near wall is in the message, because air_error's buffer is 256+ * bytes (air_private.h) and vsnprintf cuts the rest off mid-word without+ * saying so: a longer sentence would have ended at "-featureProfile" in+ * every report, and the second wall is exactly the half that would have+ * gone. The message names the one that decides and points here for the one+ * that would decide next.+ */+ if (md_key(node, "air.imageblock") >= 0)+ return air_error(c, "interface: an imageblock input -- a tile shader nothing on "+ "this device can ask for: CoreAnimation builds these only "+ "behind -[MTLDevice supportsTileShaders] = "+ "supportsFamily:Apple4, and this port claims at most Apple2 "+ "(docs/air.md 3 has the other wall)");+/* system values */for (unsigned i = 0; i < ARRAY_SIZE(sysvals); i++) {if (md_key(node, sysvals[i].key) < 0)@@ -709,25 +2320,198 @@ translate_input(struct air_ctx *c, const struct air_md *node, unsigned *next_var/* interpolated inputs of a fragment shader, and vertex attributes */if (md_key(node, "air.position") >= 0) {if (c->nir->info.stage == MESA_SHADER_FRAGMENT) {- nir_def *pos = nir_load_frag_coord(&c->b);+ /* A position the shader does not read gets no input at all, and the order of+ * these two tests is the whole reason the test is here: every one of Apple's+ * seventeen texturedFragment* declares `float4 pos [[position]]` and marks it+ * air.arg_unused, so putting the position machinery before this check would+ * give all seventeen a dead varying slot and a dead FS_COORD property -- a+ * change to 17 shaders already compared pixel by pixel, bought for nothing.+ */+ if (md_key(node, "air.arg_unused") >= 0) {+ c->params[param] = air_val_ssa(c, nir_undef(&c->b, air_nir_components(t),+ air_nir_bit_size(c, t)));+ return true;+ }+ /*+ * Metal's [[position]] in a fragment function is the window coordinate with+ * the origin at the TOP left, y down, and the pixel centre at a half-integer+ * -- which is what origin_upper_left plus !pixel_center_integer says in NIR.+ * NIR's default is OpenGL's bottom-left origin, so leaving these unset would+ * give every position-dependent Metal shader a y counted from the wrong edge.+ *+ * And the value is a SYSTEM-VALUE VARIABLE, not a bare nir_load_frag_coord,+ * because the two Gallium drivers this frontend feeds want opposite forms and+ * only one direction has a pass:+ *+ * - freedreno sets pipe_caps.fs_position_is_sysval, and ir3 compile_asserts+ * `slot != VARYING_SLOT_POS` in its fragment-input setup+ * (ir3_compiler_nir.c): it will only take the intrinsic, which is what+ * nir_lower_system_values turns this variable into;+ * - softpipe does NOT set that cap, and nir_to_tgsi turns the intrinsic into+ * ureg_DECL_system_value(TGSI_SEMANTIC_POSITION), which softpipe fills only+ * for compute (sp_compute.c is the one writer of machine->SystemValue). A+ * fragment shader therefore read (0,0,0,0) -- measured, not reasoned: a+ * hand-written module returning its own position printed four zeros while+ * the same module returning a constant printed the constant. Softpipe wants+ * the position as an input varying, which only+ * nir_lower_sysvals_to_varyings produces, and only from a variable.+ *+ * So the frontend emits the form spirv_to_nir emits, for the same reason, and+ * the consumer lowers it per its screen's cap (finalize_for_gallium).+ */+ nir_variable *var = nir_variable_create(c->nir, nir_var_system_value,+ glsl_vec4_type(), "position");+ var->data.location = SYSTEM_VALUE_FRAG_COORD;+ var->data.interpolation = INTERP_MODE_NOPERSPECTIVE;+ nir_def *pos = nir_load_var(&c->b, var);c->params[param] = air_val_ssa(c, fit(c, pos, t));c->nir->info.fs.origin_upper_left = true;c->nir->info.fs.pixel_center_integer = false;+ /* nothing in this path calls nir_shader_gather_info, and nir_to_tgsi emits the+ * FS_COORD_ORIGIN property only for a shader whose info says it reads the+ * position -- without this the property is absent and softpipe falls back on+ * TGSI_FS_COORD_ORIGIN_UPPER_LEFT (0), which is the right answer by accident.+ */+ BITSET_SET(c->nir->info.system_values_read, SYSTEM_VALUE_FRAG_COORD);+ c->nir->info.inputs_read |= VARYING_BIT_POS;return true;}return air_error(c, "interface: air.position as an input of a %s shader",gl_shader_stage_name(c->nir->info.stage));}++ /* Metal's `[[color(n)]]` argument to a fragment function: the value render target n+ * holds NOW, read inside the shader. That is framebuffer fetch, and Metal's name for+ * it is programmable blending.+ *+ * The modules say which feature it is rather than this being inferred from the name:+ * every one of the twenty in QuartzCore's library carries+ * `!"air.compile.framebuffer_fetch_enable"` among its `air.compile_options`, the+ * option Apple's compiler sets exactly when a fragment argument is `[[color(n)]]`.+ *+ * The shape is ONE integer after the key -- the target -- where the output form has+ * two, the second being the dual-source index:+ *+ * output !{!"air.render_target", i32 0, i32 0, !"air.arg_type_name", !"float4", ...}+ * input !{i32 1, !"air.render_target", i32 2, !"air.arg_type_name", !"float", ...}+ *+ * so the two are read here and in translate_output at their own offsets, not through+ * one shared helper that would read a dual-source index out of an argument name.+ *+ * NIR's form of a framebuffer fetch is a READ OF THE OUTPUT: load_output carrying+ * io_semantics.fb_fetch_output, which is what nir_lower_io makes of a GLSL `inout`+ * colour and what nir_lower_blend emits for a blend it has to lower in software. The+ * intrinsic is built here directly instead of as a second nir_var_shader_out variable+ * at the same location, because both consumers of this frontend lower IO before the+ * backend sees it and two output variables at one location would take two driver+ * locations and two entries in ir3's output table for one attachment.+ *+ * It is read as FOUR 32-BIT FLOATS whatever the argument's type, and narrowed after:+ * nir_lower_fb_read -- the pass ir3 runs in ir3_nir_post_finalize -- rewrites+ * load_output into a txf_ms_fb whose destination is fixed at 4 x f32 and rewrites the+ * intrinsic's uses with it. So `float` (path_blit_frag's coverage mask) loses three+ * channels and `half4` (every _lph build) loses precision here, where the conversion+ * is visible, rather than silently inside the lowering.+ */+ int rt_in = md_key(node, "air.render_target");+ if (rt_in >= 0) {+ if (c->nir->info.stage != MESA_SHADER_FRAGMENT)+ return air_error(c, "interface: air.render_target as an input of a %s shader",+ gl_shader_stage_name(c->nir->info.stage));+ /* An argument nothing reads must not turn the shader into a framebuffer-fetch+ * shader: info.fs.uses_fbfetch_output alone makes a6xx flush the pipeline per+ * overlapping primitive (fd6_emit.cc, build_prim_mode), and an unused argument+ * needs no value at all. None of the corpus's twenty is marked this way; the+ * order is here so that one would cost nothing rather than a flush.+ */+ if (md_key(node, "air.arg_unused") >= 0) {+ c->params[param] = air_val_ssa(c, nir_undef(&c->b, air_nir_components(t),+ air_nir_bit_size(c, t)));+ return true;+ }+ uint64_t n = 0;+ if (!md_int(node, rt_in + 1, &n))+ return air_error(c, "interface: an air.render_target input without a target");+ if (n >= MAX_DRAW_BUFFERS)+ return air_error(c, "interface: air.render_target input %u", (unsigned)n);+ /* Only a float target. The lowering hands back floats converted from the+ * attachment's format, so an int4 or uint4 `[[color(n)]]` would arrive as the+ * wrong value rather than the wrong type -- and nothing would say so. The corpus+ * declares float4, half4 and float and nothing else, so the refusal is a guard on+ * a case not yet seen rather than a gap with modules behind it.+ */+ unsigned bits = air_nir_bit_size(c, t);+ if (!air_type_is_float(air_type_scalar(t)) || (bits != 16 && bits != 32)) {+ const char *tn = md_key_str(node, "air.arg_type_name");+ return air_error(c, "interface: an air.render_target input of type %s",+ tn ? tn : "?");+ }++ nir_io_semantics sem = {+ .location = FRAG_RESULT_DATA0 + n,+ .num_slots = 1,+ .fb_fetch_output = true,+ /* Coherent, which is Metal's guarantee and not a safe default picked here: a+ * `[[color(n)]]` read sees what the primitives already drawn left behind, which+ * is the whole of how CoreAnimation blends a layer in place. On a6xx the flag+ * is what makes the GMEM path flush per overlap instead of not at all+ * (fd6_emit.cc); the sysmem path flushes either way.+ */+ .fb_fetch_output_coherent = true,+ };+ nir_def *fetched = nir_load_output(&c->b, 4, 32, nir_imm_int(&c->b, 0),+ .dest_type = nir_type_float32,+ .io_semantics = sem);+ unsigned want = air_nir_components(t);+ if (want < 4)+ fetched = nir_trim_vector(&c->b, fetched, want);+ if (bits == 16)+ fetched = nir_f2f16(&c->b, fetched);++ /* nir_shader_gather_info does not run on this path -- ir3 calls it only for a+ * binning variant -- so what it would have set from the intrinsic+ * (nir_gather_info.c) is set here. Without uses_fbfetch_output the draw gets no+ * flush at all and the read returns whatever the tile buffer happened to hold.+ */+ c->nir->info.outputs_read |= BITFIELD64_BIT(sem.location);+ c->nir->info.fs.uses_fbfetch_output = true;+ c->nir->info.fs.fbfetch_coherent = true;+ c->info->fbfetch_targets |= 1u << n;++ /* The layer the fetch reads is not this frontend's to choose, and it is+ * not safe by default, so it goes in the log rather than only in a+ * document. nir_lower_fb_read builds the coordinate as (x, y, layer)+ * with the layer from nir_load_layer_id, which ir3_nir_lower_layer_id+ * turns into a flat varying read of VARYING_SLOT_LAYER. If the pipeline's+ * vertex stage writes air.render_target_array_index, ir3_link_shaders+ * finds it by slot and the fetch reads the right layer; if nothing wrote+ * it, ir3_link_add skips the entry (regid(63, 0)) and the varying holds+ * whatever the VPC location held. Which of the two it is cannot be known+ * from this module -- the fragment shader is identical either way -- so+ * the condition is stated for whoever builds the pipeline.+ */+ air_note(c, "the framebuffer fetch reads its layer from VARYING_SLOT_LAYER, which "+ "nir_lower_fb_read asks for and this module cannot write: for a LAYERED "+ "render target the pipeline's vertex stage must write "+ "air.render_target_array_index, or the fetch reads an undefined layer. "+ "A single-layer target is unaffected -- there is no second layer in the "+ "descriptor to reach");++ c->params[param] = air_val_ssa(c, fetched);+ return true;+ }+if (md_key(node, "air.fragment_input") >= 0 || md_key(node, "air.vertex_input") >= 0 ||md_key(node, "air.stage_in") >= 0) {- unsigned slot;- if (c->nir->info.stage == MESA_SHADER_VERTEX) {- uint64_t index = 0;- md_key_int(node, "air.location_index", &index);- slot = VERT_ATTRIB_GENERIC0 + index;- } else {- slot = VARYING_SLOT_VAR0 + (*next_varying)++;- }+ /* A vertex attribute and an interpolated varying are two different things+ * that AIR spells with keys in the same family, and only the one below is+ * a varying: it is linked to the stage before it, interpolated, and takes+ * its slot from the order it appears in. An attribute is fetched from a+ * vertex buffer at Metal's own index and has nothing to interpolate.+ */+ if (c->nir->info.stage == MESA_SHADER_VERTEX)+ return translate_vertex_attribute(c, node, param, t);+ unsigned slot = VARYING_SLOT_VAR0 + (*next_varying)++;const char *name = md_key_str(node, "air.arg_name");nir_variable *var = make_io_var(c, nir_var_shader_in, t, slot, name);var->data.driver_location = c->nir->num_inputs++;@@ -741,7 +2525,10 @@ translate_input(struct air_ctx *c, const struct air_md *node, unsigned *next_varvar->data.sample = md_key(node, "air.sample") >= 0;if (c->nir->info.stage == MESA_SHADER_FRAGMENT)add_varying(c, name, slot, air_nir_components(t), node);- c->params[param] = air_val_ssa(c, nir_load_var(&c->b, var));+ nir_def *loaded = nir_load_var(&c->b, var);+ if (io_is_boolean(t))+ loaded = nir_ine_imm(&c->b, loaded, 0); /* see io_is_boolean */+ c->params[param] = air_val_ssa(c, loaded);return true;}@@ -811,6 +2598,41 @@ translate_output(struct air_ctx *c, const struct air_md *node, unsigned field,return air_error(c, "interface: an output with no key this translator knows");}+ /* An output whose type is not a number has no varying here, and until this+ * was checked it got one anyway: an `air.clip_distance` output arrives as+ * an ARRAY of float, and air_type_scalar has no answer for an array, so the+ * varying was declared with whatever air_nir_bit_size made of that. NIR's+ * own form is an output of several slots, which this frontend does not+ * build, so it is refused by name instead.+ *+ * The length is in the message because it decides how much work closing+ * this is, and the measured length is not the one the general case+ * suggests: over all 86 of Resident Evil 4's modules that stop here --+ * every one a vertex shader -- the member is `[1 x float]`, an array of+ * exactly ONE, and Metal's own `air.arg_type_name` on the node says+ * `float`. So there is no module in reach that declares two clip planes,+ * and the shape that is actually in the way is an array of one, which is a+ * scalar wearing a bracket. Closing that is two things and not one: this+ * declaration would take the element type, and emit_return would have to+ * store elems[0] of a one-element aggregate instead of skipping a field+ * that is not an AIR_VAL_SSA -- which is what silently drops the plane+ * today. It is left refused rather than half built, because a clip distance+ * that is written to nothing is a shader that draws without the clipping it+ * asked for, and nothing in this tree can check a vertex clip distance+ * numerically.+ */+ if (t->kind == AIR_TYPE_ARRAY || t->kind == AIR_TYPE_STRUCT) {+ const char *tn = md_key_str(node, "air.arg_type_name");+ if (t->kind == AIR_TYPE_ARRAY)+ return air_error(c, "interface: an output (%s), declared %s, that arrives as an "+ "array of %u, which this frontend has no varying for",+ name ? name : "unnamed", tn ? tn : "an aggregate",+ (unsigned)t->count);+ return air_error(c, "interface: an output (%s), declared %s, that arrives as a "+ "struct of %u members, which this frontend has no varying for",+ name ? name : "unnamed", tn ? tn : "an aggregate", t->num_members);+ }+nir_variable *var = make_io_var(c, nir_var_shader_out, t, location, name);var->data.index = index;var->data.driver_location = c->nir->num_outputs++;@@ -822,22 +2644,72 @@ translate_output(struct air_ctx *c, const struct air_md *node, unsigned field,/* -- instructions ------------------------------------------------------- */+/* A load through a pointer this frontend could not resolve: load through BOTH+ * sides of the choice and select the values. That is the only form NIR has --+ * there is no select of pointers in the 32bit_index_offset model -- and it is+ * not free of consequence, so it is said out loud: both sides are read, so a+ * side that is not a valid place to read from is read anyway. In the shaders+ * that need this one side is a module constant array and the other a buffer+ * the pipeline binds, so both are readable; a pipeline that bound neither+ * would read a descriptor it did not set up.+ */+static nir_def *+load_through_choice(struct air_ctx *c, struct air_val *v, const struct air_type *t,+ unsigned align)+{+ if (v->kind == AIR_VAL_PTR)+ return air_load_ptr(c, v->ptr, t, align);+ if (v->kind != AIR_VAL_PTR_SELECT) {+ air_error(c, "instruction: a load through something that is not a pointer");+ return NULL;+ }+ nir_def *a = load_through_choice(c, v->elems[0], t, align);+ nir_def *b = load_through_choice(c, v->elems[1], t, align);+ if (!a || !b)+ return NULL;+ air_note(c, "a pointer chosen at run time (`select` between two buffers): both sides "+ "are LOADED and the values selected, because NIR has no select of "+ "pointers -- so a side the pipeline left unbound is read anyway, and its "+ "value discarded");+ return nir_bcsel(&c->b, v->def, a, b);+}+static bool-translate_gep(struct air_ctx *c, const struct air_inst *in)+gep_into(struct air_ctx *c, struct air_val *base, const struct air_inst *in,+ struct air_val **out){- struct air_val *base = air_get(c, in->operands[0]);- if (!base)- return false;+ /* A getelementptr on a pointer CHOSEN at run time: the index goes into both+ * sides and the choice stays. Both sides have the same pointee type -- the+ * select that made the choice required it -- so this is one walk per side+ * and not a different offset on each.+ */+ if (base->kind == AIR_VAL_PTR_SELECT) {+ struct air_val *a = NULL, *b = NULL;+ if (!gep_into(c, base->elems[0], in, &a) ||+ !gep_into(c, base->elems[1], in, &b))+ return false;+ *out = air_val_ptr_select(c, base->def, a, b);+ return true;+ }if (base->kind != AIR_VAL_PTR)- return air_error(c, "instruction: getelementptr on something that is not a pointer");+ return air_error(c, "instruction: getelementptr on something that is not a "+ "pointer");struct air_ptr p = base->ptr;const struct air_type *t = in->aux_type;- /* index 0 steps over objects of the source type */+ /* index 0 steps over objects of the source type. A constant one folds, for+ * the reason the element index below gives: `getelementptr inbounds i8, ptr+ * %local, i64 4` is the shape a byte offset into a local arrives in, and a+ * slot can only resolve an offset it can read (slot_cell).+ */uint64_t stride = air_type_alloc_size(c->m, t);uint64_t first_const;- if (!air_const_uint(in->operands[1], &first_const) || first_const != 0) {+ if (air_const_uint(in->operands[1], &first_const)) {+ uint32_t off = (uint32_t)first_const * (uint32_t)stride;+ if (off)+ p = air_ptr_add(c, p, nir_imm_int(&c->b, off));+ } else {nir_def *first = air_get_ssa(c, in->operands[1]);if (!first)return false;@@ -862,14 +2734,44 @@ translate_gep(struct air_ctx *c, const struct air_inst *in)} else if (t->kind == AIR_TYPE_ARRAY || t->kind == AIR_TYPE_VECTOR) {uint64_t es = t->kind == AIR_TYPE_ARRAY ? air_type_alloc_size(c->m, t->elem): MAX2(air_type_bit_size(t->elem) / 8, 1);- p = air_ptr_add(c, p, nir_imul_imm(&c->b, nir_u2u32(&c->b, idx), es));+ /* A CONSTANT INDEX IS A CONSTANT OFFSET, and saying so here is not an+ * optimisation: a slot resolves a byte offset into one of its members+ * only when that offset is decidable while translating (slot_cell), and+ * `getelementptr inbounds i8, ptr %local, i64 4` -- which is how a byte+ * offset into a struct reaches one -- used to arrive as an SSA value+ * because nir_builder does not fold. The arithmetic is the same one+ * the SSA path performs: nir_u2u32 takes the low 32 bits and+ * nir_imul_imm multiplies in 32-bit wrapping, so this folds to the+ * identical number rather than to a more correct one.+ */+ uint64_t ci;+ if (air_const_uint(in->operands[i], &ci)) {+ uint32_t off = (uint32_t)ci * (uint32_t)es;+ if (off)+ p = air_ptr_add(c, p, nir_imm_int(&c->b, off));+ } else {+ p = air_ptr_add(c, p, nir_imul_imm(&c->b, nir_u2u32(&c->b, idx), es));+ }t = t->elem;} else {return air_error(c, "instruction: getelementptr into a scalar");}}p.pointee = t;- air_set(c, &in->base, air_val_ptr(c, p));+ *out = air_val_ptr(c, p);+ return true;+}++static bool+translate_gep(struct air_ctx *c, const struct air_inst *in)+{+ struct air_val *base = air_get(c, in->operands[0]);+ if (!base)+ return false;+ struct air_val *res = NULL;+ if (!gep_into(c, base, in, &res))+ return false;+ air_set(c, &in->base, res);return true;}@@ -933,10 +2835,21 @@ translate_cast(struct air_ctx *c, const struct air_inst *in)nir_def *x = air_get_ssa(c, in->operands[0]);if (!x)return false;- unsigned bits = air_nir_bit_size(c, dt);+ /* A truncation into a width NIR has no register for is the one way such a+ * value is ever made (narrow_int_bits), so the width is taken from there+ * rather than from air_nir_bit_size, which would refuse it. Anything else+ * that touches the result or the source is refused by narrow_int_gate.+ */+ unsigned narrow = narrow_int_bits(dt);+ unsigned bits = narrow ? narrow_promoted_bits(narrow) : air_nir_bit_size(c, dt);nir_def *res;switch (in->sub) {case AIR_CAST_TRUNC:+ if (narrow) {+ /* into canonical form: the low N bits, zero above them */+ res = nir_iand_imm(&c->b, nir_u2uN(&c->b, x, bits), narrow_mask(narrow));+ break;+ }res = bits == 1 ? nir_i2b(&c->b, nir_iand_imm(&c->b, x, 1)) : nir_u2uN(&c->b, x, bits);break;case AIR_CAST_ZEXT:@@ -957,12 +2870,88 @@ translate_cast(struct air_ctx *c, const struct air_inst *in)break;case AIR_CAST_FPTRUNC: res = nir_f2fN(&c->b, x, bits); break;case AIR_CAST_FPEXT: res = nir_f2fN(&c->b, x, bits); break;- case AIR_CAST_BITCAST:- if (air_nir_components(dt) != air_nir_components(st) ||- air_nir_bit_size(c, dt) != air_nir_bit_size(c, st))- return air_error(c, "instruction: a bitcast that changes the shape of its value");- res = nir_mov(&c->b, x);+ /* A bitcast moves no bits: it renames them. When the two types have the+ * same shape -- the same component count and the same component width --+ * the rename is a NIR `mov`, because a NIR def carries a width and a+ * component count and no notion of float or integer, so `i32 -> float` and+ * `<4 x i32> -> <4 x float>` are already the same def.+ *+ * When the shape differs the bits have to be redistributed between+ * components, and NIR has an operation for exactly one redistribution of+ * exactly one width: pack_32_2x16 and unpack_32_2x16. That is the whole of+ * what this corpus asks for, and the number is the reason this is two cases+ * and not a general repacker -- `tools/air/typecensus.py` over all 7 879 of+ * Resident Evil 4's modules, reading the types out of llvm-dis rather than+ * out of our own reader, finds 111 657 bitcasts of 14 distinct type pairs,+ * and the ONLY two that change shape are:+ *+ * i32 -> <2 x half> 44 136 of them, in 2 927 modules+ * <2 x half> -> i32 4 333 of them, in 688 modules+ *+ * with 0 shape-changing bitcasts in the whole 456-module system corpus.+ * There is no third shape to be found behind these, so the pair below is+ * not a sample of a family: it is the family. Anything else keeps a refusal+ * -- and one that names the two shapes, so that a third, if some other+ * library has one, arrives as an arithmetic fact instead of a mystery.+ *+ * The half/half-pair of an `i32` is how an HLSL `min16float2` reaches here+ * through SPIRV-Cross: the pair travels through the shader packed in one+ * 32-bit register and is unpacked where it is used.+ *+ * Keyed on the SHAPE and not on the AIR type names, deliberately. NIR's+ * SSA has no float at 32 bits that is distinct from an integer at 32 bits,+ * so `float -> <2 x half>` -- which this corpus does not contain -- is the+ * same bit movement as `i32 -> <2 x half>` and is right for the same+ * reason. What a case keyed on the names would buy is a refusal for a+ * module whose bits are being moved correctly.+ *+ * Which half goes where is little-endian, which is not a choice: LLVM lays+ * element 0 of a vector at the lowest address, so element 0 of the+ * `<2 x half>` is bits 0..15 of the `i32`, and `unpack_32_2x16` is defined+ * `dst.x = src0.x; dst.y = src0.x >> 16` -- the same halves in the same+ * order. Getting that backwards would not crash and would not look wrong:+ * both halves are still finite halves, so the shader would draw with every+ * pair of 16-bit values transposed. tools/air/loweringcheck.py gives the+ * two halves values that cannot be confused for each other (13 and 113, 1+ * and -2) and checks them twice -- against LLVM running the same module on+ * this processor, and against the four numbers written out in the case+ * list, which come from Python's own binary16 codec and so are not LLVM's+ * endianness restated. It also says what the transposed answer would have+ * been. The drawn reference is NOT available for these: softpipe's+ * nir_to_tgsi has no 16-bit arithmetic at all, with or without a bitcast.+ */+ case AIR_CAST_BITCAST: {+ unsigned dn = air_nir_components(dt), sn = air_nir_components(st);+ unsigned db = bits, sb = air_nir_bit_size(c, st);+ if (dn == sn && db == sb)+ res = nir_mov(&c->b, x);+ else if (sn == 1 && sb == 32 && dn == 2 && db == 16)+ res = nir_unpack_32_2x16(&c->b, x);+ else if (sn == 2 && sb == 16 && dn == 1 && db == 32)+ res = nir_pack_32_2x16(&c->b, x);+ /* i32 <-> <4 x i8>: MPS's and CoreMedia's image kernels (mediaserverd's+ * hbf_compute_residual_error_kernel) take a word apart into its bytes; the+ * same little-endian order, element 0 the low byte */+ else if (sn == 1 && sb == 32 && dn == 4 && db == 8)+ res = nir_unpack_32_4x8(&c->b, x);+ else if (sn == 4 && sb == 8 && dn == 1 && db == 32)+ res = nir_pack_32_4x8(&c->b, x);+ /* i64 <-> <2 x i32>, the low word first */+ else if (sn == 1 && sb == 64 && dn == 2 && db == 32)+ res = nir_unpack_64_2x32(&c->b, x);+ else if (sn == 2 && sb == 32 && dn == 1 && db == 64)+ res = nir_pack_64_2x32(&c->b, x);+ /* any other repacking of as many bits (<2 x i32> -> <4 x half> in+ * mediaserverd's CNN kernels): nir_extract_bits moves them in the same+ * little-endian order the cases above spell out */+ else if (sn * sb == dn * db && sb >= 8 && db >= 8 && dn <= NIR_MAX_VEC_COMPONENTS)+ res = nir_extract_bits(&c->b, &x, 1, 0, dn, db);+ else+ return air_error(c, "instruction: a bitcast from %u x %u bits to %u x %u bits, "+ "which is a repacking this frontend does not do",+ sn, sb, dn, db);break;+ }default:return air_error(c, "instruction: cast %u", in->sub);}@@ -1012,15 +3001,64 @@ translate_cmp(struct air_ctx *c, const struct air_inst *in)/* an aggregate value's field, by the index path of extract/insertvalue */static struct air_val **-agg_slot(struct air_ctx *c, struct air_val *agg, const uint32_t *idx, unsigned n)+agg_slot(struct air_ctx *c, struct air_val *agg, const struct air_type *type,+ const uint32_t *idx, unsigned n){+ /* extractvalue and insertvalue always carry at least one index. Without this+ * the loop below never runs and the function returns the address of its own+ * parameter -- which gcc points out (-Wreturn-local-addr) and which would be+ * a write through a dead stack slot if anything ever took the path.+ */+ if (!n) {+ air_error(c, "instruction: an aggregate access with no index");+ return NULL;+ }struct air_val **slot = &agg;+ const struct air_type *t = type;for (unsigned i = 0; i < n; i++) {- if (!*slot || (*slot)->kind != AIR_VAL_AGG || idx[i] >= (*slot)->num_elems) {+ /* The TYPE is walked beside the value because the value alone cannot say+ * how many elements a level has that nothing has written yet.+ */+ unsigned count;+ const struct air_type *et;+ if (t && t->kind == AIR_TYPE_STRUCT) {+ count = t->num_members;+ et = idx[i] < count ? t->members[idx[i]] : NULL;+ } else if (t && t->kind == AIR_TYPE_ARRAY) {+ count = t->count > UINT_MAX ? UINT_MAX : (unsigned)t->count;+ et = idx[i] < count ? t->elem : NULL;+ } else {+ count = 0;+ et = NULL;+ }+ if (!et) {+ air_error(c, "instruction: an aggregate index out of its type");+ return NULL;+ }+ /* A level nothing has written yet, made here rather than refused.+ *+ * `insertvalue [3 x [2 x float]] undef, float %x, 0, 0` reaches the row+ * before the row exists: an undef aggregate is built with its fields+ * NONE on purpose, so a NESTED one has no row for the column to go in.+ * Making it changes no value -- an aggregate whose every field is NONE+ * *is* the undef of that type, which is exactly what air_constant_val+ * builds for the top level -- so this is the same value said in the form+ * the walk can continue through.+ *+ * It is 16 of RenderBox's 47 modules, every one of them building a 3x2+ * transform this way before passing it to a shape function, and the+ * refusal it used to raise ("an aggregate index out of its value") named+ * the symptom at the second index rather than the missing level at the+ * first.+ */+ if (!*slot)+ *slot = air_val_agg(c, count);+ if ((*slot)->kind != AIR_VAL_AGG || idx[i] >= (*slot)->num_elems) {air_error(c, "instruction: an aggregate index out of its value");return NULL;}slot = &(*slot)->elems[idx[i]];+ t = et;}return slot;}@@ -1074,12 +3112,65 @@ air_value_kind_name(const struct air_value *v)return "a constant of an unknown kind";}+/* The one place a value of a width NIR has no register for is let through.+ *+ * Such a value is carried in the next width up with the bits above it zero+ * (narrow_int_bits), which is exact for a short list of operations and wrong+ * for the rest -- and wrong in the quiet way: `add i3` in a 8-bit register+ * carries into bit 3 and the shader computes something, so nothing fails. The+ * list is therefore checked here, over every instruction of every module,+ * rather than relied on from the four modules that happen to need it: the gate+ * is what makes the representation safe, and a representation whose safety+ * rests on nobody writing the other case is not safe.+ *+ * The ones that pass are the ones canonical form makes exact: a truncation+ * into the type, a truncation or a zero-extension out of it, an unsigned+ * conversion to float, an equality or an unsigned comparison, and a switch,+ * which the CFG walk emits as equalities against constants of the same width.+ * Everything else is refused by width, opcode and operand number.+ */static bool-translate_instruction(struct air_ctx *c, const struct air_inst *in)+narrow_int_gate(struct air_ctx *c, const struct air_inst *in){- switch (in->op) {- case AIR_OP_BINOP:- return translate_binop(c, in);+ unsigned n = narrow_int_bits(in->base.type);+ if (n && !(in->op == AIR_OP_CAST && in->sub == AIR_CAST_TRUNC))+ return air_error(c, "instruction: a %u-bit integer made by opcode %u, a width NIR "+ "has no register for; only a truncation into one is exact in "+ "the next width up", n, in->op);++ for (unsigned i = 0; i < in->num_operands; i++) {+ const struct air_value *v = in->operands[i];+ unsigned m = v ? narrow_int_bits(v->type) : 0;+ if (!m)+ continue;+ if (in->op == AIR_OP_SWITCH)+ continue;+ if (in->op == AIR_OP_CMP &&+ (in->sub == AIR_ICMP_EQ || in->sub == AIR_ICMP_NE ||+ in->sub == AIR_ICMP_ULT || in->sub == AIR_ICMP_ULE ||+ in->sub == AIR_ICMP_UGT || in->sub == AIR_ICMP_UGE))+ continue;+ if (in->op == AIR_OP_CAST &&+ (in->sub == AIR_CAST_TRUNC || in->sub == AIR_CAST_ZEXT ||+ in->sub == AIR_CAST_UITOFP))+ continue;+ return air_error(c, "instruction: a %u-bit integer reaching opcode %u as operand "+ "%u, a width NIR has no register for; only a switch, an "+ "equality, an unsigned comparison and a widening that ignores "+ "the sign are exact in the next width up", m, in->op, i);+ }+ return true;+}++static bool+translate_instruction(struct air_ctx *c, const struct air_inst *in)+{+ if (!narrow_int_gate(c, in))+ return false;++ switch (in->op) {+ case AIR_OP_BINOP:+ return translate_binop(c, in);case AIR_OP_UNOP: {nir_def *x = air_get_ssa(c, in->operands[0]);if (!x)@@ -1094,6 +3185,61 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)case AIR_OP_GEP:return translate_gep(c, in);case AIR_OP_SELECT: {+ /* A select between two POINTERS, which the GL-on-Metal uber shaders use+ * to fall back on a module constant array when the pipeline bound no+ * buffer. It is not a number, so it cannot be a bcsel here; the choice+ * is kept and made where the pointer is loaded through.+ */+ struct air_val *pa = air_get(c, in->operands[1]);+ struct air_val *pb = air_get(c, in->operands[2]);+ if (!pa || !pb)+ return false;+ if (val_is_pointerish(pa) || val_is_pointerish(pb)) {+ if (!val_is_pointerish(pa) || !val_is_pointerish(pb))+ return air_error(c, "instruction: a select between a pointer and "+ "something that is not one");+ /* A choice that is not a choice. The GL-on-Metal uber shaders pick+ * between the pipeline's sampler and their own inline one on a+ * function-constant predicate, and once that predicate is an+ * immediate there is one pointer here and not two -- which matters+ * because the alternative below keeps the choice and a TEXTURE or a+ * SAMPLER is not a value it can load. Folding first is also what+ * answers the argument-buffer case under it for free when the+ * condition is known, instead of refusing a select that has already+ * been decided.+ */+ nir_def *pcond = air_get_ssa(c, in->operands[0]);+ bool ptaken;+ if (pcond && pcond->num_components == 1 &&+ cond_const(c, pcond, &ptaken)) {+ air_set(c, &in->base, ptaken ? pa : pb);+ return true;+ }+ /* A choice between two members of an argument buffer cannot be kept+ * the way a choice between two buffers can: AIR_VAL_PTR_SELECT loads+ * both sides and selects the values, and a texture or a sampler is+ * not a value to load. Two names for ONE member is no choice at all+ * and is answered; anything else is refused rather than drawn over+ * whichever of the two the code happened to take.+ */+ if ((pa->kind == AIR_VAL_PTR && pa->ptr.class == AIR_PTR_ARGBUF) ||+ (pb->kind == AIR_VAL_PTR && pb->ptr.class == AIR_PTR_ARGBUF)) {+ if (pa->kind == AIR_VAL_PTR && pb->kind == AIR_VAL_PTR &&+ same_ptr(&pa->ptr, &pb->ptr)) {+ air_set(c, &in->base, pa);+ return true;+ }+ return air_error(c, "instruction: a select between two different places in "+ "an argument buffer");+ }+ nir_def *pc = air_get_ssa(c, in->operands[0]);+ if (!pc)+ return false;+ if (pc->num_components != 1)+ return air_error(c, "instruction: a vector select between pointers");+ air_set(c, &in->base, air_val_ptr_select(c, pc, pa, pb));+ return true;+ }nir_def *cond = air_get_ssa(c, in->operands[0]);nir_def *x = air_get_ssa(c, in->operands[1]);nir_def *y = air_get_ssa(c, in->operands[2]);@@ -1109,7 +3255,17 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)nir_def *i = air_get_ssa(c, in->operands[1]);if (!v || !i)return false;- air_set(c, &in->base, air_val_ssa(c, nir_vector_extract(&c->b, v, nir_u2u32(&c->b, i))));+ /* A BOOLEAN index. `extractelement <2 x i1> %54, i1 %55` is in+ * RenderBox's stroke shaders -- the line's two caps picked by which end+ * of the segment the pixel is nearer -- and `u2u32` is the wrong+ * conversion for it: a NIR boolean is a 1-bit def, not a 1-bit integer,+ * and widening one with an unsigned conversion is undefined where+ * nir_b2i32 is exactly 0 or 1. ir3 says so outright ("invalid src bit+ * size: 1") and refuses the shader; a backend that did not say so would+ * have read lane ~0u instead of lane 1.+ */+ nir_def *idx = i->bit_size == 1 ? nir_b2i32(&c->b, i) : nir_u2u32(&c->b, i);+ air_set(c, &in->base, air_val_ssa(c, nir_vector_extract(&c->b, v, idx)));return true;}case AIR_OP_INSERTELT: {@@ -1118,11 +3274,27 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)nir_def *i = air_get_ssa(c, in->operands[2]);if (!v || !x || !i)return false;- if (!nir_src_is_const(nir_src_for_ssa(i)))- return air_error(c, "instruction: insertelement at a run-time index");- air_set(c, &in->base,- air_val_ssa(c, nir_vector_insert_imm(&c->b, v, x,- nir_src_as_uint(nir_src_for_ssa(i)))));+ /* AN INDEX DECIDED WHILE THE SHADER RUNS, which NIR has its own lowering+ * for: nir_vector_insert builds `bcsel(i == {0,1,2,3}, scalar, vec)`,+ * one comparison per component, and takes the constant path when the+ * index is one. So this is Mesa's rule rather than one written here, and+ * the constant case goes through exactly the code it went through before+ * (nir_vector_insert calls nir_vector_insert_imm itself).+ *+ * The one place the two languages differ is an index PAST the vector.+ * LLVM says an out-of-range insertelement is poison; the bcsel chain+ * misses every component and yields the vector unchanged. Poison permits+ * any value, so an unchanged vector is a refinement of it and not a+ * disagreement -- which is the only reason this can be done without+ * inventing a semantics.+ *+ * It is the wall fixed_frag_lph_cph reaches once its texture and sampler+ * choices are resolved, under BOTH the all-ones configuration and the+ * compositor's own (docs/air.md section 3), which is what made it worth+ * taking rather than refusing: two configurations that disagree about+ * everything else converge here.+ */+ air_set(c, &in->base, air_val_ssa(c, nir_vector_insert(&c->b, v, x, i)));return true;}case AIR_OP_SHUFFLEVEC: {@@ -1186,14 +3358,23 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)struct air_val *agg = air_get(c, in->operands[0]);if (!agg)return false;- struct air_val **slot = agg_slot(c, agg, in->indices, in->num_indices);+ struct air_val **slot = agg_slot(c, agg, in->operands[0]->type,+ in->indices, in->num_indices);if (!slot)return false;if (!*slot) {- /* a field of an undef aggregate */- air_set(c, &in->base,- air_val_ssa(c, nir_undef(&c->b, air_nir_components(in->base.type),- air_nir_bit_size(c, in->base.type))));+ /* A field of an undef aggregate: the undef of that field's OWN type,+ * built the one way this translator knows how to build an undef of+ * anything -- which matters because the field may itself be an+ * aggregate, and a nir_undef cannot be one.+ */+ struct air_constant u = { .base = { .kind = AIR_VALUE_CONSTANT,+ .type = in->base.type },+ .ckind = AIR_CONST_UNDEF };+ struct air_val *v = air_constant_val(c, &u);+ if (!v)+ return false;+ air_set(c, &in->base, v);return true;}air_set(c, &in->base, *slot);@@ -1205,7 +3386,8 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)if (!agg || !val)return false;struct air_val *copy = clone_val(c, agg);- struct air_val **slot = agg_slot(c, copy, in->indices, in->num_indices);+ struct air_val **slot = agg_slot(c, copy, in->operands[0]->type,+ in->indices, in->num_indices);if (!slot)return false;*slot = val;@@ -1219,6 +3401,48 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)uint64_t count;if (!air_const_uint(in->operands[0], &count))return air_error(c, "instruction: an alloca of a run-time size");+ /* An alloca is bytes in scratch -- unless it holds a POINTER, which+ * scratch cannot: a pointer in this frontend is a resolution, a binding+ * and a byte offset decided while translating, and has no bytes to put+ * anywhere. Then it is a translation-time SLOT instead (AIR_PTR_SLOT in+ * air_private.h, and the whole-store and whole-load cases above).+ *+ * Spelled out because what stood here twice described code that was not+ * here. It first promised a slot "see AIR_VAL_SLOT" when no such value+ * kind existed -- that slot had been written and removed -- and the+ * correction then said no slot would help, which was true of the module+ * count and not of the wall: the slot closes the FIRST of four walls in+ * the two GL-on-Metal fragment shaders, and the three behind it are+ * either closed by specialising on the function constants+ * (plan_global_folding) or, in the case of air.get_null_texture_2d, a+ * refusal that now says whose wall it is. tools/air/chosencheck.sh is+ * the record of all four.+ *+ * The same comment also said those two shaders "store their last two+ * buffer arguments into such a local and never read them back".+ * Parameters 34 and 35 of fixed_frag_lph_cph are `air.sampler`+ * (img_samp_0, img_samp_1), not buffers, and the module does read them+ * back: `select` against its own inline sampler, then `load`, into the+ * sampler argument of 27 air.sample_texture_2d calls and 12 calls to its+ * own box-sample functions. That is why the slot has to hold a whole+ * air_val and not a number.+ */+ /* ONE object that holds a POINTER: a translation-time slot, because+ * scratch cannot hold one. `count == 1` and nothing else, so an array of+ * pointers still goes to memory and still refuses at the store -- an+ * array would need an index, and an index into a tree of values is the+ * thing this deliberately does not do.+ */+ if (count == 1 && type_has_pointer(in->aux_type, 8)) {+ struct air_ptr sp = {+ .class = AIR_PTR_SLOT,+ .binding = new_slot(c, in->aux_type, NULL),+ .pointee = in->aux_type,+ };+ air_set(c, &in->base, air_val_ptr(c, sp));+ return true;+ }+uint64_t size = air_type_alloc_size(c->m, in->aux_type) * count;unsigned align = MAX2(in->align, air_type_abi_align(c->m, in->aux_type));c->scratch_size = ALIGN_POT(c->scratch_size, MAX2(align, 1));@@ -1232,11 +3456,71 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)return true;}case AIR_OP_LOAD: {+ /* A program-scope variable whose single static-initialiser store turned+ * out to be a constant: the load IS that constant, rebuilt in whatever+ * builder is asking. Rebuilt and not shared, because a nir_def belongs+ * to one impl and a callee may load the same variable -- which is the+ * same rule materialise_globals follows for the offset.+ */+ struct air_global_plan *gpl = foldable_global_plan(c, in->operands[0]);+ if (gpl && gpl->known) {+ air_set(c, &in->base,+ air_val_ssa(c, nir_imm_intN_t(&c->b, gpl->bits, gpl->bit_size)));+ return true;+ }struct air_val *p = air_get(c, in->operands[0]);if (!p)return false;+ unsigned slot;+ if (slot_base(c, p, &slot)) {+ struct air_val *v = slot_at_zero(c->slots[slot].val, in->base.type);+ if (!v)+ return air_error(c, "instruction: a load from a local that holds a "+ "pointer and has not been written");+ air_set(c, &in->base, v);+ return true;+ }+ /* Not the slot's base: a member of it, if the offset and the loaded type+ * name one between them (slot_cell). Tried AFTER slot_base and not+ * instead of it, so that nothing that translated before this existed can+ * take a different path: offset zero with a type slot_at_zero's looser+ * descent accepts still goes the way it went.+ */+ uint64_t soff;+ if (slot_at_const(c, p, &slot, &soff)) {+ struct air_val **cell = slot_cell(c, slot, soff, in->base.type);+ if (cell && *cell) {+ air_set(c, &in->base, *cell);+ return true;+ }+ if (cell)+ return air_error(c, "instruction: a load from a member of a local that "+ "holds a pointer and has not been written");+ }+ if (p->kind == AIR_VAL_PTR && p->ptr.class == AIR_PTR_SLOT)+ return air_error(c, "instruction: a load at an offset into a local that holds "+ "a pointer which does not name one of its members, so "+ "there is no value there to give");+ if (p->kind == AIR_VAL_PTR_SELECT) {+ nir_def *res = load_through_choice(c, p, in->base.type, in->align);+ if (!res)+ return false;+ air_set(c, &in->base, air_val_ssa(c, res));+ return true;+ }if (p->kind != AIR_VAL_PTR)return air_error(c, "instruction: a load from something that is not a pointer");+ /* An argument buffer resolves before it loads: the byte offset names a+ * member and the member's descriptor names a binding, so this load+ * becomes a pointer and not an instruction.+ */+ if (p->ptr.class == AIR_PTR_ARGBUF) {+ struct air_val *res = argbuf_load(c, p->ptr, in->base.type);+ if (!res)+ return false;+ air_set(c, &in->base, res);+ return true;+ }nir_def *res = air_load_ptr(c, p->ptr, in->base.type, in->align);if (!res)return false;@@ -1245,11 +3529,81 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)}case AIR_OP_STORE: {struct air_val *p = air_get(c, in->operands[1]);+ if (!p)+ return false;+ /* A POINTER stored into memory. There is nothing to write: a pointer in+ * this frontend is a resolution -- a binding and a byte offset decided+ * while translating -- and not an address, so it has no bytes. Naming+ * that here rather than letting air_get_ssa say "a pointer where a+ * number belongs" is the difference between a wall with a reason and a+ * wall with a symptom; the reason is in docs/air.md section 3.+ */+ struct air_val *sv = air_get(c, in->operands[0]);+ if (!sv)+ return false;+ unsigned sslot;+ uint64_t soff;+ if (slot_base(c, p, &sslot) &&+ in->operands[0]->type == c->slots[sslot].type) {+ if (c->slots[sslot].init)+ return air_error(c, "instruction: a store into a read-only global, which is "+ "where its own initialiser lives");+ c->slots[sslot].val = sv;+ return true;+ }+ /* A store into one MEMBER of a slot. The type stored and the byte offset+ * name the member between them, and neither does on its own+ * (slot_cell). This is what the four `disparity_refinement_*` stop at:+ * CoreImage's stitched functions rebuild a `coreimage::Sampler` in a+ * local out of the four arguments it was flattened into.+ */+ if (slot_at_const(c, p, &sslot, &soff)) {+ if (c->slots[sslot].init)+ return air_error(c, "instruction: a store into a read-only global, which is "+ "where its own initialiser lives");+ struct air_val **cell = slot_cell(c, sslot, soff,+ in->operands[0]->type);+ if (cell) {+ *cell = sv;+ return true;+ }+ return air_error(c, "instruction: a store into a local that holds a pointer at "+ "byte %u, which names no member of it: a tree of values has "+ "no bytes for an offset that straddles two of them",+ (unsigned)soff);+ }+ if (p->kind == AIR_VAL_PTR && p->ptr.class == AIR_PTR_SLOT)+ return air_error(c, "instruction: a store at an offset computed at run time "+ "into a local that holds a pointer, so which member it "+ "names is not decidable here");+ if (val_is_pointerish(sv))+ return air_error(c, "instruction: a store of a pointer into memory, which this "+ "frontend has no address to write");+ if (p->kind == AIR_VAL_PTR_SELECT)+ return air_error(c, "instruction: a store through a chosen pointer, which would "+ "have to write both of them");nir_def *v = air_get_ssa(c, in->operands[0]);- if (!p || !v)+ if (!v)return false;if (p->kind != AIR_VAL_PTR)return air_error(c, "instruction: a store to something that is not a pointer");+ /* The one store into a foldable program-scope variable. Its value is+ * recorded when it is a constant, and the memory is written either way:+ * plan_global_folding has proved that nothing else reaches these bytes,+ * so the store is dead when the fold answers -- and leaving it in costs+ * one DCE'd instruction, where leaving it OUT would be a promise about+ * every access shape that scan might have missed.+ */+ struct air_global_plan *gpl = foldable_global_plan(c, in->operands[1]);+ if (gpl) {+ nir_const_value cv[NIR_MAX_VEC_COMPONENTS];+ if (v->num_components == 1 &&+ def_const(v, cv, 16, c->nir->info.float_controls_execution_mode)) {+ gpl->known = true;+ gpl->bits = nir_const_value_as_uint(cv[0], v->bit_size);+ gpl->bit_size = v->bit_size;+ }+ }return air_store_ptr(c, p->ptr, v, in->align);}case AIR_OP_CALL: {@@ -1260,6 +3614,7 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)}case AIR_OP_RET:case AIR_OP_BR:+ case AIR_OP_SWITCH:case AIR_OP_UNREACHABLE:case AIR_OP_PHI:return true; /* handled by the CFG walk */@@ -1296,6 +3651,30 @@ emit_return(struct air_ctx *c, const struct air_inst *in)* caller passed, because a NIR call has no destination. The entry point* has no such parameter and its return is its render targets, below.*/+ if (c->num_ret_fields) {+ /* A callee returning a struct: one out-parameter per member, because a+ * NIR parameter carries a number and a struct here is a tree of values.+ * The members with no parameter are the ones nothing can read (see+ * ret_fields in air_private.h), and a member the callee left out of its+ * insertvalue chain is undef -- Metal's own vertex-output structs leave+ * their padding field exactly that way, so this is the normal case and+ * not a defect.+ */+ if (v->kind != AIR_VAL_AGG || v->num_elems != c->num_ret_fields)+ return air_error(c, "instruction: a returned struct that was never built, "+ "from a function called in the module");+ for (unsigned i = 0; i < v->num_elems; i++) {+ if (!c->ret_fields[i] || !v->elems[i])+ continue;+ if (v->elems[i]->kind != AIR_VAL_SSA)+ return air_error(c, "instruction: field %u of a returned struct is not a "+ "number, from a function called in the module", i);+ nir_def *d = v->elems[i]->def;+ nir_store_deref(&c->b, c->ret_fields[i], d,+ nir_component_mask(d->num_components));+ }+ return true;+ }if (c->ret_deref) {nir_def *d = air_get_ssa(c, in->operands[0]);if (!d)@@ -1314,8 +3693,11 @@ emit_return(struct air_ctx *c, const struct air_inst *in)return air_error(c, "interface: field %u of the result has no output", i);if (!v->elems[i] || v->elems[i]->kind != AIR_VAL_SSA)continue; /* a field left undef is a target nothing writes */- nir_store_var(&c->b, c->outputs[i].var, v->elems[i]->def,- BITFIELD_MASK(v->elems[i]->def->num_components));+ nir_def *d = v->elems[i]->def;+ if (io_is_boolean(rt->members[i]) && d->bit_size == 1)+ d = nir_b2iN(&c->b, d, 32); /* see io_is_boolean */+ nir_store_var(&c->b, c->outputs[i].var, d,+ BITFIELD_MASK(d->num_components));}return true;}@@ -1373,6 +3755,372 @@ reverse_post_order(struct air_ctx *c, const struct air_function *f, unsigned *orstatic bool translate_body(struct air_ctx *c);static void materialise_globals(struct air_ctx *c);+/* A module's globals and its static initialisers: per MODULE, and a stitched+ * shader is built from several, so these run once for each of them rather than+ * once for the shader (stitch_graph below).+ */+static void plan_globals(struct air_ctx *c);+static void run_static_initializers(struct air_ctx *c);++/* Does this value hold a pointer anywhere inside it? */+static bool+val_holds_ptr(const struct air_val *v)+{+ if (!v)+ return false;+ if (val_is_pointerish(v))+ return true;+ if (v->kind != AIR_VAL_AGG)+ return false;+ for (unsigned i = 0; i < v->num_elems; i++)+ if (val_holds_ptr(v->elems[i]))+ return true;+ return false;+}++/* -- an aggregate handed to a call of a module function -------------------+ *+ * A NIR function parameter carries a NUMBER -- a scalar, or a vector of them.+ * A struct or an array is a tree of values in this translator and never a NIR+ * value at all (air_private.h), so a function called with one BY VALUE has to+ * take the numbers inside it, one parameter each, and put the tree back+ * together on the other side.+ *+ * This is what Metal's own compiler asks for here and not an exotic case.+ * RenderBox splits its shader state into one-field `RenderState` structs and+ * hands every shape, blend, filter and resolve function its globals as a+ * struct of matrices -- `{ [3 x [2 x float]], float }`, and for the blur+ * `{ float, [2 x float], [2 x float], [2 x float], [4 x [2 x float]],+ * [4 x half] }`, nineteen numbers in one argument. 26 of that library's 47+ * modules stop at such a call.+ *+ * The order is extractvalue's: member 0's numbers, then member 1's, depth+ * first. The caller and the callee walk the same type with the same function,+ * which is what keeps the sources and the parameters in step -- and every call+ * site checks the count it built against nir_call_instr.num_params as well,+ * because "in step" is a claim and a call whose sources are one short is a+ * segfault inside nir_instr_insert rather than a message.+ */+#define AIR_MAX_ARG_LEAVES 64++/* the numbers inside a type, or why it has one that no parameter can carry */+static bool+type_leaves(struct air_ctx *c, const struct air_type *t,+ const struct air_type **out, unsigned *n)+{+ if (!t)+ return air_error(c, "intrinsic: a call argument of no type");+ if (t->kind == AIR_TYPE_STRUCT || t->kind == AIR_TYPE_ARRAY) {+ unsigned count = t->kind == AIR_TYPE_STRUCT ? t->num_members+ : (unsigned)MIN2(t->count, UINT_MAX);+ for (unsigned i = 0; i < count; i++) {+ const struct air_type *et = t->kind == AIR_TYPE_STRUCT ? t->members[i] : t->elem;+ if (!type_leaves(c, et, out, n))+ return false;+ }+ return true;+ }+ if (t->kind != AIR_TYPE_VECTOR && t->kind != AIR_TYPE_INT && !air_type_is_float(t))+ return air_error(c, "intrinsic: a call passing %s inside a value, which is not a "+ "number a parameter can carry",+ t->kind == AIR_TYPE_POINTER ? "a pointer" : "a type of no size");+ if (*n >= AIR_MAX_ARG_LEAVES)+ return air_error(c, "intrinsic: a call passing a value of more than %u numbers",+ AIR_MAX_ARG_LEAVES);+ if (out)+ out[*n] = t;+ (*n)++;+ return true;+}++/* the caller's side: the defs to pass, in that same order */+static bool+val_leaves(struct air_ctx *c, const struct air_type *t, struct air_val *v,+ nir_def **out, unsigned *n)+{+ if (!t)+ return air_error(c, "intrinsic: a call argument of no type");+ if (t->kind == AIR_TYPE_STRUCT || t->kind == AIR_TYPE_ARRAY) {+ if (v && v->kind != AIR_VAL_AGG)+ return air_error(c, "intrinsic: a call passing %s where its type says an "+ "aggregate",+ v->kind == AIR_VAL_SSA ? "a number" : "a pointer");+ unsigned count = t->kind == AIR_TYPE_STRUCT ? t->num_members+ : (unsigned)MIN2(t->count, UINT_MAX);+ for (unsigned i = 0; i < count; i++) {+ const struct air_type *et = t->kind == AIR_TYPE_STRUCT ? t->members[i] : t->elem;+ if (!val_leaves(c, et, v && i < v->num_elems ? v->elems[i] : NULL, out, n))+ return false;+ }+ return true;+ }+ if (*n >= AIR_MAX_ARG_LEAVES)+ return air_error(c, "intrinsic: a call passing a value of more than %u numbers",+ AIR_MAX_ARG_LEAVES);+ if (!v) {+ /* A number nothing wrote is undef, which is what the aggregate already+ * says: an undef struct is built with its fields NONE (air_constant_val),+ * and Metal's compiler does pass whole `undef` globals -- filter_blur is+ * handed `PlaneGlobals undef` because the blur reads none of it.+ */+ out[(*n)++] = nir_undef(&c->b, air_nir_components(t), air_narrow_bits(c, t));+ return true;+ }+ if (v->kind != AIR_VAL_SSA)+ return air_error(c, "intrinsic: a call passing %s where a number belongs",+ v->kind == AIR_VAL_AGG ? "an aggregate" : "a pointer");+ out[(*n)++] = v->def;+ return true;+}++/* the callee's side: the tree, back out of its parameters */+static struct air_val *+val_from_leaves(struct air_ctx *c, const struct air_type *t, nir_def **in, unsigned *n)+{+ if (t->kind == AIR_TYPE_STRUCT || t->kind == AIR_TYPE_ARRAY) {+ unsigned count = t->kind == AIR_TYPE_STRUCT ? t->num_members+ : (unsigned)MIN2(t->count, UINT_MAX);+ struct air_val *agg = air_val_agg(c, count);+ for (unsigned i = 0; i < count; i++) {+ const struct air_type *et = t->kind == AIR_TYPE_STRUCT ? t->members[i] : t->elem;+ if (!(agg->elems[i] = val_from_leaves(c, et, in, n)))+ return NULL;+ }+ return agg;+ }+ return air_val_ssa(c, in[(*n)++]);+}++/* Whether a resolution inside this value carries an offset computed at run+ * time. Checked before anything is built, so the refusal names the argument+ * rather than arriving from the middle of the callee's body.+ */+static bool+val_offset_is_computed(const struct air_val *v)+{+ if (!v)+ return false;+ if (v->kind == AIR_VAL_AGG) {+ for (unsigned i = 0; i < v->num_elems; i++)+ if (val_offset_is_computed(v->elems[i]))+ return true;+ return false;+ }+ return v->kind == AIR_VAL_PTR && v->ptr.offset &&+ !nir_src_is_const(nir_src_for_ssa(v->ptr.offset));+}++/* The caller's resolution, with every byte offset in it rebuilt in whichever+ * builder is current now.+ *+ * A pointer here is *(where, byte offset)* and the offset is an nir_def, which+ * belongs to the impl that made it; handing the callee the caller's def is the+ * cross-function def/use that the globals used to cause, and that is why an+ * argument carrying one was refused outright.+ *+ * A CONSTANT offset need not be refused, and every one that occurs is+ * constant: what a RenderBox shader hands its shape function is an ALLOCA,+ * whose offset is the scratch cursor at translation time (AIR_OP_ALLOCA builds+ * nir_imm_int of it), and the callee reads the struct back out of scratch with+ * geps of its own. Nothing is shared -- the NUMBER is copied and a new def+ * made here -- so the callee's offset is a def of the callee's own impl while+ * the resolution is the same resolution. The scratch it names is the same+ * scratch because air_ctx.scratch_size is the SHADER's, not the function's,+ * and nir_inline_functions folds the callee into the caller afterwards anyway.+ *+ * FOURTEEN of RenderBox's 47 modules stop here once the aggregate-index barrier+ * above is lifted, chronod's first fault among them, and thirteen of the+ * fourteen need only this: the one that does not is uber_fragment, whose+ * offset is read out of its globals and goes through ptr_offset_to_pass. The+ * number is the frontend's own count of the modules that reported this+ * refusal, not a reading of the IR -- a scan of the bitcode for "an alloca+ * passed to a call" gave 12 and then 14 depending on how the call line was+ * matched, which is a reason to take the count from the thing that counts.+ */+/* The byte offset of a pointer argument that has to be PASSED as a number+ * rather than copied as one.+ *+ * Where a pointer points is a translation-time fact -- which buffer, at which+ * Metal index -- and that half is bound into the callee whatever the offset is.+ * The offset is the half that can be a VALUE: uber_fragment reads an index out+ * of its globals and hands `device void *` at that byte to sample_image+ * (`getelementptr inbounds i8, ptr addrspace(1) %7, i64 %416`), so there is+ * nothing constant to copy and nothing resolved to bind.+ *+ * So the binding is bound and the offset becomes one more NIR parameter, which+ * puts the def on the right side of the function boundary: the caller passes+ * its own def as a call source, and the callee's offset is its own+ * load_param. That is the same thing nir_call does for every other number and+ * it is why this is exact rather than a workaround.+ *+ * Only a pointer that IS the argument, not one inside a struct: a resolution+ * nested in an aggregate with a run-time offset has no case in either corpus,+ * and rebind_here still refuses it by name rather than guessing a layout for+ * the parameters it would take.+ */+static nir_def *+ptr_offset_to_pass(struct air_val *v)+{+ if (!v || v->kind != AIR_VAL_PTR || !v->ptr.offset)+ return NULL;+ return nir_src_is_const(nir_src_for_ssa(v->ptr.offset)) ? NULL : v->ptr.offset;+}++/* A SLOT PASSED TO A CALL, and the parameters it costs.+ *+ * A slot NUMBER means the same thing in a callee as in a caller -- air_ctx.slots+ * is the SHADER's table, not the function's, the same way scratch_size is -- so+ * what cannot cross a function boundary is not the slot but the DEFS inside it.+ * A slot holds a tree of resolutions, and two kinds of thing in that tree belong+ * to the caller's impl: a plain value (`coreimage::Sampler`'s `float2 coord`),+ * and a pointer's byte offset when it is computed rather than constant. Both go+ * the way the run-time offset of a plain pointer argument goes+ * (ptr_offset_to_pass): one more nir_parameter the caller supplies as a call+ * source and the callee reads with load_param, which puts the def on the side of+ * the boundary it belongs to instead of hiding the crossing.+ *+ * It used to be refused outright, as a latent hole nothing asked for. Something+ * asks for it now: the four `disparity_refinement_*` of+ * CIPortraitBlurStitchableV3 are a stitched function whose first act is to+ * rebuild the `coreimage::Sampler` CoreImage flattened into four arguments, and+ * 126 modules of the image do the same thing (docs/air.md 10).+ *+ * THE TWO FUNCTIONS BELOW ARE A PAIR AND THE ORDER IS THE CONTRACT. slot_leaves+ * collects the caller's defs; rebind_slot_contents rebuilds the tree with the+ * callee's load_params in their place. They walk identically -- aggregate+ * elements in index order, then a pointer's offset, then a value -- and the+ * caller checks the two counts against each other rather than trusting them,+ * because a call one source short is a null nir_src that nir_instr_insert+ * dereferences while adding uses: a segfault, not a message. That is the same+ * rule, for the same reason, as the type_leaves/val_leaves pair below.+ */+#define AIR_MAX_SLOT_LEAVES 16++static bool+slot_leaves(struct air_ctx *c, struct air_val *v, nir_def **out, unsigned *n)+{+ if (!v)+ return true;+ switch (v->kind) {+ case AIR_VAL_AGG:+ for (unsigned i = 0; i < v->num_elems; i++)+ if (!slot_leaves(c, v->elems[i], out, n))+ return false;+ return true;+ case AIR_VAL_PTR:+ if (!v->ptr.offset || nir_src_is_const(nir_src_for_ssa(v->ptr.offset)))+ return true;+ break;+ case AIR_VAL_SSA:+ if (!v->def)+ return false;+ break;+ default:+ /* a pointer chosen at run time, or a cell nothing wrote: neither has a+ * form here, and both are named by the caller+ */+ return false;+ }+ if (*n >= AIR_MAX_SLOT_LEAVES)+ return false;+ if (out)+ out[*n] = v->kind == AIR_VAL_SSA ? v->def : v->ptr.offset;+ (*n)++;+ return true;+}++/* The caller's tree, rebuilt for the function being translated: `got` holds one+ * load_param per leaf slot_leaves found, in that order, and `k` walks it.+ */+static struct air_val *+rebind_slot_contents(struct air_ctx *c, struct air_val *v, nir_def **got,+ unsigned ngot, unsigned *k)+{+ if (!v)+ return v;+ switch (v->kind) {+ case AIR_VAL_AGG: {+ struct air_val *n = air_val_agg(c, v->num_elems);+ for (unsigned i = 0; i < v->num_elems; i++) {+ if (!v->elems[i])+ continue;+ if (!(n->elems[i] = rebind_slot_contents(c, v->elems[i], got, ngot, k)))+ return NULL;+ }+ return n;+ }+ case AIR_VAL_PTR: {+ struct air_ptr p = v->ptr;+ if (!p.offset)+ return v;+ nir_src src = nir_src_for_ssa(p.offset);+ if (nir_src_is_const(src)) {+ /* a constant of the caller is still a def of the caller's impl */+ p.offset = nir_imm_int(&c->b, nir_src_as_uint(src));+ return air_val_ptr(c, p);+ }+ if (*k >= ngot)+ return NULL;+ p.offset = got[(*k)++];+ return air_val_ptr(c, p);+ }+ case AIR_VAL_SSA:+ if (*k >= ngot)+ return NULL;+ return air_val_ssa(c, got[(*k)++]);+ default:+ return NULL;+ }+}++/* The defs a slot argument must pass, or n = 0 for an argument that is not one.+ * One function so that the three places that have to agree -- the parameter+ * count, the parameter declarations and the call's sources -- ask the same+ * question of the same value.+ */+static bool+slot_arg_leaves(struct air_ctx *c, struct air_val *v, nir_def **out, unsigned *n)+{+ *n = 0;+ if (!v || v->kind != AIR_VAL_PTR || v->ptr.class != AIR_PTR_SLOT)+ return true;+ if (v->ptr.binding >= c->num_slots)+ return false;+ if (v->ptr.offset &&+ (!nir_src_is_const(nir_src_for_ssa(v->ptr.offset)) ||+ nir_src_as_uint(nir_src_for_ssa(v->ptr.offset)) != 0))+ return false; /* a member of a slot, not the whole of it */+ return slot_leaves(c, c->slots[v->ptr.binding].val, out, n);+}++static struct air_val *+rebind_here(struct air_ctx *c, struct air_val *v, unsigned arg)+{+ if (!v)+ return v;+ if (v->kind == AIR_VAL_AGG) {+ struct air_val *n = air_val_agg(c, v->num_elems);+ for (unsigned i = 0; i < v->num_elems; i++) {+ if (!v->elems[i])+ continue;+ if (!(n->elems[i] = rebind_here(c, v->elems[i], arg)))+ return NULL;+ }+ return n;+ }+ if (v->kind != AIR_VAL_PTR || !v->ptr.offset)+ return v;+ nir_src src = nir_src_for_ssa(v->ptr.offset);+ if (!nir_src_is_const(src)) {+ air_error(c, "intrinsic: a call passing a pointer at an offset computed at run "+ "time as argument %u, whose value belongs to the caller's own "+ "function", arg);+ return NULL;+ }+ struct air_ptr p = v->ptr;+ p.offset = nir_imm_int(&c->b, nir_src_as_uint(src));+ return air_val_ptr(c, p);+}/* A function of the module, translated as a nir_function so that* nir_inline_functions can fold it into its callers afterwards.@@ -1401,18 +4149,52 @@ translate_callee(struct air_ctx *c, const struct air_function *f,* callee to its call site -- so it is not cached. Each of the four modules* that needs this has exactly one call site, counted beforehand.*- * A pointer carrying a byte offset is refused instead: that offset is an- * nir_def of the CALLER's impl, and using it here would be the same- * cross-function def/use that the globals used to cause.+ * A pointer carrying a byte offset that is a CONSTANT is bound the same way,+ * with the number copied into a def of this function's own impl+ * (rebind_here); one computed at run time is refused, because using the+ * caller's def here is the cross-function def/use that the globals used to+ * cause.*/bool specialised = false;for (unsigned i = 0; i < f->num_args; i++) {- bool ptr = argv && argv[i] && argv[i]->kind == AIR_VAL_PTR;- if (ptr && argv[i]->ptr.offset) {- air_error(c, "intrinsic: a call passing a pointer at a computed offset as "- "argument %u, which would use the caller's own value here", i);+ /* A struct holding a pointer counts as one too: CoreImage passes a+ * texture and a sampler as metal::texture2d and metal::sampler, which+ * are one-field structs wrapping the address-space-1 and -2 pointers.+ * Passing such a struct as a NIR value would hand the callee something+ * it then takes extractvalue of, and extractvalue of a value that is not+ * an aggregate here is exactly the error this used to produce.+ */+ bool ptr = argv && argv[i] && (argv[i]->kind == AIR_VAL_PTR ||+ val_holds_ptr(argv[i]));+ /* A constant byte offset is rebuilt in the callee's own builder+ * (rebind_here) and a run-time one on the pointer ITSELF is passed as a+ * parameter (ptr_offset_to_pass). What is left -- a run-time offset on a+ * pointer buried inside a struct -- is refused.+ */+ if (ptr && !ptr_offset_to_pass(argv[i]) && val_offset_is_computed(argv[i])) {+ air_error(c, "intrinsic: a call passing a pointer at an offset computed at run "+ "time as argument %u, whose value belongs to the caller's own "+ "function", i);return NULL;}+ /* The same reason, for the two pointers that are values: a chosen+ * pointer's condition is a def of the CALLER's impl, and a slot's+ * soundness rests on every use being in one block of one function.+ */+ if (argv && argv[i] && argv[i]->kind == AIR_VAL_PTR_SELECT) {+ air_error(c, "intrinsic: a call passing a pointer chosen at run time as "+ "argument %u", i);+ return NULL;+ }+ /* A SLOT used to be refused outright here, as a latent cross-function+ * def/use nothing asked for. Something does ask for it now -- the four+ * `disparity_refinement_*` of CIPortraitBlurStitchableV3 pass a+ * `coreimage::Sampler` they rebuilt in a local, and 126 modules of the+ * image do the same thing -- so it goes through rebind_here, which makes+ * the callee a slot of its own and copies the resolutions into it. What+ * is still refused, and by rebind_slot_contents rather than here, is a+ * slot holding a plain VALUE: that is a def of the caller's impl.+ */is_param[i] = !ptr;specialised |= ptr;}@@ -1429,7 +4211,48 @@ translate_callee(struct air_ctx *c, const struct air_function *f,const struct air_type *rt = f->fn_type->elem; /* FUNCTION: the return type */bool has_ret = rt && rt->kind != AIR_TYPE_VOID;- unsigned nparams = f->num_args + (has_ret ? 1 : 0);+ /* A struct return takes one out-parameter per member rather than one for the+ * whole value. Four of QuartzCore's vertex shaders are nothing but a wrapper+ * around such a function -- `tail call fastcc`, then extractvalue of each+ * field -- and before this they stopped at the call with "an aggregate where+ * a number belongs", which was emit_return inside the callee asking for the+ * returned struct as a register.+ */+ bool ret_struct = has_ret && rt->kind == AIR_TYPE_STRUCT;+ unsigned nret = 0;+ if (ret_struct) {+ for (unsigned i = 0; i < rt->num_members; i++)+ nret += ret_member_is_number(rt->members[i]);+ } else if (has_ret) {+ nret = 1;+ }+ /* one parameter per NUMBER in each argument, not one per argument: see+ * type_leaves. Counted before fn exists, because fn->params is sized from it.+ */+ unsigned nargparams = 0;+ for (unsigned i = 0; i < f->num_args; i++) {+ if (!is_param[i]) {+ if (ptr_offset_to_pass(argv ? argv[i] : NULL)) {+ nargparams++;+ continue;+ }+ /* a slot's value leaves, which are the caller's defs: slot_arg_leaves */+ unsigned ns = 0;+ if (!slot_arg_leaves(c, argv ? argv[i] : NULL, NULL, &ns)) {+ air_error(c, "intrinsic: a call passing a local that holds a pointer as "+ "argument %u, whose contents this cannot hand across a "+ "function boundary", i);+ return NULL;+ }+ nargparams += ns;+ continue;+ }+ unsigned n = 0;+ if (!type_leaves(c, f->args[i].type, NULL, &n))+ return NULL;+ nargparams += n;+ }+ unsigned nparams = nargparams + nret;nir_function *fn = nir_function_create(c->nir, f->base.name ? f->base.name : "callee");fn->num_params = nparams;fn->params = rzalloc_array(c->nir, nir_parameter, MAX2(nparams, 1));@@ -1437,11 +4260,19 @@ translate_callee(struct air_ctx *c, const struct air_function *f,unsigned idx = 0;const struct glsl_type *ret_glsl = NULL;- if (has_ret) {- unsigned bits = air_nir_bit_size(c, rt);- unsigned comps = air_nir_components(rt);- ret_glsl = glsl_vector_type(bits == 16 ? GLSL_TYPE_FLOAT16 :- bits == 64 ? GLSL_TYPE_UINT64 : GLSL_TYPE_UINT, comps);+ if (ret_struct) {+ for (unsigned i = 0; i < rt->num_members; i++) {+ if (!ret_member_is_number(rt->members[i]))+ continue;+ fn->params[idx++] = (nir_parameter){+ .num_components = 1, .bit_size = 32,+ .is_return = true,+ .mode = nir_var_function_temp,+ .type = param_glsl_type(c, rt->members[i]),+ };+ }+ } else if (has_ret) {+ ret_glsl = param_glsl_type(c, rt);fn->params[idx++] = (nir_parameter){.num_components = 1, .bit_size = 32,.is_return = true,@@ -1450,13 +4281,36 @@ translate_callee(struct air_ctx *c, const struct air_function *f,};}for (unsigned i = 0; i < f->num_args; i++) {- if (!is_param[i])+ if (!is_param[i]) {+ /* a bound pointer whose byte offset is a value: see ptr_offset_to_pass */+ if (ptr_offset_to_pass(argv ? argv[i] : NULL)) {+ fn->params[idx++] = (nir_parameter){ .num_components = 1, .bit_size = 32 };+ continue;+ }+ /* a slot's value leaves. Each parameter is declared from its own def's+ * shape and the source is passed unconverted, so the two match by+ * construction rather than by a width this would have to choose.+ */+ nir_def *sv[AIR_MAX_SLOT_LEAVES];+ unsigned ns = 0;+ if (!slot_arg_leaves(c, argv ? argv[i] : NULL, sv, &ns))+ return NULL; /* counted above, so this cannot fail; not assumed */+ for (unsigned k = 0; k < ns; k++)+ fn->params[idx++] = (nir_parameter){+ .num_components = (uint8_t)sv[k]->num_components,+ .bit_size = (uint8_t)sv[k]->bit_size,+ };continue;- const struct air_type *at = f->args[i].type;- fn->params[idx++] = (nir_parameter){- .num_components = (uint8_t)air_nir_components(at),- .bit_size = (uint8_t)air_nir_bit_size(c, at),- };+ }+ const struct air_type *lv[AIR_MAX_ARG_LEAVES];+ unsigned n = 0;+ if (!type_leaves(c, f->args[i].type, lv, &n))+ return NULL; /* counted above, so this cannot fail; not assumed */+ for (unsigned k = 0; k < n; k++)+ fn->params[idx++] = (nir_parameter){+ .num_components = (uint8_t)air_nir_components(lv[k]),+ .bit_size = (uint8_t)air_nir_bit_size(c, lv[k]),+ };}fn->num_params = idx; /* the pointers took no parameter */@@ -1472,6 +4326,12 @@ translate_callee(struct air_ctx *c, const struct air_function *f,struct hash_table *sv = c->values, *sb = c->blocks,*sp = c->phi_cursors, *sr = c->phi_regs;nir_deref_instr *saved_ret = c->ret_deref;+ nir_deref_instr **saved_ret_fields = c->ret_fields;+ unsigned saved_num_ret_fields = c->num_ret_fields;+ /* a constant of the caller is a def of the caller's impl, so the place+ * constants are built belongs to the function being translated, like the+ * values table above */+ nir_cursor saved_consts = c->const_cursor;c->fn = f;c->values = _mesa_pointer_hash_table_create(c->mem_ctx);@@ -1484,25 +4344,106 @@ translate_callee(struct air_ctx *c, const struct air_function *f,* structured by default, so a callee crashed on its first branch -- which* is what the corpus's reader count caught, 456 falling to 454. */c->b.impl->structured = false;+ c->const_cursor = nir_before_impl(c->b.impl);c->ret_deref = NULL;+ c->ret_fields = NULL;+ c->num_ret_fields = 0;/* this function's own globals, built in its own builder */materialise_globals(c);bool ok = true;- if (has_ret) {- /* The caller passes the deref's own def and the callee casts it back,- * which is the convention nir_functions.c documents: load_param is- * replaced by the call's source at inline time and nir_opt_deref removes- * the cast afterwards.- */+ /* The caller passes each deref's own def and the callee casts it back, which+ * is the convention nir_functions.c documents: load_param is replaced by the+ * call's source at inline time and nir_opt_deref removes the cast afterwards.+ */+ if (ret_struct) {+ c->ret_fields = rzalloc_array(c->mem_ctx, nir_deref_instr *,+ MAX2(rt->num_members, 1));+ c->num_ret_fields = rt->num_members;+ unsigned p = 0;+ for (unsigned i = 0; i < rt->num_members; i++) {+ if (!ret_member_is_number(rt->members[i]))+ continue;+ c->ret_fields[i] =+ nir_build_deref_cast(&c->b, nir_load_param(&c->b, p++),+ nir_var_function_temp,+ param_glsl_type(c, rt->members[i]), 0);+ }+ } else if (has_ret) {c->ret_deref = nir_build_deref_cast(&c->b, nir_load_param(&c->b, 0),nir_var_function_temp, ret_glsl, 0);}- unsigned pidx = has_ret ? 1 : 0;+ unsigned pidx = nret;for (unsigned i = 0; i < f->num_args && ok; i++) {if (!is_param[i]) {- air_set(c, &f->args[i], argv[i]); /* the caller's own resolution */+ if (ptr_offset_to_pass(argv[i])) {+ /* the binding from the caller, the offset from a parameter */+ struct air_ptr pp = argv[i]->ptr;+ pp.offset = nir_load_param(&c->b, pidx++);+ air_set(c, &f->args[i], air_val_ptr(c, pp));+ continue;+ }+ /* A SLOT: a new slot of the same type for this function, filled with+ * the caller's resolutions rebuilt here and its values read out of the+ * parameters declared above, in the order slot_leaves walked them.+ */+ if (argv[i] && argv[i]->kind == AIR_VAL_PTR &&+ argv[i]->ptr.class == AIR_PTR_SLOT) {+ nir_def *sv[AIR_MAX_SLOT_LEAVES];+ unsigned ns = 0;+ if (!slot_arg_leaves(c, argv[i], sv, &ns)) {+ ok = false;+ break;+ }+ for (unsigned k = 0; k < ns; k++)+ sv[k] = nir_load_param(&c->b, pidx++);+ unsigned used = 0;+ const struct air_type *st = c->slots[argv[i]->ptr.binding].type;+ struct air_val *inner =+ rebind_slot_contents(c, c->slots[argv[i]->ptr.binding].val,+ sv, ns, &used);+ if (used != ns || (!inner && c->slots[argv[i]->ptr.binding].val)) {+ air_error(c, "intrinsic: a call passing a local that holds a pointer as "+ "argument %u, rebuilt with %u of its %u parameters",+ i, used, ns);+ ok = false;+ break;+ }+ struct air_ptr sp = {+ .class = AIR_PTR_SLOT,+ .binding = new_slot(c, st, NULL),+ .pointee = argv[i]->ptr.pointee ? argv[i]->ptr.pointee : st,+ };+ c->slots[sp.binding].val = inner;+ air_set(c, &f->args[i], air_val_ptr(c, sp));+ continue;+ }+ /* the caller's own resolution, with its offsets rebuilt here */+ struct air_val *bound = rebind_here(c, argv[i], i);+ if (!bound && argv[i]) {+ ok = false;+ break;+ }+ air_set(c, &f->args[i], bound);+ continue;+ }+ const struct air_type *at = f->args[i].type;+ if (at->kind == AIR_TYPE_STRUCT || at->kind == AIR_TYPE_ARRAY) {+ nir_def *lv[AIR_MAX_ARG_LEAVES];+ unsigned need = 0, taken = 0;+ if (!type_leaves(c, at, NULL, &need)) {+ ok = false;+ break;+ }+ for (unsigned k = 0; k < need; k++)+ lv[k] = nir_load_param(&c->b, pidx++);+ struct air_val *v = val_from_leaves(c, at, lv, &taken);+ if (!v) {+ ok = false;+ break;+ }+ air_set(c, &f->args[i], v);continue;}air_set(c, &f->args[i], air_val_ssa(c, nir_load_param(&c->b, pidx++)));@@ -1516,7 +4457,10 @@ translate_callee(struct air_ctx *c, const struct air_function *f,c->blocks = sb;c->phi_cursors = sp;c->phi_regs = sr;+ c->const_cursor = saved_consts;c->ret_deref = saved_ret;+ c->ret_fields = saved_ret_fields;+ c->num_ret_fields = saved_num_ret_fields;return ok ? fn : NULL;}@@ -1565,36 +4509,174 @@ air_call_in_module(struct air_ctx *c, const struct air_inst *in, const char *namconst struct air_type *rt = f->fn_type->elem; /* FUNCTION: the return type */bool has_ret = rt && rt->kind != AIR_TYPE_VOID;+ bool ret_struct = has_ret && rt->kind == AIR_TYPE_STRUCT;nir_variable *tmp = NULL;+ /* One local per member for a struct return, in the member order+ * translate_callee made its parameters from; the ones with no parameter stay+ * NULL and the aggregate's element for them stays NULL too, which is the+ * same thing an extractvalue of an undef field already reads.+ */+ nir_variable **fields = ret_struct ?+ rzalloc_array(c->mem_ctx, nir_variable *, MAX2(rt->num_members, 1)) : NULL;nir_call_instr *call = nir_call_instr_create(c->nir, fn);unsigned idx = 0;- if (has_ret) {- unsigned bits = air_nir_bit_size(c, rt);- unsigned comps = air_nir_components(rt);- const struct glsl_type *t =- glsl_vector_type(bits == 16 ? GLSL_TYPE_FLOAT16 :- bits == 64 ? GLSL_TYPE_UINT64 : GLSL_TYPE_UINT, comps);- tmp = nir_local_variable_create(c->b.impl, t, "callret");+ if (ret_struct) {+ for (unsigned i = 0; i < rt->num_members; i++) {+ if (!ret_member_is_number(rt->members[i]))+ continue;+ fields[i] = nir_local_variable_create(c->b.impl,+ param_glsl_type(c, rt->members[i]),+ "callret");+ call->params[idx++] =+ nir_src_for_ssa(&nir_build_deref_var(&c->b, fields[i])->def);+ }+ } else if (has_ret) {+ tmp = nir_local_variable_create(c->b.impl, param_glsl_type(c, rt), "callret");call->params[idx++] = nir_src_for_ssa(&nir_build_deref_var(&c->b, tmp)->def);}- /* operand 0 is the callee; the arguments follow it */+ /* operand 0 is the callee; the arguments follow it. One source per NUMBER+ * inside each argument, in the order translate_callee made its parameters+ * (type_leaves): a struct or an array passed by value is not one NIR value.+ */for (unsigned i = 0; i < f->num_args; i++) {- if (!is_param[i])+ if (!is_param[i]) {+ /* a bound pointer's byte offset, if it is a value: ptr_offset_to_pass.+ * Narrowed to the 32 bits air_ptr documents an offset as and the+ * parameter declares, because a gep of an i64 index can leave a wider+ * def behind and a call source must match its parameter exactly.+ */+ nir_def *off = ptr_offset_to_pass(argv[i]);+ if (off) {+ call->params[idx++] = nir_src_for_ssa(nir_u2u32(&c->b, off));+ continue;+ }+ /* a slot's value leaves, in the order translate_callee declared them+ * (slot_leaves), and unconverted because each parameter was declared+ * from the def that fills it+ */+ nir_def *sv[AIR_MAX_SLOT_LEAVES];+ unsigned ns = 0;+ if (!slot_arg_leaves(c, argv[i], sv, &ns))+ return air_error(c, "intrinsic: a call passing a local that holds a pointer "+ "as argument %u, whose contents this cannot hand across "+ "a function boundary", i);+ for (unsigned k = 0; k < ns; k++)+ call->params[idx++] = nir_src_for_ssa(sv[k]);continue;- nir_def *a = air_get_ssa(c, in->operands[i + 1]);- if (!a)- return air_error(c, "intrinsic: argument %u of %s is not a value this can pass",- i, name);- call->params[idx++] = nir_src_for_ssa(a);+ }+ nir_def *lv[AIR_MAX_ARG_LEAVES];+ unsigned n = 0;+ if (!val_leaves(c, f->args[i].type, argv[i], lv, &n))+ return false; /* val_leaves has said what the argument was */+ for (unsigned k = 0; k < n; k++)+ call->params[idx++] = nir_src_for_ssa(lv[k]);}+ /* The two sides derive the count from the same type with the same function,+ * so they agree -- and the agreement is checked rather than trusted, because+ * a call left one source short is a null nir_src that nir_instr_insert+ * dereferences while adding uses: a segfault, not a message.+ */+ if (idx != call->num_params)+ return air_error(c, "intrinsic: a call of %s built with %u sources for %u "+ "parameters", name, idx, call->num_params);nir_builder_instr_insert(&c->b, &call->instr);- if (has_ret)+ if (ret_struct) {+ struct air_val *agg = air_val_agg(c, rt->num_members);+ for (unsigned i = 0; i < rt->num_members; i++)+ if (fields[i])+ agg->elems[i] = air_val_ssa(c, nir_load_var(&c->b, fields[i]));+ air_set(c, &in->base, agg);+ } else if (has_ret) {air_set(c, &in->base, air_val_ssa(c, nir_load_var(&c->b, tmp)));+ }return true;}+/* The air.visible_input node for argument i, or NULL. The nodes carry a+ * parameter number as their first operand rather than being in order, so they+ * are searched by it.+ */+static const struct air_md *+arg_md(const struct air_md *inputs, unsigned i)+{+ for (unsigned k = 0; k < (inputs ? inputs->num_ops : 0); k++) {+ const struct air_md *n = inputs->ops[k];+ uint64_t which;+ if (n && md_int(n, 0, &which) && which == i)+ return n;+ }+ return NULL;+}++/* A texture, a sampler or a constant buffer: an argument that is BOUND rather+ * than passed, at the binding the caller names.+ *+ * Eleven of the corpus's library functions are CoreImage's sampling helpers,+ * and they take what CoreImage hands a stitched function: a texture, a sampler+ * and a transform. Apple passes the first two as one-field structs named+ * metal::texture2d and metal::sampler -- the field being exactly the+ * address-space-1 or -2 pointer an entry point's texture or sampler argument+ * is -- and the transform as a bare constant-space pointer. A texture is not a+ * value in this frontend (section 7.4), so each of the three resolves to a+ * binding and the runtime binds a texture, a sampler and a constant buffer+ * there.+ *+ * Keyed on the struct's name, which is Apple's and is in these modules, and+ * not on "a struct with one pointer in it": the second would quietly accept+ * anything shaped alike.+ *+ * *kind is what the argument turned out to be, so a caller that was told by a+ * graph which resource an argument is can check that against the type instead+ * of trusting either one alone. Returns NULL without an error when the type is+ * none of the three -- then the argument is a value and the caller supplies it.+ */+static struct air_val *+bind_resource_argument(struct air_ctx *c, const struct air_type *t, unsigned arg,+ const struct air_md *md, unsigned binding,+ enum air_resource_kind *kind)+{+ const char *sname = t->kind == AIR_TYPE_STRUCT ? t->name : NULL;+ bool is_tex = sname && !strncmp(sname, "struct.metal::texture", 21);+ bool is_samp = sname && !strcmp(sname, "struct.metal::sampler");++ if (is_tex || is_samp) {+ if (t->num_members != 1) {+ air_error(c, "library: %s has %u members, not the one pointer this expects",+ sname, t->num_members);+ return NULL;+ }+ struct air_ptr p = {+ .class = is_tex ? AIR_PTR_TEXTURE : AIR_PTR_SAMPLER,+ .binding = binding,+ };+ *kind = is_tex ? AIR_RESOURCE_TEXTURE : AIR_RESOURCE_SAMPLER;+ add_resource(c, *kind, binding, 0, true, false, arg, md);+ /* the value is the struct, so the pointer goes inside an aggregate of+ * one: the body reaches it with extractvalue+ */+ struct air_val *agg = air_val_agg(c, 1);+ agg->elems[0] = air_val_ptr(c, p);+ return agg;+ }++ if (t->kind == AIR_TYPE_POINTER && t->addrspace == 2) {+ /* the transform, as a constant buffer the runtime fills. With the+ * argument's own metadata, so the resource carries Apple's name for it --+ * "float4x4" for the transform these functions take, which is what lets a+ * harness put something meaningful there rather than the byte pattern.+ */+ struct air_ptr p = { .class = AIR_PTR_UBO, .binding = binding, .pointee = t->elem };+ *kind = AIR_RESOURCE_CONSTANT_BUFFER;+ add_resource(c, *kind, binding, 256, true, false, arg, md);+ return air_val_ptr(c, p);+ }++ *kind = AIR_RESOURCE_DEVICE_BUFFER; /* not a bound resource: a value */+ return NULL;+}+/* An air.visible library function, inside an entry point we generate for it.** A library function is not a shader. Metal links one into a caller at@@ -1619,7 +4701,8 @@ air_call_in_module(struct air_ctx *c, const struct air_inst *in, const char *nam* points.*/static bool-wrap_library_function(struct air_ctx *c, const struct air_md *out_md)+wrap_library_function(struct air_ctx *c, const struct air_md *out_md,+ const struct air_md *in_md){const struct air_function *f = c->fn;@@ -1637,16 +4720,54 @@ wrap_library_function(struct air_ctx *c, const struct air_md *out_md)MAX2(f->num_args, 1));bool *is_param = rzalloc_array(c->mem_ctx, bool, MAX2(f->num_args, 1));+ /* Eleven of the corpus's library functions are CoreImage's sampling helpers,+ * and they take what CoreImage hands a stitched function: a texture, a+ * sampler and a transform. Apple passes the first two as one-field structs+ * named metal::texture2d and metal::sampler -- the field being exactly the+ * address-space-1 or -2 pointer an entry point's texture or sampler+ * argument is -- and the transform as a bare constant-space pointer. Those+ * three shapes are bound rather than loaded: a texture is not a value in+ * this frontend (section 7.4), so the wrapper resolves each to a binding+ * and the harness binds its own texture, sampler and constant buffer there.+ *+ * Keyed on the struct's name, which is Apple's and is in these modules, and+ * not on "a struct with one pointer in it": the second would quietly accept+ * anything shaped alike.+ */+ unsigned next_tex = 0, next_samp = 0, next_ubo = 0;for (unsigned i = 0; i < f->num_args; i++) {const struct air_type *t = f->args[i].type;+ const char *sname = t->kind == AIR_TYPE_STRUCT ? t->name : NULL;+ bool is_tex = sname && !strncmp(sname, "struct.metal::texture", 21);+ bool is_samp = sname && !strcmp(sname, "struct.metal::sampler");+ bool is_ubo = t->kind == AIR_TYPE_POINTER && t->addrspace == 2;++ if (is_tex || is_samp || is_ubo) {+ /* The wrapper numbers these itself, in the order the arguments come:+ * nothing tells it otherwise, where a stitched graph is given each+ * one's bind index by the runtime (stitch_graph).+ */+ enum air_resource_kind got;+ argv[i] = bind_resource_argument(c, t, i, arg_md(in_md, i),+ is_tex ? next_tex : is_samp ? next_samp+ : next_ubo, &got);+ if (!argv[i])+ return false; /* bind_resource_argument has said why */+ next_tex += is_tex;+ next_samp += is_samp;+ next_ubo += is_ubo;+ is_param[i] = false;+ continue;+ }+const struct air_type *e = t->kind == AIR_TYPE_VECTOR ? t->elem : t;bool numeric = e && (e->kind == AIR_TYPE_FLOAT || e->kind == AIR_TYPE_HALF ||e->kind == AIR_TYPE_INT);if (!numeric || (t->kind != AIR_TYPE_VECTOR && t->kind != AIR_TYPE_FLOAT &&t->kind != AIR_TYPE_HALF && t->kind != AIR_TYPE_INT))- return air_error(c, "library: argument %u is neither a scalar nor a vector, "- "and this wrapper passes each argument in a 16-byte slot "- "of one buffer", i);+ return air_error(c, "library: argument %u is none of a scalar, a vector, a "+ "metal::texture, a metal::sampler or a constant pointer, "+ "which is all this wrapper can supply", i);if (air_nir_components(t) * air_nir_bit_size(c, t) > 16 * 8)return air_error(c, "library: argument %u does not fit the wrapper's 16-byte ""slot", i);@@ -1682,21 +4803,527 @@ wrap_library_function(struct air_ctx *c, const struct air_md *out_md)nir_call_instr *call = nir_call_instr_create(c->nir, fn);unsigned idx = 0;call->params[idx++] = nir_src_for_ssa(&nir_build_deref_var(&c->b, tmp)->def);- for (unsigned i = 0; i < f->num_args; i++)- call->params[idx++] = nir_src_for_ssa(argv[i]->def);+ for (unsigned i = 0; i < f->num_args; i++) {+ /* only the arguments translate_callee made parameters: a texture, a+ * sampler or a constant pointer was bound into the callee instead, and+ * has no SSA def to pass. Passing one anyway inserts a call whose source+ * has no value, which NIR dereferences while adding uses -- a segfault+ * inside nir_instr_insert rather than a message.+ */+ if (!is_param[i]) {+ /* no argument this wrapper supplies has a run-time offset -- they are+ * bindings and 16-byte slots -- but the parameter would exist if one+ * did, so the source is pushed here rather than left to the count+ * check below to catch+ */+ nir_def *off = ptr_offset_to_pass(argv[i]);+ if (off) {+ call->params[idx++] = nir_src_for_ssa(nir_u2u32(&c->b, off));+ continue;+ }+ /* and no argument this wrapper supplies is a slot either, for the same+ * reason: a slot is something a module's own code allocates+ */+ nir_def *sv[AIR_MAX_SLOT_LEAVES];+ unsigned ns = 0;+ if (!slot_arg_leaves(c, argv[i], sv, &ns))+ return air_error(c, "library: argument %u is a local holding a pointer "+ "whose contents cannot be passed", i);+ for (unsigned k = 0; k < ns; k++)+ call->params[idx++] = nir_src_for_ssa(sv[k]);+ continue;+ }+ nir_def *a = argv[i] ? argv[i]->def : NULL;+ if (!a)+ return air_error(c, "library: argument %u has no value to pass", i);+ call->params[idx++] = nir_src_for_ssa(a);+ }+ /* Every argument here is a scalar or a vector -- the loop above refuses+ * anything else -- so each takes exactly one parameter and this holds. It is+ * checked because translate_callee decides the parameters and this decides+ * the sources, and the two would otherwise agree only by inspection.+ */+ if (idx != call->num_params)+ return air_error(c, "library: a call built with %u sources for %u parameters",+ idx, call->num_params);nir_builder_instr_insert(&c->b, &call->instr);struct air_ptr out = { .class = AIR_PTR_SSBO, .binding = 1, .pointee = rt };- if (!air_store_ptr(c, out, nir_load_var(&c->b, tmp), 16))+ if (!air_store_ptr(c, out, nir_load_var(&c->b, tmp), 16))+ return false;++ /* the generated block ends the way a translated one does: this impl is+ * unstructured until nir_lower_goto_ifs runs over it+ */+ nir_goto(&c->b, c->b.impl->end_block);+ return true;+}++/* -- function stitching ----------------------------------------------------+ *+ * A graph of air.visible functions compiled into one entry point, which is+ * what Metal does with these functions and what CoreImage actually uses+ * (newLibraryWithStitchedDescriptor:, docs/air.md section 5, "The stitching+ * ABI, before any of it is built"). The wrapper above runs one body; this runs+ * a DAG of them, each node's arguments coming from another node's result, from+ * an argument of the stitched function, or from a bound resource.+ *+ * A node's arguments are the callee's OWN arguments, one for one, and not the+ * CoreImage-level ones: `_pf_sobelH` takes a `coreimage::Sampler` in Apple's+ * source and five arguments in the module, and the flattening between them is+ * CoreImage's -- its StitchedFunctionDescriptor::MetalArgumentInfo has+ * `ci_sampler_builder_args_start` and `..._end`, a RANGE of Metal arguments for+ * one of its own. So the graph this takes is the Metal-level one, which is the+ * level MTLFunctionStitchingFunctionNode.arguments is at.+ */+static const struct air_function *+visible_function_of(const struct air_module *m, const struct air_md **entry)+{+ if (module_stage(m, entry) != MESA_SHADER_NONE)+ return NULL;+ const struct air_md *e = *entry;+ if (!e->num_ops || !e->ops[0] || e->ops[0]->kind != AIR_MD_VALUE ||+ e->ops[0]->value->kind != AIR_VALUE_FUNCTION)+ return NULL;+ return (const struct air_function *)e->ops[0]->value;+}++/* One function a graph's nodes can name: read once, however many nodes call+ * it, so that two nodes calling the same function meet the same+ * air_function and translate_callee's cache applies to the second.+ */+struct stitch_fn {+ const struct air_function *f;+ const struct air_md *entry; /* its air.visible node */+ struct air_module *m;+ struct air_global_plan *plan; /* its module's globals, planned */+};++static bool+stitch_prepare(struct air_ctx *c, struct stitch_fn *sf, const char *name,+ struct stitch_fn *done, unsigned num_done)+{+ /* A function two nodes call is read once: the second node then meets the+ * same air_function, so translate_callee's cache gives it the same+ * nir_function and the body is translated once however often it is called.+ */+ for (unsigned i = 0; i < num_done; i++) {+ if (done[i].f && done[i].f->base.name && !strcmp(done[i].f->base.name, name)) {+ *sf = done[i];+ return true;+ }+ }++ const struct air_stitch_function *src = NULL;+ for (unsigned i = 0; i < c->options->num_stitch_functions; i++) {+ const struct air_stitch_function *k = &c->options->stitch_functions[i];+ if (k->name && name && !strcmp(k->name, name)) {+ src = k;+ break;+ }+ }+ if (!src)+ return air_error(c, "stitch: the graph names %s, which is not one of the %u "+ "functions given to stitch from", name ? name : "(nothing)",+ c->options->num_stitch_functions);++ const char *why = NULL;+ sf->m = air_module_read(c->mem_ctx, src->bitcode, src->size, &why);+ if (!sf->m)+ return air_error(c, "stitch: %s: reader: %s", name, why ? why : "?");+ sf->f = visible_function_of(sf->m, &sf->entry);+ if (!sf->f)+ return air_error(c, "stitch: %s is not an air.visible function of its module, so "+ "it is not a function a graph can call", name);+ if (sf->f->base.name && strcmp(sf->f->base.name, name))+ return air_error(c, "stitch: the module given for %s holds %s instead", name,+ sf->f->base.name);++ /* its module's globals, planned in its own module: plan_globals indexes by+ * the module's own global numbering, so the plan belongs to the module and+ * not to the shader, and translate_callee materialises from whichever plan+ * is current when it runs.+ */+ const struct air_module *saved_m = c->m;+ struct air_global_plan *saved_plan = c->global_plan;+ c->m = sf->m;+ c->global_plan = NULL;+ plan_globals(c);+ sf->plan = c->global_plan;+ if (!c->error[0])+ run_static_initializers(c); /* in the entry point, before any call */+ c->m = saved_m;+ c->global_plan = saved_plan;+ return !c->error[0];+}++/* The air.visible_input node list of a function, which arg_md searches. */+static const struct air_md *+stitch_inputs_md(const struct stitch_fn *sf)+{+ return sf->entry->num_ops > 2 ? sf->entry->ops[2] : NULL;+}++/* Its result's metadata: air.visible_output, which carries Apple's own name+ * for the type and is what the result buffer is labelled with.+ */+static const struct air_md *+stitch_output_md(const struct stitch_fn *sf)+{+ const struct air_md *rn = sf->entry->num_ops > 1 ? sf->entry->ops[1] : NULL;+ return rn && rn->num_ops ? rn->ops[0] : NULL;+}++/* The result type of a node, which is what an argument fed from it receives. */+static const struct air_type *+stitch_result_type(const struct stitch_fn *sf)+{+ return sf->f->fn_type->elem;+}++static bool+stitch_graph(struct air_ctx *c)+{+ const struct air_stitch_graph *g = c->options->stitch;+ if (!g->num_nodes)+ return air_error(c, "stitch: a graph with no nodes");+ if (g->output_node >= g->num_nodes)+ return air_error(c, "stitch: the output node is %u of %u nodes", g->output_node,+ g->num_nodes);++ struct stitch_fn *fns = rzalloc_array(c->mem_ctx, struct stitch_fn, g->num_nodes);+ nir_def **value = rzalloc_array(c->mem_ctx, nir_def *, g->num_nodes);++ /* The stitched function's own arguments, and the resources its nodes bind.+ * The argument slots are the library wrapper's -- argument i at byte 16*i of+ * device buffer 0, the result in device buffer 1 -- on purpose: a one-node+ * graph then has to produce exactly the words the wrapper produces for the+ * same function, which is a comparison that already has a reference.+ */+ unsigned num_inputs = 0;+ for (unsigned n = 0; n < g->num_nodes; n++)+ for (unsigned a = 0; a < g->nodes[n].num_args; a++)+ if (g->nodes[n].args[a].kind == AIR_STITCH_INPUT)+ num_inputs = MAX2(num_inputs, g->nodes[n].args[a].index + 1);++ add_resource(c, AIR_RESOURCE_DEVICE_BUFFER, 0, 16 * MAX2(num_inputs, 1),+ true, false, ~0u, NULL);++ for (unsigned n = 0; n < g->num_nodes; n++) {+ const struct air_stitch_node *node = &g->nodes[n];+ if (!stitch_prepare(c, &fns[n], node->function, fns, n))+ return false;+ const struct air_function *f = fns[n].f;++ if (node->num_args != f->num_args)+ return air_error(c, "stitch: node %u gives %s %u arguments and it takes %u",+ n, node->function, node->num_args, f->num_args);++ struct air_val **argv = rzalloc_array(c->mem_ctx, struct air_val *,+ MAX2(f->num_args, 1));+ bool *is_param = rzalloc_array(c->mem_ctx, bool, MAX2(f->num_args, 1));++ for (unsigned a = 0; a < f->num_args; a++) {+ const struct air_type *t = f->args[a].type;+ struct air_stitch_arg src = node->args[a];+ enum air_resource_kind want;++ switch (src.kind) {+ case AIR_STITCH_TEXTURE:+ case AIR_STITCH_SAMPLER:+ case AIR_STITCH_BUFFER: {+ want = src.kind == AIR_STITCH_TEXTURE ? AIR_RESOURCE_TEXTURE :+ src.kind == AIR_STITCH_SAMPLER ? AIR_RESOURCE_SAMPLER :+ AIR_RESOURCE_CONSTANT_BUFFER;+ enum air_resource_kind got;+ argv[a] = bind_resource_argument(c, t, a,+ arg_md(stitch_inputs_md(&fns[n]), a),+ src.index, &got);+ if (!argv[a])+ return c->error[0] ? false :+ air_error(c, "stitch: node %u binds argument %u of %s as a resource "+ "and its type is not a texture, a sampler or a "+ "constant pointer", n, a, node->function);+ /* The graph says which resource it is and the module's own type+ * says which it can be. Both are checked against each other+ * because a bind index that reached the wrong KIND of resource+ * would still compile and still read bytes.+ */+ if (got != want)+ return air_error(c, "stitch: node %u binds argument %u of %s as a %s and "+ "the module declares it a %s", n, a, node->function,+ want == AIR_RESOURCE_TEXTURE ? "texture" :+ want == AIR_RESOURCE_SAMPLER ? "sampler" : "buffer",+ got == AIR_RESOURCE_TEXTURE ? "texture" :+ got == AIR_RESOURCE_SAMPLER ? "sampler" : "buffer");+ is_param[a] = false;+ continue;+ }++ case AIR_STITCH_NODE: {+ if (src.index >= n)+ return air_error(c, "stitch: node %u takes the result of node %u, which "+ "is not an earlier node: a graph is given in "+ "dependency order", n, src.index);+ const struct air_type *rt = stitch_result_type(&fns[src.index]);+ if (air_nir_components(rt) != air_nir_components(t) ||+ air_nir_bit_size(c, rt) != air_nir_bit_size(c, t))+ return air_error(c, "stitch: node %u takes node %u's result into an "+ "argument of a different shape (%ux%u into %ux%u)",+ n, src.index, air_nir_components(rt),+ air_nir_bit_size(c, rt), air_nir_components(t),+ air_nir_bit_size(c, t));+ argv[a] = air_val_ssa(c, value[src.index]);+ is_param[a] = true;+ continue;+ }++ case AIR_STITCH_INPUT: {+ const struct air_type *e = t->kind == AIR_TYPE_VECTOR ? t->elem : t;+ bool numeric = e && (e->kind == AIR_TYPE_FLOAT || e->kind == AIR_TYPE_HALF ||+ e->kind == AIR_TYPE_INT);+ if (!numeric || (t->kind != AIR_TYPE_VECTOR && t->kind != AIR_TYPE_FLOAT &&+ t->kind != AIR_TYPE_HALF && t->kind != AIR_TYPE_INT))+ return air_error(c, "stitch: node %u takes the stitched function's "+ "argument %u into argument %u of %s, which is "+ "neither a scalar nor a vector", n, src.index, a,+ node->function);+ if (air_nir_components(t) * air_nir_bit_size(c, t) > 16 * 8)+ return air_error(c, "stitch: argument %u does not fit the 16-byte slot "+ "the stitched function's arguments are read from",+ src.index);+ struct air_ptr p = { .class = AIR_PTR_SSBO, .binding = 0, .pointee = t };+ nir_def *v = air_load_ptr(c, air_ptr_add(c, p, nir_imm_int(&c->b,+ 16 * src.index)),+ t, 16);+ if (!v)+ return false;+ argv[a] = air_val_ssa(c, v);+ is_param[a] = true;+ continue;+ }++ case AIR_STITCH_GRID_POSITION:+ /* In the API and not implemented, which is not the same as+ * forgotten: no case here dispatches more than one thread, so a+ * grid position would be translated and never measured, and an+ * untested path is worse than a refusal.+ */+ return air_error(c, "stitch: node %u takes thread_position_in_grid, which "+ "is not implemented", n);+ }+ return air_error(c, "stitch: node %u argument %u has an unknown source", n, a);+ }++ /* its module is current while its body is translated: the globals the+ * body reads are at the offsets ITS plan decided+ */+ const struct air_module *saved_m = c->m;+ struct air_global_plan *saved_plan = c->global_plan;+ c->m = fns[n].m;+ c->global_plan = fns[n].plan;+ nir_function *fn = translate_callee(c, f, argv, is_param);+ c->m = saved_m;+ c->global_plan = saved_plan;+ if (!fn)+ return false; /* translate_callee has said why */++ const struct air_type *rt = stitch_result_type(&fns[n]);+ if (!rt || rt->kind == AIR_TYPE_VOID)+ return air_error(c, "stitch: node %u calls %s, which returns nothing, so no "+ "other node can take its result", n, node->function);+ const struct air_type *re = rt->kind == AIR_TYPE_VECTOR ? rt->elem : rt;+ if (!re || (re->kind != AIR_TYPE_FLOAT && re->kind != AIR_TYPE_HALF &&+ re->kind != AIR_TYPE_INT))+ return air_error(c, "stitch: node %u calls %s, whose result is neither a "+ "scalar nor a vector", n, node->function);++ unsigned bits = air_nir_bit_size(c, rt);+ unsigned comps = air_nir_components(rt);+ const struct glsl_type *gt =+ glsl_vector_type(bits == 16 ? GLSL_TYPE_FLOAT16 :+ bits == 64 ? GLSL_TYPE_UINT64 : GLSL_TYPE_UINT, comps);+ nir_variable *tmp = nir_local_variable_create(c->b.impl, gt, "node");++ nir_call_instr *call = nir_call_instr_create(c->nir, fn);+ unsigned idx = 0;+ call->params[idx++] = nir_src_for_ssa(&nir_build_deref_var(&c->b, tmp)->def);+ for (unsigned a = 0; a < f->num_args; a++) {+ if (!is_param[a]) {+ /* as in the wrapper above: a bound pointer takes no parameter+ * unless its byte offset is a value (ptr_offset_to_pass)+ */+ nir_def *off = ptr_offset_to_pass(argv[a]);+ if (off) {+ call->params[idx++] = nir_src_for_ssa(nir_u2u32(&c->b, off));+ continue;+ }+ nir_def *sv[AIR_MAX_SLOT_LEAVES];+ unsigned ns = 0;+ if (!slot_arg_leaves(c, argv[a], sv, &ns))+ return air_error(c, "stitch: node %u argument %u is a local holding a "+ "pointer whose contents cannot be passed", n, a);+ for (unsigned k = 0; k < ns; k++)+ call->params[idx++] = nir_src_for_ssa(sv[k]);+ continue;+ }+ nir_def *v = argv[a] ? argv[a]->def : NULL;+ if (!v)+ return air_error(c, "stitch: node %u argument %u has no value to pass", n, a);+ call->params[idx++] = nir_src_for_ssa(v);+ }+ /* as in the wrapper above: one parameter per argument holds here because+ * every argument is a scalar or a vector, and is checked anyway+ */+ if (idx != call->num_params)+ return air_error(c, "stitch: node %u built with %u sources for %u parameters",+ n, idx, call->num_params);+ nir_builder_instr_insert(&c->b, &call->instr);+ value[n] = nir_load_var(&c->b, tmp);+ }++ /* the result buffer, declared once the output node's own metadata is in+ * hand: it carries Apple's air.arg_type_name for the result, which is how a+ * harness knows what the bytes coming back are+ */+ const struct air_type *rt = stitch_result_type(&fns[g->output_node]);+ add_resource(c, AIR_RESOURCE_DEVICE_BUFFER, 1, 16, false, true, ~0u,+ stitch_output_md(&fns[g->output_node]));+ struct air_ptr out = { .class = AIR_PTR_SSBO, .binding = 1, .pointee = rt };+ if (!air_store_ptr(c, out, value[g->output_node], 16))return false;- /* the generated block ends the way a translated one does: this impl is- * unstructured until nir_lower_goto_ifs runs over it- */+ c->info->entry_point = g->name ? g->name : "stitched";+ c->nir->info.name = ralloc_strdup(c->nir, c->info->entry_point);nir_goto(&c->b, c->b.impl->end_block);return true;}+/* A phi of POINTERS, resolved rather than carried in a register.+ *+ * A pointer here is *(which resource, which byte offset)* decided while+ * translating, so it has no bits to put in a register and a phi of two of them+ * is not a value NIR can hold. It need not be: LLVM duplicates a+ * `getelementptr` onto two paths and phis the copies back, so both operands+ * name the same member -- 207 of the pointer phis in Resident Evil 4's library+ * are exactly that, and not one names two different members.+ *+ * Refused when they name different resources. This COULD go through the+ * AIR_VAL_PTR_SELECT machinery that already loads both sides and selects the+ * values, but nothing in reach needs it, a texture or a sampler is not a value+ * that path can load, and an untested path is worse than a refusal.+ *+ * ONE RESOURCE AT DIFFERENT OFFSETS is not that, and is carried. Its offset is+ * a 32-bit number like any phi's value, so it gets a register: each edge+ * stores the offset its operand has, and the phi is the resource at the+ * register's value. That is what LLVM's duplicated GEP needs when the copies+ * are two defs (mediaserverd's kernelComputeStage1: the same member of the+ * same buffer, computed in each of two blocks, which same_ptr cannot prove+ * equal), and what a pointer stepped round a loop needs -- an operand over a+ * back edge is then stored when the edge is, and checked to name the same+ * resource there (the phi stores below), not assumed to.+ *+ * `live` is why this is sound rather than optimistic: the blocks are+ * walked in reverse post order, so an operand coming in over a BACK edge has+ * not been translated yet. Skipping an operand merely because it is missing+ * would accept a loop-carried pointer by ignoring the edge that changes it, so+ * a missing operand from a live block is a refusal and only an edge the entry+ * cannot take at all is passed over.+ *+ * `live` is narrower than CFG reachability and that is the point: a block+ * reached only through a branch whose condition is decided while translating+ * is not entered, so a phi between two different resources has ONE operand+ * and resolves. That is how a shader specialised on its function constants+ * gets through here (translate_body, and docs/air.md section 3), and nothing+ * about this function had to change for it -- only what it is told is live.+ */+/* An `undef` or `poison` operand: the phi may be anything over that edge, so+ * it is whatever its other operands name. LLVM leaves one when it splits a+ * pointer into a phi per path and only one path uses each (MPS's+ * cnnConvArray_*: two threadgroup pointers, each undef over the other's edge),+ * and counting it as a resource of its own refused a phi that names one.+ */+static bool+undef_operand(const struct air_value *v)+{+ if (!v || v->kind != AIR_VALUE_CONSTANT)+ return false;+ const enum air_const_kind k = ((const struct air_constant *)v)->ckind;+ return k == AIR_CONST_UNDEF || k == AIR_CONST_POISON;+}++static bool+resolve_pointer_phi(struct air_ctx *c, const struct air_inst *in, const bool *live,+ const int *taken_succ, unsigned my_block)+{+ struct air_val *first = NULL;+ bool carried = false; /* an operand over a loop edge, or offsets same_ptr cannot equate */+ for (unsigned k = 0; k < in->num_operands; k++) {+ if (!in->targets[k] || !live[in->targets[k]->index])+ continue;+ /* AND THE EDGE, not just the block. A block whose branch folded still+ * EXECUTES -- it is live -- but one of its two edges does not exist, and+ * a phi on the far side of the one that does not must not count the+ * operand arriving over it. Getting this wrong is not academic: it is+ * exactly what the fifteen texture phis of fixed_frag_lph_cph look like,+ * where `br i1 %144, label %null, label %join` folds to the null side+ * and `%join`'s phi still saw an operand from the branching block. The+ * refusal then said "a phi between pointers that name different places"+ * about a shader that names one.+ */+ int taken = taken_succ[in->targets[k]->index];+ if (taken >= 0 && taken != (int)my_block)+ continue;+ if (undef_operand(in->operands[k]))+ continue;+ struct hash_entry *e = _mesa_hash_table_search(c->values, in->operands[k]);+ if (!e) {+ carried = true; /* over a loop edge: stored, and checked, when the edge is */+ continue;+ }+ struct air_val *v = e->data;+ if (!v || v->kind != AIR_VAL_PTR)+ return air_error(c, "instruction: a phi of pointers one of whose operands is "+ "%s", v && v->kind == AIR_VAL_PTR_SELECT+ ? "itself a pointer chosen at run time"+ : "not a resolved pointer");+ if (!first)+ first = v;+ else if (first->ptr.class != v->ptr.class || first->ptr.binding != v->ptr.binding ||+ first->ptr.is_image != v->ptr.is_image)+ return air_error(c, "instruction: a phi between pointers that name different "+ "places, which would have to be chosen at run time");+ else if (!same_ptr(&first->ptr, &v->ptr))+ carried = true;+ }+ if (!first && carried)+ return air_error(c, "instruction: a phi of pointers whose operand comes in over "+ "a loop edge, so which resource it is depends on the "+ "iteration"); /* every live operand over a back edge: nothing to name the resource by */+ if (!first)+ return air_error(c, "instruction: a phi of pointers with no reachable operand");+ if (!carried) {+ air_set(c, &in->base, first);+ return true;+ }+ /* memory alone has an offset a register can carry: a texture, a sampler or a+ * slot is a handle, and an argument buffer's offset is resolved to a member+ * while translating, which a value known only at run time cannot be+ */+ const enum air_ptr_class cl = first->ptr.class;+ if (first->ptr.is_image || (cl != AIR_PTR_UBO && cl != AIR_PTR_SSBO && cl != AIR_PTR_SHARED &&+ cl != AIR_PTR_SCRATCH && cl != AIR_PTR_CONSTANT_DATA))+ return air_error(c, "instruction: a phi of a pointer that is not into memory (class %u) "+ "and changes round a loop or is computed twice", (unsigned)cl);+ nir_builder entry = nir_builder_at(nir_before_impl(c->b.impl));+ nir_def *reg = nir_decl_reg(&entry, 1, 32, 0);+ _mesa_hash_table_insert(c->phi_regs, in, reg);+ struct air_ptr p = first->ptr;+ p.offset = nir_load_reg(&c->b, reg);+ air_set(c, &in->base, air_val_ptr(c, p));+ return true;+}+static booltranslate_body(struct air_ctx *c){@@ -1704,17 +5331,40 @@ translate_body(struct air_ctx *c)/* phis first: each becomes a register, loaded where the phi is and stored* at the end of every predecessor (docs/air.md: NIR's own- * nir_lower_goto_ifs does the same to make an unstructured CFG safe)+ * nir_lower_goto_ifs does the same to make an unstructured CFG safe) --+ * except a phi of POINTERS, which has no bits and is resolved where it+ * stands (resolve_pointer_phi). It gets no register, and the store loop at+ * the end skips a phi that has none.*/nir_builder entry = nir_builder_at(nir_before_impl(c->b.impl));for (unsigned bi = 0; bi < f->num_blocks; bi++) {const struct air_block *blk = &f->blocks[bi];for (unsigned i = 0; i < blk->num_insts; i++) {const struct air_inst *in = blk->insts[i];- if (in->op != AIR_OP_PHI)+ if (in->op != AIR_OP_PHI || in->base.type->kind == AIR_TYPE_POINTER)continue;+ /* A phi of an AGGREGATE, named here rather than by whatever notices+ * the consequence further down. A struct is a tree of values in this+ * translator and never a NIR value, so there is no register to carry+ * one across an edge; the honest form would be one phi per member,+ * which this does not build. Until it was named, a register of ZERO+ * bits was declared for it and the module failed later with+ * "a returned struct that was never built" -- true of the module, and+ * not what is wrong with it.+ */+ if (in->base.type->kind == AIR_TYPE_STRUCT ||+ in->base.type->kind == AIR_TYPE_ARRAY)+ return air_error(c, "instruction: a phi of %s, which NIR has no register for",+ in->base.type->kind == AIR_TYPE_STRUCT ? "a struct"+ : "an array");+ /* A phi of a width NIR has no register for is exact: the register+ * is the next width up and a phi only COPIES, so a canonical value+ * in is a canonical value out (narrow_int_bits). What may be done+ * with it afterwards is narrow_int_gate's business, and every use+ * of it goes through there.+ */nir_def *reg = nir_decl_reg(&entry, air_nir_components(in->base.type),- air_nir_bit_size(c, in->base.type), 0);+ air_narrow_bits(c, in->base.type), 0);_mesa_hash_table_insert(c->phi_regs, in, reg);}}@@ -1727,8 +5377,50 @@ translate_body(struct air_ctx *c)unsigned *order = rzalloc_array(c->mem_ctx, unsigned, MAX2(f->num_blocks, 1));unsigned num_order = reverse_post_order(c, f, order);+ /* WHICH BLOCKS ARE ENTERED, which is not the same as which are reachable+ * in the CFG. A branch whose condition is already decided while translating+ * -- a function constant, through the program-scope variable the module's+ * air.static_init copies it into -- has one edge and not two, and the block+ * on the other side is never entered. It is then not translated either,+ * because translating it would build instructions for a path that does not+ * exist and, worse, would make a phi of pointers see two operands where the+ * shader has one (resolve_pointer_phi).+ *+ * This is filled as the walk goes rather than before it: the condition is a+ * translated value, so it cannot be known earlier. Reverse post order is+ * what makes that work -- every block but a loop header is reached after+ * all of its predecessors, so by the time a block is visited every forward+ * edge into it has been decided. A block nothing live branches to stays+ * false and is skipped; it gets no NIR block, no phi cursor, and the phi+ * store loop at the end already passes over an edge whose source has no+ * cursor.+ *+ * It is also strictly narrower than what stood here, which marked every+ * block of `order` live: a module that folds nothing gets exactly the old+ * behaviour, because then every edge of every branch is taken.+ */+ bool *live = rzalloc_array(c->mem_ctx, bool, MAX2(f->num_blocks, 1));+ if (num_order)+ live[order[0]] = true;++ /* THE ONE SUCCESSOR A BLOCK ACTUALLY BRANCHES TO, per block, or -1 for "all+ * of them". Recorded rather than derived, because it is the only thing a+ * phi on the far side can use: a block whose terminator folded still+ * EXECUTES -- it is live -- and the operand arriving over an edge it does+ * not take must not be counted.+ *+ * It is the taken successor and not the untaken ones because a folded+ * SWITCH prunes every case but one, and one number says that as cheaply as+ * it says a branch's single pruned edge.+ */+ int *taken_succ = rzalloc_array(c->mem_ctx, int, MAX2(f->num_blocks, 1));+ for (unsigned i = 0; i < f->num_blocks; i++)+ taken_succ[i] = -1;+for (unsigned oi = 0; oi < num_order; oi++) {unsigned bi = order[oi];+ if (!live[bi])+ continue;const struct air_block *blk = &f->blocks[bi];nir_block *nb = nir_block_of(c, blk);c->b.cursor = nir_after_block(nb);@@ -1738,6 +5430,11 @@ translate_body(struct air_ctx *c)const struct air_inst *in = blk->insts[i];if (in->op != AIR_OP_PHI)continue;+ if (in->base.type->kind == AIR_TYPE_POINTER) {+ if (!resolve_pointer_phi(c, in, live, taken_succ, bi))+ return false;+ continue;+ }struct hash_entry *e = _mesa_hash_table_search(c->phi_regs, in);air_set(c, &in->base,air_val_ssa(c, nir_load_reg(&c->b, e->data)));@@ -1747,7 +5444,12 @@ translate_body(struct air_ctx *c)const struct air_inst *in = blk->insts[i];if (in->op == AIR_OP_PHI)continue;- if (!translate_instruction(c, in)) {+ /* c->error as well as the return value: a refusal raised by something+ * that cannot report one upwards -- air_nir_bit_size, handed a width+ * NIR does not have -- must stop the walk here rather than let the+ * next instruction be built on top of it.+ */+ if (!translate_instruction(c, in) || c->error[0]) {/* say where, so the corpus report points at a line of the module */char where[64];snprintf(where, sizeof(where), " (block %u, instruction %u, opcode %u)",@@ -1778,24 +5480,147 @@ translate_body(struct air_ctx *c)break;case AIR_OP_BR:if (term->num_targets == 1) {+ live[term->targets[0]->index] = true;nir_goto(&c->b, nir_block_of(c, term->targets[0]));} else {nir_def *cond = air_get_ssa(c, term->operands[0]);if (!cond)return false;- if (term->targets[0] == term->targets[1])+ bool taken;+ if (term->targets[0] == term->targets[1]) {+ live[term->targets[0]->index] = true;nir_goto(&c->b, nir_block_of(c, term->targets[0]));- else+ } else if (cond_const(c, cond, &taken)) {+ /* ONE EDGE, because the condition is a constant here and not+ * merely foldable later. The other block is left out of `live`+ * and never translated, which is what lets a phi of pointers on+ * the far side resolve; emitting nir_goto_if with a constant+ * and trusting nir_opt_dead_cf would be too late for that, and+ * would also leave the dead block's instructions translated.+ */+ struct air_block *to = term->targets[taken ? 0 : 1];+ live[to->index] = true;+ taken_succ[bi] = (int)to->index;+ nir_goto(&c->b, nir_block_of(c, to));+ } else {+ live[term->targets[0]->index] = true;+ live[term->targets[1]->index] = true;nir_goto_if(&c->b, nir_block_of(c, term->targets[0]), cond,nir_block_of(c, term->targets[1]));+ }+ }+ break;+ case AIR_OP_SWITCH: {+ /* NIR has no switch, so the cases become a chain of nir_goto_if:+ * each test gets its own block, and the last one falls to the+ * default. nir_lower_goto_ifs structures the result like every other+ * branch this translator emits.+ *+ * What makes the chain safe is where the phi stores already are: all+ * of this block's successors' stores sit at phi_cursors[blk], above+ * the chain, and each successor's phis have registers of their own --+ * so storing on an edge the chain does not take writes a register+ * that the block actually entered never reads. That is the same+ * property the conditional br above relies on, and the reason a+ * switch needs no per-edge bookkeeping.+ */+ nir_def *cond = air_get_ssa(c, term->operands[0]);+ if (!cond)+ return false;+ /* A SWITCH WHOSE SELECTOR IS ALREADY DECIDED takes one case, exactly+ * as a branch does.+ *+ * What stood here said the opposite, and said why: "the only switches+ * in this corpus are the texture/blend/colour function dispatches of+ * the GL-on-Metal uber shaders, where the selector is a uchar function+ * constant and folding it would delete thirty-odd blocks at once -- a+ * much larger change in what is compiled than the branch fold, for no+ * module that the branch fold does not already reach." The first half+ * is true and the last clause is FALSE, and fixed_frag_lph_cph is the+ * counterexample: its asg77 filter case loads through the weights+ * pointer, that pointer is `select fc_has_asg_77_weights, null, %14`+ * and folds to NULL, and the case is reachable only through a switch+ * on `fc_filter0`. Leaving the switch alone therefore translated a+ * block the shader never enters and refused the module for a null+ * dereference inside it -- a true sentence about code that does not+ * run. The branch fold could not reach that; only this can.+ *+ * The default is targets[0] and the cases follow, so a selector that+ * matches no case takes the default -- which is what makes folding+ * total rather than a guess: there is always exactly one answer.+ */+ uint64_t sel;+ if (uint_const(c, cond, &sel)) {+ const struct air_block *to = term->targets[0]; /* the default */+ for (unsigned k = 1; k < term->num_targets; k++) {+ nir_def *cv = air_get_ssa(c, term->operands[k]);+ uint64_t cval;+ if (!cv)+ return false;+ if (uint_const(c, cv, &cval) && cval == sel) {+ to = term->targets[k];+ break;+ }+ }+ live[to->index] = true;+ taken_succ[bi] = (int)to->index;+ nir_goto(&c->b, nir_block_of(c, (struct air_block *)to));+ break;+ }+ for (unsigned k = 0; k < term->num_targets; k++)+ live[term->targets[k]->index] = true;+ nir_block *deflt = nir_block_of(c, term->targets[0]);+ unsigned n = term->num_targets - 1;+ for (unsigned k = 0; k < n; k++) {+ nir_def *val = air_get_ssa(c, term->operands[1 + k]);+ if (!val)+ return false;+ nir_block *dest = nir_block_of(c, term->targets[1 + k]);+ bool last = k + 1 == n;+ nir_block *next = deflt;+ if (!last) {+ next = nir_block_create(c->nir);+ exec_list_push_tail(&c->b.impl->body, &next->cf_node.node);+ next->cf_node.parent = &c->b.impl->cf_node;+ }+ /* a last case that goes where the default goes is not a branch at+ * all, and nir_goto_if with one block on both sides is what the+ * conditional br above refuses for the same reason+ */+ if (dest == next)+ nir_goto(&c->b, dest);+ else+ nir_goto_if(&c->b, dest, nir_ieq(&c->b, cond, val), next);+ if (!last)+ c->b.cursor = nir_after_block(next);}+ if (n == 0)+ nir_goto(&c->b, deflt);break;+ }case AIR_OP_UNREACHABLE:nir_goto(&c->b, c->b.impl->end_block);break;default:return air_error(c, "instruction: block %u ends with opcode %u", bi, term->op);}++ /* The same reason as inside the instruction loop, for the TERMINATOR,+ * which is translated here and not there. Without this a refusal raised+ * while building a switch's case constants was only noticed at the first+ * instruction of the next block in reverse-post-order, and the report+ * then pointed at that instruction: resolve_sample_fragment's 3-bit+ * switch in block 0 was attributed to a shufflevector in block 26, which+ * is a location that sends the reader to the wrong line of the module.+ */+ if (c->error[0]) {+ char where[64];+ snprintf(where, sizeof(where), " (block %u, terminator, opcode %u)",+ bi, term->op);+ if (strlen(c->error) + strlen(where) < sizeof(c->error))+ strcat(c->error, where);+ return false;+ }}/* the phi stores, now that every value exists */@@ -1807,13 +5632,32 @@ translate_body(struct air_ctx *c)continue;struct hash_entry *re = _mesa_hash_table_search(c->phi_regs, in);if (!re)- continue; /* a phi in an unreachable block */+ continue; /* a phi in an unreachable block, or one of pointers */for (unsigned k = 0; k < in->num_operands; k++) {struct hash_entry *ce = _mesa_hash_table_search(c->phi_cursors, in->targets[k]);if (!ce)continue; /* an edge from a block the entry cannot reach */c->b.cursor = *(nir_cursor *)ce->data;- nir_def *val = air_get_ssa(c, in->operands[k]);+ nir_def *val;+ if (in->base.type->kind == AIR_TYPE_POINTER && undef_operand(in->operands[k]))+ continue; /* the register holds whatever it held: undef over this edge */+ if (in->base.type->kind == AIR_TYPE_POINTER) {+ /* a carried pointer phi (resolve_pointer_phi): the operand's offset, once+ * it is known to name the phi's resource -- over a loop edge this is the+ * first time it is seen at all+ */+ struct hash_entry *pe = _mesa_hash_table_search(c->values, &in->base);+ struct hash_entry *oe = _mesa_hash_table_search(c->values, in->operands[k]);+ const struct air_val *pv = pe ? pe->data : NULL, *ov = oe ? oe->data : NULL;+ if (!pv || !ov || ov->kind != AIR_VAL_PTR || ov->ptr.class != pv->ptr.class ||+ ov->ptr.binding != pv->ptr.binding || ov->ptr.is_image != pv->ptr.is_image)+ return air_error(c, "instruction: a phi of pointers whose operand over "+ "an edge names another resource, so which one it is "+ "depends on the path taken");+ val = ov->ptr.offset ? nir_u2u32(&c->b, ov->ptr.offset) : nir_imm_int(&c->b, 0);+ } else {+ val = air_get_ssa(c, in->operands[k]);+ }if (!val)return false;nir_store_reg(&c->b, val, re->data);@@ -1850,11 +5694,388 @@ function_constant_index(struct air_ctx *c, const struct air_global *g)return -1;}+/* The NAME and the required-ness of a function constant, by index: the other+ * direction from function_constant_index, and needed where only the index is+ * in hand -- the load path, which has a binding and not a global.+ *+ * `!{ptr @g, !"uint", !"name", i32 index, i1 required}`: the name is the last+ * string before the first integer, and the required flag the boolean after it.+ * Matched that way rather than by position, because the two keys this reads+ * are the two whose places are attested across the corpus and nothing says the+ * node cannot grow an operand.+ */+static const char *+function_constant_decl(struct air_ctx *c, unsigned index, bool *required)+{+ const struct air_named_md *n = named_md(c->m, "air.function_constants");+ for (unsigned i = 0; i < (n ? n->num_ops : 0); i++) {+ const struct air_md *node = n->ops[i];+ if (!node || node->kind != AIR_MD_NODE)+ continue;+ const char *name = NULL;+ for (unsigned k = 0; k < node->num_ops; k++) {+ const char *str = md_str(node, k);+ if (str) {+ name = str;+ continue;+ }+ uint64_t idx;+ if (!md_int(node, k, &idx))+ continue;+ if (idx != index)+ break; /* a different constant: on to the next node */+ if (required) {+ uint64_t req = 0;+ *required = md_int(node, k + 1, &req) && req != 0;+ }+ return name;+ }+ }+ return NULL;+}+/* Decide what every global is: once per shader, because this appends inline* samplers to info->resources and lays out the shared and scratch memory. It* records rather than builds -- making the value needs a builder, and which* function's builder matters (see materialise_globals).*/+/* The 64-bit `constexpr sampler` word, field by field (nir_air.h has where the+ * layout comes from and why it took three sources). Bit 63 marks the one-word+ * form; the two-word form of AIR 2.6 carries a LOD bias in the second word's+ * low 16 bits and no module of this corpus uses it, so it is reported rather+ * than decoded.+ */+struct air_sampler_state+air_decode_sampler_state(uint64_t w)+{+#define FIELD(lo, n) ((unsigned)((w >> (lo)) & ((1ull << (n)) - 1)))+ struct air_sampler_state s = {+ .address = { FIELD(0, 3), FIELD(3, 3), FIELD(6, 3) },+ .mag_filter = FIELD(9, 2),+ .min_filter = FIELD(11, 2),+ .mip_filter = FIELD(13, 2),+ .pixel_coord = FIELD(15, 1) != 0,+ .compare_func = FIELD(16, 4),+ .max_anisotropy = FIELD(20, 4) + 1,+ .lod_min = FIELD(24, 16),+ .lod_max = FIELD(40, 16),+ .border_color = FIELD(56, 2),+ .reduction = FIELD(58, 2),+ };+#undef FIELD+ return s;+}++/* Does this value, or any constant built out of it, name that global?+ *+ * A constant expression is where a global's address escapes without any+ * instruction mentioning it: `getelementptr (%struct, ptr @g, 0, 3)` is a+ * CONSTANT in the bitcode, and the instruction that uses it names the+ * expression and not @g. A scan that only compared instruction operands would+ * therefore have called @_ZL25fc_fragment_shader_config -- whose every field+ * is reached through exactly that shape -- a global nothing but whole loads+ * touch, and folded a byte of it over the struct the shader actually reads.+ */+static bool+const_mentions(const struct air_value *v, const struct air_value *target,+ unsigned depth)+{+ if (!v || depth == 0)+ return false;+ if (v == target)+ return true;+ const struct air_constant *k = air_value_as_constant(v);+ if (!k)+ return false;+ /* ON THE KIND, and not on whichever count happens to be non-zero.+ * air_constant shares `num_elems` between an AGGREGATE, whose members are+ * in `elems`, and DATA, whose numbers are in `data` and whose `elems` is+ * NULL (air_module.c, CST_CODE_DATA) -- so walking `elems` whenever+ * num_elems is set dereferences NULL on every string and every+ * `<4 x i8>` data constant. That is what it did: 42 modules of the corpus+ * died in here with SIGSEGV, every one of them a module whose+ * air.static_init made this run at all.+ */+ switch (k->ckind) {+ case AIR_CONST_AGGREGATE:+ for (unsigned i = 0; i < k->num_elems; i++)+ if (const_mentions(k->elems[i], target, depth - 1))+ return true;+ return false;+ case AIR_CONST_EXPR:+ for (unsigned i = 0; i < k->num_operands; i++)+ if (const_mentions(k->operands[i], target, depth - 1))+ return true;+ return false;+ default:+ /* INT, FLOAT, NULL, UNDEF, POISON, DATA: numbers, and no value in them */+ return false;+ }+}++/* THE RESERVED SLOT A NULL TEXTURE LIVES IN -- road B of docs/air.md section 3,+ * the frontend's half of it.+ *+ * `air.get_null_texture_2d` and its fifteen siblings are what a Metal shader+ * asks for when the pipeline bound no image, and there is no index here that+ * means "nothing bound": a texture is Metal's own `[[texture(n)]]`, resolved+ * while translating, which is what lets a sample be one instruction with no+ * descriptor indirection. So one index past the module's own is reserved for+ * it and the resource is flagged, for the runtime to bind a 1x1 texture of+ * zeros at (air_resource.null_texture in nir_air.h, which carries what that+ * rests on).+ *+ * ONE PAST THE HIGHEST DECLARED INDEX, and that is deliberately NOT Apple's+ * rule. Apple uses the texture COUNT -- createTextureHeapIndexArray stores the+ * SmallVector's own Size (0x1bbe988b4) -- which is the same number only when+ * the indices run 0..n-1. For a module that declares 0, 1 and 5, Apple's rule+ * gives 3, which is a slot nothing declared but which a later version of the+ * same shader could; one past the highest gives 6, which cannot collide with+ * anything the module asked for. The safe one is chosen over the faithful one+ * because a collision here binds a zero texture over a real image in silence.+ *+ * REFUSED BY NAME when it does not fit, rather than wrapped. AIR_MAX_TEXTURES+ * is freedreno's own 16 (view_seqno[16] in fd6_texture_state), and+ * fixed_frag_lph_cph declares 0..14 -- so its reserved slot is 15, the last+ * one there is, and it fits with nothing to spare. A module that declares+ * sixteen textures gets a refusal here and not an index of 16: the runtime's+ * own check would otherwise refuse it too, but with the sentence "binds a+ * texture at index 16", which blames the module for a slot THIS function+ * added. That is the fourth fixed array of the day found smaller than+ * something real, and the only one where the array's own maximum is the+ * reserved slot itself.+ */+int+air_null_texture_slot(struct air_ctx *c)+{+ if (c->null_texture_index >= 0)+ return c->null_texture_index;++ struct air_shader_info *info = c->info;+ int highest = -1;+ for (unsigned i = 0; i < info->num_resources; i++) {+ const struct air_resource *r = &info->resources[i];+ if (r->kind == AIR_RESOURCE_TEXTURE || r->kind == AIR_RESOURCE_IMAGE)+ highest = MAX2(highest, (int)r->index);+ }+ unsigned slot = (unsigned)(highest + 1);+ if (slot >= AIR_MAX_TEXTURES) {+ air_error(c, "intrinsic: a null texture needs a reserved slot and this module "+ "declares textures up to %d, so the slot would be %u -- past the %u "+ "this GPU's descriptor table has (AIR_MAX_TEXTURES)",+ highest, slot, AIR_MAX_TEXTURES);+ return -1;+ }++ info->resources = reralloc(c->mem_ctx, info->resources, struct air_resource,+ info->num_resources + 1);+ /* Not through add_resource: there is no metadata node behind this one, and+ * the two "no group"/"no argument buffer" fields have to be written rather+ * than left zero -- a zeroed raster_order_group reads as group 0 and a+ * zeroed argbuf_set as "member 0 of the set at Metal index 0", which is the+ * same care plan_globals takes for an inline sampler.+ */+ info->resources[info->num_resources++] = (struct air_resource) {+ .kind = AIR_RESOURCE_TEXTURE, .index = slot, .read = true,+ .null_texture = true, .param = ~0u, .name = "(null texture)",+ .raster_order_group = -1, .argbuf_set = ~0u, .argbuf_id = ~0u,+ };+ c->null_texture_index = (int)slot;+ return (int)slot;+}++/* Which program-scope variables are a VALUE rather than memory.+ *+ * This is the whole of what "specialise the shader on its function constants"+ * needs from the frontend, and it is a census rather than an optimisation: the+ * GL-on-Metal uber shaders decide 58 function constants in an air.static_init+ * constructor, copy them into program-scope variables, and then branch on+ * those variables fifteen times to choose between a bound texture and a null+ * one. plan_globals puts such a variable in SCRATCH, so what the body sees is+ * a load_scratch -- opaque -- and every one of those branches survives+ * translation, which is what left the two modules with fifteen phis between+ * two different textures (docs/air.md section 3). Metal does not compile them+ * that way: it specialises at pipeline creation, and this frontend already has+ * every function constant as an immediate (AIR_PTR_FUNCTION_CONSTANT). The+ * only thing in the way was the round trip through memory.+ *+ * A global qualifies only when all four hold, and each one is a way of being+ * wrong that was considered rather than a box ticked:+ *+ * - its type is a scalar integer of 8, 16, 32 or 64 bits. Not a float, not a+ * vector, not an aggregate: `bits` has to BE the value, and a narrower or+ * odd width would go through air_narrow_bits and stop being the thing a+ * later load wants.+ * - every instruction that names it names it as the whole pointer of a load+ * or of a store, and the type of that load or store is the global's own.+ * A getelementptr, a call argument, a stored VALUE, a phi, a select: all+ * disqualify, because then some other access reaches the same bytes and+ * the value and the memory could disagree.+ * - no constant expression and no other global's initialiser mentions it, so+ * its address does not escape into something this scan cannot follow.+ * - exactly one store, and that store is in a function whose section is+ * air.static_init. One store is what makes the value independent of where+ * in the shader the load is; the section is what makes it run before the+ * body, which run_static_initializers inlines at the top of the entry+ * point. A variable written by the body is a variable, and stays memory.+ *+ * What this does NOT decide is whether the value is a constant at all. That is+ * known only when the store is translated, so `fold` means "a load of this may+ * be answered" and `known` means "and here is the answer"; a store of+ * something that is not constant simply never sets `known`, and the scratch+ * underneath answers as it always did.+ *+ * AND IT DECIDES NOTHING ABOUT WHOSE VALUE IT IS. The constant comes from+ * air_load_ptr's AIR_PTR_FUNCTION_CONSTANT case, which answers with+ * air_constant_value.bits and does not look at `supplied` -- and+ * air_module_function_constants registers every constant a module DECLARES+ * with bits = 0 and supplied = false, so in every harness in tools/air this+ * folds a shader on zeros that nobody set. That is not a new invention: the+ * unfolded shader took the same branch for the same reason, at run time, off+ * an immediate the same case produced. What changes is where the consequence+ * lives -- a block that is not compiled rather than a branch that is not+ * taken -- so the specialisation is exactly as good as the constants the+ * runtime hands in and no better.+ *+ * It matters most for the one kind of constant that is allowed to be absent.+ * The law (`!{ptr, !"type", !"name", i32 index, i1 required}`) is that a+ * declaration ending `i1 false` is always guarded by+ * air.is_function_constant_defined and one ending `i1 true` never is: 58 of+ * 58 in fixed_frag_lph_cph, 24 required and unguarded against 34 optional and+ * guarded, no exceptions. Fourteen of that module's fifteen texture+ * predicates are REQUIRED, so Metal obliges the pipeline to supply them and+ * folding them is specialisation. The fifteenth, fc_has_lut_tex at index 23,+ * is optional -- and its constructor copies the RAW load into the variable+ * rather than the guarded one, so the module itself reads an unsupplied+ * constant as whatever the implementation gives. This passes that reading+ * through unchanged instead of deciding it, which is the only honest thing to+ * do from here: the decision belongs with whoever fills `supplied` in.+ */+static void+plan_global_folding(struct air_ctx *c)+{+ for (unsigned i = 0; i < c->m->num_globals; i++) {+ struct air_global_plan *pl = &c->global_plan[i];+ const struct air_global *g = c->m->globals[i];+ if (pl->skip || pl->class != AIR_PTR_SCRATCH)+ continue;++ const struct air_type *t = g->value_type;+ if (!t || t->kind != AIR_TYPE_INT ||+ (t->bits != 8 && t->bits != 16 && t->bits != 32 && t->bits != 64))+ continue;++ bool ok = true;+ unsigned stores = 0;++ for (unsigned k = 0; ok && k < c->m->num_globals; k++) {+ if (k != i && const_mentions(c->m->globals[k]->init, &g->base, 8))+ ok = false;+ }++ for (unsigned fi = 0; ok && fi < c->m->num_functions; fi++) {+ const struct air_function *f = c->m->functions[fi];+ bool is_init = f->section && !strcmp(f->section, "air.static_init");+ for (unsigned bi = 0; ok && bi < f->num_blocks; bi++) {+ const struct air_block *blk = &f->blocks[bi];+ for (unsigned ii = 0; ok && ii < blk->num_insts; ii++) {+ const struct air_inst *in = blk->insts[ii];+ for (unsigned oi = 0; ok && oi < in->num_operands; oi++) {+ const struct air_value *v = in->operands[oi];+ if (v == &g->base) {+ if (in->op == AIR_OP_LOAD && oi == 0 &&+ in->base.type == t) {+ continue;+ }+ if (in->op == AIR_OP_STORE && oi == 1 &&+ in->operands[0]->type == t && is_init) {+ stores++;+ continue;+ }+ ok = false;+ } else if (const_mentions(v, &g->base, 8)) {+ ok = false;+ }+ }+ }+ }+ }++ if (ok && stores == 1)+ pl->fold = true;+ }+}++/* The global this operand IS -- not one it points into. Only a global used+ * directly as a load's or a store's pointer can be folded, and+ * plan_global_folding has already refused every other shape, so a linear scan+ * over the module's globals answers it exactly.+ */+static struct air_global_plan *+foldable_global_plan(struct air_ctx *c, const struct air_value *v)+{+ if (!c->global_plan || !v || v->kind != AIR_VALUE_GLOBAL)+ return NULL;+ for (unsigned i = 0; i < c->m->num_globals; i++) {+ if (&c->m->globals[i]->base == v)+ return c->global_plan[i].fold ? &c->global_plan[i] : NULL;+ }+ return NULL;+}++/* A constant's bytes as the module lays the type out (air_type_alloc_size, member+ * offsets): what a read-only global holds, for nir->constant_data. False for what+ * is not bytes -- a constant expression (a pointer's value), a type with no size.+ */+static bool+write_constant(struct air_ctx *c, const struct air_type *t, const struct air_value *v, uint8_t *dst)+{+ const struct air_constant *k = air_value_as_constant(v);+ if (!k)+ return false;+ const uint64_t size = air_type_alloc_size(c->m, t);+ switch (k->ckind) {+ case AIR_CONST_NULL:+ case AIR_CONST_UNDEF:+ case AIR_CONST_POISON:+ memset(dst, 0, size);+ return true;+ case AIR_CONST_INT:+ case AIR_CONST_FLOAT: {+ const unsigned bytes = t->kind == AIR_TYPE_INT ? (t->bits + 7) / 8 : t->kind == AIR_TYPE_HALF || t->kind == AIR_TYPE_BFLOAT ? 2 :+ t->kind == AIR_TYPE_FLOAT ? 4 : t->kind == AIR_TYPE_DOUBLE ? 8 : 0;+ if (bytes == 0 || bytes > 8)+ return false;+ memcpy(dst, &k->u, bytes); /* little-endian, as the GPU reads it */+ return true;+ }+ case AIR_CONST_DATA:+ case AIR_CONST_AGGREGATE: {+ if (t->kind != AIR_TYPE_ARRAY && t->kind != AIR_TYPE_VECTOR && t->kind != AIR_TYPE_STRUCT)+ return false;+ memset(dst, 0, size);+ for (unsigned i = 0; i < k->num_elems; i++) {+ const struct air_type *et = t->kind == AIR_TYPE_STRUCT ? t->members[i] : t->elem;+ /* a vector's elements are packed at the element's size, an array's at its stride */+ const uint64_t at = t->kind == AIR_TYPE_STRUCT ? air_struct_member_offset(c->m, t, i)+ : t->kind == AIR_TYPE_VECTOR ? i * (uint64_t)((air_type_alloc_size(c->m, et)))+ : i * air_type_alloc_size(c->m, et);+ if (k->ckind == AIR_CONST_DATA) {+ const uint64_t eb = air_type_alloc_size(c->m, et);+ if (eb > 8)+ return false;+ memcpy(dst + at, &k->data[i], eb);+ } else if (!write_constant(c, et, k->elems[i], dst + at)) {+ return false;+ }+ }+ return true;+ }+ default:+ return false;+ }+}+static voidplan_globals(struct air_ctx *c){@@ -1908,6 +6129,12 @@ plan_globals(struct air_ctx *c).kind = AIR_RESOURCE_SAMPLER, .index = next, .read = true,.param = ~0u, .name = name,.inline_sampler = true, .sampler_state = word,+ .sampler = air_decode_sampler_state(word),+ /* not through add_resource, so the "no group" value has to be+ * written here: a zeroed field would read as group 0, and a zeroed+ * argbuf_set as "member 0 of the set at Metal index 0" */+ .raster_order_group = -1,+ .argbuf_set = ~0u, .argbuf_id = ~0u,};} else if (g->addrspace == 3) {/* a static threadgroup array */@@ -1917,6 +6144,59 @@ plan_globals(struct air_ctx *c)pl->offset = c->shared_size;pl->has_offset = true;c->shared_size += air_type_alloc_size(c->m, g->value_type);+ } else if (g->is_const && g->init && type_has_pointer(g->value_type, 8)) {+ /* A READ-ONLY GLOBAL WHOSE INITIALISER HOLDS A POINTER, which scratch+ * cannot hold any more than an alloca can: a pointer here is a+ * resolution and not bytes. It becomes a translation-time slot+ * (AIR_PTR_SLOT), pre-filled from its own initialiser by+ * materialise_globals -- in whatever builder is asking, because the+ * values inside it are defs and a def belongs to one impl.+ *+ * The shape this is for is one line of Metal:+ *+ * constexpr sampler unused_sampler;+ *+ * which Apple's compiler emits as+ * `constant %"struct.metal::sampler" { ptr addrspace(2) @__air_sampler_state }`+ * and QuartzCore's two YCbCr uber shaders then `select` against, before+ * loading the sampler back out of whichever side won. Put in scratch,+ * that global was bytes nothing ever wrote -- its initialiser holds a+ * pointer, so there was nothing to write -- and a load through it+ * would have read uninitialised memory and called it a sampler. The+ * refusal it produced instead ("a pointer loaded from memory") was+ * right, and this is what answers it.+ *+ * `is_const` is what makes the slot sound rather than convenient: the+ * value cannot change, so one fill per function is the whole story,+ * and AIR_OP_STORE refuses a store into a slot that has an+ * initialiser rather than quietly overwriting one.+ */+ pl->class = AIR_PTR_SLOT;+ pl->binding = new_slot(c, g->value_type, g->init);+ } else if (g->is_const && g->init && !type_has_pointer(g->value_type, 8) &&+ air_type_alloc_size(c->m, g->value_type) > 0) {+ /* A READ-ONLY GLOBAL WITH ITS DATA -- a table, `constant float4 at[6]+ * = {...}` -- is the shader's constant data (nir->constant_data, read+ * by load_constant), its bytes written here from the initialiser. As a+ * program-scope variable it went to scratch, where nothing writes the+ * initialiser: sokol's bufferless triangle read its corners as zeros+ * and drew nothing.+ */+ unsigned align = MAX2(g->align, air_type_abi_align(c->m, g->value_type));+ uint32_t at = ALIGN_POT(c->nir->constant_data_size, MAX2(align, 4));+ uint64_t size = air_type_alloc_size(c->m, g->value_type);+ uint8_t *data = rzalloc_size(c->nir, at + size);+ if (c->nir->constant_data_size)+ memcpy(data, c->nir->constant_data, c->nir->constant_data_size);+ if (!write_constant(c, g->value_type, g->init, data + at)) {+ air_error(c, "interface: the constant global %s has an initialiser that is not plain data", name);+ return;+ }+ c->nir->constant_data = data;+ c->nir->constant_data_size = at + size;+ pl->class = AIR_PTR_CONSTANT_DATA;+ pl->offset = at;+ pl->has_offset = true;} else {/* a program-scope variable: private to the invocation, written by* the static initializers below@@ -1929,6 +6209,9 @@ plan_globals(struct air_ctx *c)c->scratch_size += air_type_alloc_size(c->m, g->value_type);}}++ /* after the loop, because it reads the class every global ended up with */+ plan_global_folding(c);}/* Build the globals as values of the function being translated now. Called for@@ -1953,6 +6236,30 @@ materialise_globals(struct air_ctx *c)};air_set(c, &g->base, air_val_ptr(c, p));}++ /* THEN the slots, in a pass of their own, because an initialiser may name+ * another global: `constant %"struct.metal::sampler" { ptr addrspace(2)+ * @__air_sampler_state }` is a slot whose contents is the value of a global+ * further down the list. Filling it inside the loop above read whichever+ * globals happened to come first and left the rest unresolved, which came+ * out as "a load through a null pointer" from inside a callee -- a true+ * sentence about a pointer this function had not finished building.+ *+ * Re-filled per function rather than once, for the reason the offsets above+ * are rebuilt per function: the values inside a slot are defs, and a def+ * belongs to the impl that made it.+ */+ for (unsigned i = 0; i < c->m->num_globals; i++) {+ const struct air_global_plan *pl = &c->global_plan[i];+ if (pl->skip || pl->class != AIR_PTR_SLOT)+ continue;+ if (pl->binding >= c->num_slots || !c->slots[pl->binding].init)+ continue;+ struct air_val *v = air_get(c, c->slots[pl->binding].init);+ if (!v)+ return; /* air_get has said why */+ c->slots[pl->binding].val = v;+ }}/* The functions of @llvm.global_ctors, inlined at the top of the entry point:@@ -1966,10 +6273,32 @@ run_static_initializers(struct air_ctx *c)const struct air_function *f = c->m->functions[i];if (f->is_declaration || !f->section || strcmp(f->section, "air.static_init"))continue;- if (f->num_args || f->num_blocks != 1) {- air_error(c, "interface: a static initializer with arguments or branches");+ if (f->num_args) {+ air_error(c, "interface: a static initializer with arguments");return;}+ if (f->num_blocks != 1) {+ /* A static initializer with branches cannot be translated in place:+ * the CFG walk owns the impl's start block, so running it a second+ * time in the same impl would map two AIR blocks onto it. It becomes+ * a nir_function instead and the entry point calls it, which is the+ * machinery a call of another function in the module already uses,+ * and nir_inline_functions folds it back in afterwards. The globals+ * it writes are at the offsets plan_globals decided, which is one+ * plan for the whole module, so the callee's stores and the entry+ * point's loads reach the same scratch.+ *+ * These are the GL-on-Metal fragment shaders, whose initializer+ * decides 58 function constants and has a switch in it -- which is+ * also why this could not be reached before the reader learnt to read+ * one.+ */+ nir_function *fn = translate_callee(c, f, NULL, NULL);+ if (!fn)+ return; /* translate_callee has said why */+ nir_builder_instr_insert(&c->b, &nir_call_instr_create(c->nir, fn)->instr);+ continue;+ }/* its instructions, in place, without its ret */const struct air_block *blk = &f->blocks[0];for (unsigned k = 0; k < blk->num_insts; k++) {@@ -1982,8 +6311,141 @@ run_static_initializers(struct air_ctx *c)}}+/*+ * WHICH `device` BUFFERS THE TRANSLATED SHADER STORES THROUGH, out of the IR itself.+ *+ * A second reading of a fact the metadata already carries, taken because a runtime+ * makes a decision on it that is wrong in SILENCE when the fact is wrong: an+ * unaligned `device` buffer can be served by uploading a copy, exactly as a+ * `constant` one already is, only if nothing writes it -- a copy of a buffer the+ * shader writes would throw the writes away and the draw would look as though it had+ * worked. `air.write`/`air.read_write` answers a different question, which is what+ * Metal lets the shader do: `device T *` is declared writable whether or not the body+ * ever stores through it, and in RenderBox's mask shaders none of them does.+ *+ * The reading is exact for every module that gets this far, and that is a consequence+ * of refusals above rather than an assumption about stores. A store through a pointer+ * chosen at run time is refused ("a store through a chosen pointer, which would have+ * to write both of them"); a pointer written INTO memory, from where it could be+ * fetched back and stored through, is refused ("a store of a pointer into memory");+ * and a pointer whose binding did not resolve never becomes an AIR_PTR_SSBO at all.+ * So every write to a device buffer that a translated module can perform is a+ * store_ssbo or an ssbo_atomic carrying the immediate binding this translator put+ * there -- and this runs after nir_inline_functions, so a callee's store is in the+ * entry point's own IR by now.+ *+ * Where that argument does not reach, the answer is "written" and not "unknown": a+ * store this scan cannot attribute to a declared buffer marks every device buffer in+ * the module, because the alternative to over-refusing one draw is dropping a write.+ */+static void+device_writes_from_ir(nir_shader *nir, struct air_shader_info *info)+{+ nir_foreach_function_impl(impl, nir) {+ nir_foreach_block(block, impl) {+ nir_foreach_instr(instr, block) {+ if (instr->type != nir_instr_type_intrinsic)+ continue;++ nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);+ unsigned src;+ switch (intr->intrinsic) {+ case nir_intrinsic_store_ssbo:+ src = 1;+ break;+ case nir_intrinsic_ssbo_atomic:+ case nir_intrinsic_ssbo_atomic_swap:+ src = 0;+ break;+ default:+ continue;+ }++ bool named = false;+ if (nir_src_is_const(intr->src[src])) {+ unsigned binding = nir_src_as_uint(intr->src[src]);+ for (unsigned i = 0; i < info->num_resources; i++) {+ struct air_resource *r = &info->resources[i];+ if (r->kind == AIR_RESOURCE_DEVICE_BUFFER && r->index == binding) {+ r->ir_write = true;+ named = true;+ }+ }+ }+ if (named)+ continue;++ for (unsigned i = 0; i < info->num_resources; i++)+ if (info->resources[i].kind == AIR_RESOURCE_DEVICE_BUFFER)+ info->resources[i].ir_write = true;+ }+ }+ }+}+/* -- the entry point ---------------------------------------------------- */+/* Everything the caller is handed in INFO, copied out of the scratch context+ * that is freed before air_to_nir returns: see the comment inside. Both+ * returns that hand back a shader call it -- the interface-only one used to+ * skip it, and handed out resource tables already freed, which a thread+ * allocating at the same time overwrote: Resident Evil 4's argument encoders+ * came back with every member id 0, wrote nothing, and failed every frame.+ */+static void+keep_info(void *mem_ctx, struct air_shader_info *info)+{+ /* The info's arrays live under mem_ctx, not under the scratch context --+ * and so must every STRING in them, which is the part this used to get+ * wrong. `air_md_string` returns a NUL-terminated copy allocated as a+ * child of the metadata node it came from, so every `air.arg_name` and+ * `air.arg_type_name` in here pointed into the module, which is under+ * `ctx`, which is freed three lines below. The caller was handed+ * pointers into freed memory and mostly got away with it, because the+ * bytes usually survive; what it looks like when they do not is+ * `--interface` printing `name=on=1` for an attribute whose name is+ * `texcoord1`, which is how this was found -- vattrcheck.py counted four+ * attributes in a three-attribute shader because two garbled lines each+ * carried a stray tab.+ *+ * vertex_attributes was not copied at all, which was the same bug with+ * no bytes left to get away with.+ */+ if (info->num_resources) {+ info->resources = ralloc_memdup(mem_ctx, info->resources,+ info->num_resources * sizeof(*info->resources));+ for (unsigned i = 0; i < info->num_resources; i++) {+ info->resources[i].name = ralloc_strdup(mem_ctx, info->resources[i].name);+ info->resources[i].type_name =+ ralloc_strdup(mem_ctx, info->resources[i].type_name);+ }+ }+ if (info->num_varyings) {+ info->varyings = ralloc_memdup(mem_ctx, info->varyings,+ info->num_varyings * sizeof(*info->varyings));+ for (unsigned i = 0; i < info->num_varyings; i++)+ info->varyings[i].name = ralloc_strdup(mem_ctx, info->varyings[i].name);+ }+ if (info->num_vertex_attributes) {+ info->vertex_attributes =+ ralloc_memdup(mem_ctx, info->vertex_attributes,+ info->num_vertex_attributes * sizeof(*info->vertex_attributes));+ for (unsigned i = 0; i < info->num_vertex_attributes; i++) {+ struct air_vertex_attribute *a = &info->vertex_attributes[i];+ a->name = ralloc_strdup(mem_ctx, a->name);+ a->type_name = ralloc_strdup(mem_ctx, a->type_name);+ a->function_constant = ralloc_strdup(mem_ctx, a->function_constant);+ }+ }+ if (info->num_notes) {+ info->notes = ralloc_memdup(mem_ctx, info->notes,+ info->num_notes * sizeof(*info->notes));+ for (unsigned i = 0; i < info->num_notes; i++)+ info->notes[i] = ralloc_strdup(mem_ctx, info->notes[i]);+ }+ info->entry_point = ralloc_strdup(mem_ctx, info->entry_point);+}+nir_shader *air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_point,const struct air_to_nir_options *options,@@ -1996,6 +6458,9 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_po.mem_ctx = ctx,.options = options,.info = info ? info : &local_info,+ /* -1 and not 0: slot 0 is a real texture index, so a zeroed field would+ * read as "the null texture already lives at index 0" */+ .null_texture_index = -1,};memset(c.info, 0, sizeof(*c.info));@@ -2022,8 +6487,14 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_po* whose only job is to call it (wrap_library_function).*/bool wrap = false;+ /* A stitched graph is the third thing this can be asked for: not this+ * module as a shader, and not this module's function wrapped, but a graph+ * of library functions compiled into one entry point, of which this+ * module's function is one node like any other (stitch_graph).+ */+ bool stitch = options && options->stitch;if (stage == MESA_SHADER_NONE) {- if (!options || !options->library_wrapper) {+ if (!options || !(options->library_wrapper || options->stitch)) {if (error)*error = ralloc_strdup(mem_ctx,"stage: air.visible, a library function (not yet)");@@ -2032,6 +6503,12 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_po}stage = MESA_SHADER_COMPUTE;wrap = true;+ } else if (stitch) {+ if (error)+ *error = ralloc_strdup(mem_ctx, "stitch: the module given is an entry point and "+ "not an air.visible function");+ ralloc_free(ctx);+ return NULL;}if (!entry->num_ops || !entry->ops[0] || entry->ops[0]->kind != AIR_MD_VALUE ||@@ -2065,6 +6542,7 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_pomain->is_entrypoint = true;c.b = nir_builder_at(nir_before_impl(nir_function_impl_create(main)));c.b.impl->structured = false;+ c.const_cursor = nir_before_impl(c.b.impl);c.values = _mesa_pointer_hash_table_create(ctx);c.globals = _mesa_pointer_hash_table_create(ctx);@@ -2110,6 +6588,16 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_pofor (unsigned i = 0; i < (in_node ? in_node->num_ops : 0) && !c.error[0]; i++)translate_input(&c, in_node->ops[i], &next_in_varying);+ /* Now that every direct argument's Metal index is known, the members of the+ * argument buffers can be given bindings that cannot collide with them --+ * and before plan_globals, so that a shader's own `constexpr sampler` still+ * comes after every sampler the API binds.+ */+ if (!c.error[0] && c.num_argbufs)+ allocate_argbuf_indices(&c);+ if (!c.error[0] && options && options->interface_only)+ goto interface_read;+/* the entry point's parameters, as the body will read them */for (unsigned i = 0; i < (wrap ? 0 : c.num_params) && !c.error[0]; i++) {if (c.params[i])@@ -2120,23 +6608,38 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_poif (!c.error[0])plan_globals(&c);+ if (!c.error[0] && c.tg_param_plus1) {+ /* the threadgroup argument past the static arrays, 16-aligned as Metal+ * aligns threadgroup memory; the value is shared with every use of the+ * parameter (air_set keeps the pointer), so setting it here moves them all+ */+ c.shared_size = ALIGN_POT(c.shared_size, 16);+ c.info->has_threadgroup_arg = true;+ c.info->threadgroup_arg_base = c.shared_size;+ if (c.shared_size)+ c.params[c.tg_param_plus1 - 1]->ptr.offset = nir_imm_int(&c.b, c.shared_size);+ }if (!c.error[0])materialise_globals(&c);if (!c.error[0])run_static_initializers(&c);if (!c.error[0]) {- if (wrap) {+ if (stitch) {+ stitch_graph(&c);+ } else if (wrap) {/* air.visible's second operand is the result node, whose one entry* carries air.visible_output and the type's name*/const struct air_md *rn = entry->num_ops > 1 ? entry->ops[1] : NULL;- wrap_library_function(&c, rn && rn->num_ops ? rn->ops[0] : NULL);+ wrap_library_function(&c, rn && rn->num_ops ? rn->ops[0] : NULL,+ entry->num_ops > 2 ? entry->ops[2] : NULL);}elsetranslate_body(&c);}+interface_read:if (c.error[0]) {if (error)*error = ralloc_strdup(mem_ctx, c.error);@@ -2180,6 +6683,15 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_poc.nir->info.workgroup_size_variable = true;}+ /* the interface is all that was asked for: the passes below are for a body */+ if (options && options->interface_only) {+ ralloc_steal(mem_ctx, c.nir);+ if (info)+ keep_info(mem_ctx, info);+ ralloc_free(ctx);+ return c.nir;+ }+nir_lower_goto_ifs(c.nir);/* Fold in any function of the module the shader called, and then drop the@@ -2191,6 +6703,21 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_po* point, which every consumer of this NIR assumes.*/if (_mesa_hash_table_num_entries(c.fns)) {+ /* A callee with more than one `ret` in it. Every return this translator+ * emits is a goto to the impl's end block, and nir_lower_goto_ifs turns+ * the ones that are not the last into nir_jump_return -- which+ * nir_inline_functions ASSERTS on rather than refusing+ * (nir_functions.c: "nir_instr_as_jump(instr)->type !=+ * nir_jump_return"). nir_lower_returns is the pass that removes them,+ * by making the body's own control flow carry the early exit, and it is+ * what spirv_to_nir runs before inlining for the same reason.+ *+ * It is not hypothetical and it is not rare: RenderBox's blur returns+ * early when the kernel is one tap wide, and before this the module+ * aborted the whole air2nir process -- which a report counts as a crash+ * and not as a refusal, so the measurement was lost as well.+ */+ NIR_PASS(_, c.nir, nir_lower_returns);NIR_PASS(_, c.nir, nir_inline_functions);NIR_PASS(_, c.nir, nir_opt_deref);nir_remove_non_entrypoints(c.nir);@@ -2204,14 +6731,8 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_ponir_index_ssa_defs(nir_shader_get_entrypoint(c.nir));if (info) {- /* the info's arrays live under mem_ctx, not under the scratch context */- if (info->num_resources)- info->resources = ralloc_memdup(mem_ctx, info->resources,- info->num_resources * sizeof(*info->resources));- if (info->num_varyings)- info->varyings = ralloc_memdup(mem_ctx, info->varyings,- info->num_varyings * sizeof(*info->varyings));- info->entry_point = ralloc_strdup(mem_ctx, info->entry_point);+ device_writes_from_ir(c.nir, info);+ keep_info(mem_ctx, info);}ralloc_free(ctx);if (error)diff --git a/tools/mesa/air/meson.build b/tools/mesa/air/meson.buildindex 81be917..e1f1f24 100644--- a/tools/mesa/air/meson.build+++ b/tools/mesa/air/meson.build@@ -13,6 +13,7 @@ files_libair = files('air_module.c','air_module.h','air_print.c',+ 'air_stitch.c','air_private.h','air_to_nir.c','air_intrinsics.c',diff --git a/tools/mesa/air/nir_air.h b/tools/mesa/air/nir_air.hindex 5e24918..7a12018 100644--- a/tools/mesa/air/nir_air.h+++ b/tools/mesa/air/nir_air.h@@ -55,11 +55,146 @@ enum air_resource_kind {AIR_RESOURCE_SAMPLER, /* sampler state at .index */};+/* Metal's `constexpr sampler`, as the 64-bit word a shader carries encodes it.+ *+ * The layout is not guessed and not read off one binary: the names, the enum+ * values and the defaults are in Apple's own metal_stdlib header on the device+ * (`GPUCompiler.framework/.../include/metal/metal_texture`, struct+ * _sampler_state and _get_sampler_state), the field positions come from the+ * encoder that writes the word (llvm::air::AIRSamplerStateEncoder::encode in+ * libGPUCompilerImpl) and the decoder that reads it back+ * (AIRConstSampler::constructFlatReply in AGXCompilerCore, with the flatbuffers+ * schema beside it). Three sources, which is what it took: libGPUCompilerImpl+ * also carries its own AIRAddressMode printer in which zero prints as+ * clamp_to_border, and the string clamp_to_zero does not occur in that binary+ * at all -- read alone it gives a plausible and wrong value for the commonest+ * field.+ *+ * Decoding it against the corpus is the check that it is right: the three words+ * the 456 modules carry come out as sampler(coord::pixel),+ * sampler(coord::pixel, filter::linear) and+ * sampler(coord::pixel, filter::linear, address::clamp_to_zero), with every+ * other field equal to the header's default -- including a lod range of+ * [0, 65504], which is two separate 16-bit halves that a wrong layout would+ * not land on.+ */+enum air_sampler_address {+ AIR_SAMPLER_CLAMP_TO_ZERO = 0, /* transparent black, and not a border colour */+ AIR_SAMPLER_CLAMP_TO_EDGE = 1, /* the default */+ AIR_SAMPLER_REPEAT = 2,+ AIR_SAMPLER_MIRRORED_REPEAT = 3,+ AIR_SAMPLER_CLAMP_TO_BORDER = 4,+};++enum air_sampler_filter {+ AIR_SAMPLER_NEAREST = 0, /* the default */+ AIR_SAMPLER_LINEAR = 1,+ AIR_SAMPLER_BICUBIC = 2,+};++enum air_sampler_mip_filter {+ AIR_SAMPLER_MIP_NONE = 0, /* the default */+ AIR_SAMPLER_MIP_NEAREST = 1,+ AIR_SAMPLER_MIP_LINEAR = 2,+};++struct air_sampler_state {+ enum air_sampler_address address[3]; /* s, t, r */+ enum air_sampler_filter mag_filter, min_filter;+ enum air_sampler_mip_filter mip_filter;+ bool pixel_coord; /* coord::pixel; the default is normalized */+ unsigned compare_func;+ unsigned max_anisotropy; /* 1 to 16 */+ uint16_t lod_min, lod_max; /* halves, as their bits */+ unsigned border_color;+ unsigned reduction; /* AGX does not read this field */+};++/* HOW MANY TEXTURE SLOTS A SHADER MAY REACH, and it is freedreno's number and+ * not a round one: fd6_texture_state keeps view_seqno[16] beside textures[32],+ * and a seventeenth view walks out of the first array (RWA_MAX_TEXTURES in+ * air_metal.c has the reading, and a peer session found the fragility). The+ * public contract header cannot include this one -- air_metal.h is standalone+ * so a driver can use it without NIR -- so the two are kept equal by a+ * static assertion in air_metal.c rather than by one of them including the+ * other.+ *+ * It is here, in the frontend's half, because the frontend now INVENTS an+ * index: a null texture is a reserved slot past the module's own (see+ * air_resource.null_texture), and inventing one that nothing can bind would be+ * worse than refusing to.+ */+#define AIR_MAX_TEXTURES 16+struct air_resource {enum air_resource_kind kind;- unsigned index; /* Metal's [[buffer(n)]], [[texture(n)]], [[sampler(n)]] */+ unsigned index; /* Metal's [[buffer(n)]], [[texture(n)]], [[sampler(n)]] --+ * EXCEPT for an argument-buffer member, where it is the+ * index this frontend allocated: see argbuf_set below */unsigned size; /* air.buffer_size when given, else 0 */bool read, write;++ /* Whether the TRANSLATED SHADER stores through this binding -- a different+ * fact from `write` above and read out of a different place, kept beside it+ * rather than folded into it so that a consumer can compare the two.+ *+ * `write` is Apple's DECLARATION, out of `air.write`/`air.read_write`: a+ * `device T *` argument is writable whether or not the body ever stores+ * through it, and most of them never do. `ir_write` is what the IR the+ * backend will compile actually contains. A runtime that must know whether+ * a write can be LOST -- rwa_draw's unaligned `device` buffer, which can be+ * uploaded like a `constant` one only if nothing writes it -- wants this+ * one, and wants the declaration as a cross-check rather than as the+ * answer.+ *+ * Only ever set for AIR_RESOURCE_DEVICE_BUFFER: a `constant` buffer cannot+ * be stored to (air_store_ptr refuses it), and a texture or an image is not+ * reached through a buffer binding.+ */+ bool ir_write;++ /* A TEXTURE SLOT THE SHADER ASKED FOR AND THE APPLICATION DOES NOT BIND: the+ * null texture (`air.get_null_texture_2d` and its fifteen siblings), which a+ * Metal shader asks for when a pipeline bound no image. There is no index+ * that means "nothing bound" -- a texture here is Metal's own+ * `[[texture(n)]]`, resolved while translating -- so the frontend reserves+ * one past the module's highest declared index and says so with this flag,+ * which is Apple's own structure: where its textures are indices into a+ * heap, its null texture is a reserved index in that heap+ * (AGCLLVMUserObject::replaceTexturesAndSamplers, docs/air.md section 3).+ *+ * WHAT THE RUNTIME OWES IT, and what it must NOT do: create a 1x1+ * zero-filled texture of the slot's own type and bind it there, and never+ * demand that the application bind one. `param` is ~0u for it, as for an+ * inline sampler, because no entry-point parameter carries it.+ *+ * AND THE ASSUMPTION IT RESTS ON, written here rather than implied: a zero+ * texture is the right answer only if Metal's null texture reads as zero,+ * and nothing this project can reach says that it does -- Apple's compiler+ * substitutes a handle and leaves the result to the descriptor, so there is+ * no semantics in it to borrow (docs/air.md section 3 has the reading). What+ * this buys is not certainty but its LOCATION: the assumption lives in one+ * 1x1 texture a boot can change, instead of being folded into every sample+ * the frontend emits.+ *+ * WHICH TEXTURES TO MAKE IS NOT ONE PER SLOT. The same slot is sampled as+ * both 2D and 3D in fixed_frag_lph_cph, so a caller needs one per (type,+ * element class) PAIR -- rwa_texture_decl.sampled_as reports the set, and+ * reports an empty one for a slot no instruction reads, which means bind+ * nothing rather than bind everything.+ *+ * AND NOTHING YET SAYS THE TWO MODULES THIS EXISTS FOR ARE CORRECT. They+ * translate and ir3 compiles them; that is all. Softpipe runs neither (f16+ * registers, varyings, and an assertion in nir_to_tgsi) and cpuref compares+ * neither (air.* calls LLVM has no body for, no rasteriser for the+ * varyings), so the Adreno is the only machine that can say anything about+ * them. And a pipeline that DRAWS is not evidence either: before this it was+ * refused outright, so anything it draws is new rather than better. The+ * reading that would mean something is a difference from the same boot+ * without it.+ */+ bool null_texture;+unsigned param; /* the entry point's parameter; ~0u for an inline sampler */const char *name; /* air.arg_name */const char *type_name;@@ -70,7 +205,45 @@ struct air_resource {* samplers. The word's layout is not decoded yet -- see docs/air.md.*/bool inline_sampler;- uint64_t sampler_state;+ uint64_t sampler_state; /* the word itself, for a runtime that wants it raw */+ struct air_sampler_state sampler; /* ... and decoded */++ /* Metal's `[[raster_order_group(n)]]` on this resource: accesses to it from+ * overlapping fragments happen in rasterization order. -1 when the shader+ * asked for none. a6xx has no per-resource ordering at all -- see+ * air_shader_info.raster_order_groups for what the runtime has to do, and+ * for the note the frontend leaves when a shader asks for this.+ */+ int raster_order_group;++ /* A member of an ARGUMENT BUFFER (`air.indirect_buffer`, Metal's+ * `[[id(n)]]` struct) rather than an argument of its own, and the one place+ * in this structure where `.index` is not Metal's number: ~0u when this+ * resource is a direct argument, otherwise the Metal `[[buffer(n)]]` the+ * SET is bound at and the member's own `[[id(n)]]` inside it.+ *+ * Both halves are needed because the two ends name the member differently+ * and neither name can be derived from the other. The application names it+ * by (set, id): that is what it passes to MTLArgumentEncoder. A member's own+ * `air.location_index` is per set -- set 0's member 0 and set 1's member 0+ * both say 0, and 237 of Resident Evil 4's modules have two sets -- so it is+ * not a binding. The runtime binds by `.index`, which this frontend+ * allocates: one flat counter per kind (buffer, texture, sampler), in+ * (set, member) order, continuing above the highest index any DIRECT+ * argument of that kind already took, so that a member and an argument+ * cannot land in one Gallium slot.+ *+ * The allocation is over every member the set declares, used or not, so+ * that it depends on the metadata alone: the runtime's encoder has to answer+ * `encodedLength` for a set whichever of its members this shader reads, and+ * a numbering that skipped the unread ones would differ between two shaders+ * sharing one set. See docs/air.md section 8.2.+ */+ unsigned argbuf_set;+ unsigned argbuf_id;+ /* ...and where its eight bytes are in the set (air_argbuf_member.offset):+ * what the runtime reads to find what was encoded there */+ unsigned argbuf_offset;};struct air_varying {@@ -80,6 +253,84 @@ struct air_varying {bool flat, centroid, sample, no_perspective;};+/* What the shader wants of each component of a vertex attribute. The LLVM type+ * cannot say whether an integer is signed -- `int4` and `uint4` are both+ * `<4 x i32>` -- so the sign comes from `air.arg_type_name`, Apple's own+ * reflection string, and from nowhere else. A runtime that checks an+ * MTLVertexFormat against this is checking the shader's declared type against+ * the format's converted type, which is what Metal's own pipeline validation+ * checks.+ */+enum air_attribute_base_type {+ AIR_ATTRIB_FLOAT, /* `float`, `half` */+ AIR_ATTRIB_INT, /* `char`, `short`, `int` -- and the fallback with no type name */+ AIR_ATTRIB_UINT, /* `uchar`, `ushort`, `uint` */+};++/* One member of a vertex function's `[[stage_in]]` struct: Metal's+ * `[[attribute(n)]]`.+ *+ * AIR does not present the struct at all. Apple's compiler FLATTENS it: every+ * member is an ordinary parameter of the entry point with its own+ * `air.vertex_input` node, and the struct type is nowhere in the module. So+ * there is no layout here and none can be inferred -- an attribute's format,+ * offset, stride and buffer are the MTLVertexDescriptor's, which lives in the+ * API and not in the shader. This table is the shader's HALF of that match:+ * which indices it reads and what it wants each to be.+ */+struct air_vertex_attribute {+ unsigned index; /* Metal's `[[attribute(n)]]`, `air.location_index` */+ unsigned location; /* VERT_ATTRIB_GENERIC0 + index, the NIR input's location */+ unsigned components; /* 1 to 4 */++ /* The NIR input's driver_location, and the one field here a runtime cannot+ * do without: Gallium matches vertex elements to shader inputs BY POSITION,+ * not by location. ir3's setup_input indexes `so->inputs[]` by+ * nir_intrinsic_base -- the driver_location nir_lower_io wrote -- and+ * fd6_vertex_state_create emits VFD_DECODE[i] from `elements[i]` while+ * emit_vfd_dest emits VFD_DEST_CNTL_INSTR[i] from `vs->inputs[i]`. So+ * pipe_vertex_element[i] feeds the input whose driver_location is i, and+ * these are assigned in the order AIR lists the attributes with the UNUSED+ * ones skipped -- for std_vert0_lpf, 0 is attribute 0 and 1 is attribute 3.+ * ~0u when `used` is false.+ */+ unsigned driver_location;++ /* The width the SHADER reads, 8, 16 or 32. The NIR input is 32 bits wide+ * whatever this says, and the narrowing is an instruction in the shader --+ * see docs/air.md: ir3 puts a 16-bit vertex input in the HALF register file+ * and then reports it in `ir3_shader_variant.inputs[]`, which has no half+ * flag, as a full register, which is the number fd6_program.cc programs+ * VFD_DEST_CNTL_INSTR_REGID from. A 32-bit input keeps the hardware fetch+ * in the only regime either freedreno driver uses.+ */+ unsigned bit_size;+ enum air_attribute_base_type base_type;++ const char *name; /* air.arg_name */+ const char *type_name; /* air.arg_type_name: "float4", "half4", "uchar4" ... */++ /* False when Apple's metadata marks the parameter `air.arg_unused`: the+ * attribute is in the shader's interface and nothing reads it, so there is+ * NO NIR input for it and the runtime need not bind anything. Eight of+ * QuartzCore's fourteen `[[stage_in]]` vertex shaders have one, and a fetch+ * set up for a value nothing reads costs a VFD slot and a binding the+ * application's descriptor may legally not have configured at all.+ */+ bool used;++ /* The implicit function-constant predicate that gates this attribute, as+ * Apple's metadata names it (`vtx_attr_has_texcoords`), or NULL. The+ * parameter EXISTS either way -- the entry point's signature is fixed --+ * so the attribute is translated either way; what the predicate decides is+ * whether the shader reads it, and the value of that predicate comes from+ * the function constants the pipeline supplies. There is no index to give:+ * these globals are not in `!air.function_constants`, they are computed by+ * the module's `air.static_init` constructor from the constants that are.+ */+ const char *function_constant;+};+struct air_shader_info {gl_shader_stage stage; /* MESA_SHADER_NONE for an air.visible function */const char *entry_point;@@ -87,17 +338,184 @@ struct air_shader_info {struct air_resource *resources;unsigned num_varyings; /* vertex outputs or fragment inputs */struct air_varying *varyings;++ /* The `[[stage_in]]` members of a vertex function, in the order AIR lists+ * them (which is the order of the entry point's parameters and NOT the+ * order of the attribute indices: QuartzCore's std_vert0_lpf declares+ * 0, 1, 3). Empty for every other stage, and empty for a vertex shader that+ * reads its vertices out of a buffer indexed by air.vertex_id -- Apple's+ * three in the GL renderer and twenty-five of QuartzCore's thirty-nine.+ */+ unsigned num_vertex_attributes;+ struct air_vertex_attribute *vertex_attributes;++ /* One bit per Metal attribute index that the shader actually READS: the set+ * the pipeline's MTLVertexDescriptor must configure. An attribute marked+ * air.arg_unused is in vertex_attributes and not here. One caveat, which the+ * translation also leaves as a note: an attribute gated by a function+ * constant has its bit set whatever the constant says, because the parameter+ * exists either way -- see air_vertex_attribute.function_constant before+ * demanding a binding for a bit of this mask.+ */+ unsigned vertex_attributes_read;+unsigned num_render_targets;+ /* The render targets a fragment shader READS -- Metal's `[[color(n)]]` argument,+ * framebuffer fetch -- one bit per target. It is separate from num_render_targets,+ * which is what the stage writes, because the two need not overlap: QuartzCore's+ * `inplace_copy` reads target 0 and declares no colour output at all, and its+ * `path_blit_frag` reads target 2 while writing 0, 1 and 2.+ */+ unsigned fbfetch_targets;++ /* The raster order groups the shader declares on its resources, one bit+ * each; 0 when it declares none.+ *+ * a6xx cannot give what this asks for. Metal's guarantee is per RESOURCE --+ * two fragments that overlap are ordered against each other for the+ * resources in a group and not for anything else -- and the only knob the+ * hardware has is per DRAW: GRAS_SC_CNTL.SINGLE_PRIM_MODE, which+ * freedreno drives from pipe_blend_state.blend_coherent (fd6_emit.cc,+ * build_prim_mode). So a runtime that sets blend_coherent for any pipeline+ * whose shader has a non-zero raster_order_groups gives the shader MORE+ * synchronisation than it asked for, which is safe and slower; there is no+ * way to give exactly what it asked for, and no way to give less.+ *+ * The frontend does not refuse such a shader -- the coarser guarantee is+ * still a correct one -- but it does not pass over it in silence either:+ * every resource with a group leaves a note in `notes`, which+ * `air2nir --report` prints and tools/air/corpus.py counts.+ */+ unsigned raster_order_groups;++ /* What was translated with a caveat: a shader this frontend accepted while+ * giving something other than exactly what the module asked for. Nothing+ * here is an error -- an error is a refusal and air_to_nir returns NULL for+ * it -- and nothing here is advice. Each note names a thing the module asked+ * for and what was done instead, so that it is in the log at translation+ * time rather than only in a document.+ */+ unsigned num_notes;+ const char **notes;+unsigned threadgroup_bytes; /* static threadgroup memory */++ /* A [[threadgroup(0)]] argument, sized by the encoder (setThreadgroupMemoryLength:):+ * it starts at threadgroup_arg_base bytes, past the static arrays, so the runtime+ * gives the dispatch threadgroup_arg_base + that length.+ */+ bool has_threadgroup_arg;+ unsigned threadgroup_arg_base;};+/* The word, field by field. A runtime that holds only `sampler_state` can call+ * this; `air_resource.sampler` is what it returns.+ */+struct air_sampler_state air_decode_sampler_state(uint64_t word);+/* -- the translator --------------------------------------------------------- */struct air_constant_value {unsigned index; /* the function constant's index */uint64_t bits; /* its value, as the bits of its type */++ /*+ * Whether the PIPELINE actually supplied that value, which is not the same question+ * as whether the module declares the constant.+ *+ * air_module_function_constants() registers every constant a module declares, because+ * it reads a declaration; a declaration carries no value, so `bits` is zero until a+ * runtime fills it in. Those two facts together used to make+ * air.is_function_constant_defined answer TRUE for every constant and then hand the+ * module a zero -- and 80 of the corpus's 203 function constants exist precisely to+ * be asked that question, every one of them guarded by that intrinsic with its own+ * branch for "it was not supplied". Answering yes and giving zero takes the wrong+ * branch with an invented value.+ *+ * rzalloc_array leaves this false, so a harness or a runtime that fills nothing says+ * "nothing was supplied", which is the truth about it.+ */+ bool supplied;+};++/* -- function stitching ----------------------------------------------------+ *+ * Metal's own way of using an air.visible library function: the runtime hands+ * the device a GRAPH of calls and gets back a function compiled out of them+ * (-[MTLDevice newLibraryWithStitchedDescriptor:error:], and the SPI variant+ * CoreImage actually uses). These three structures are that graph and nothing+ * else: one field per property of Apple's own objects, read out of the+ * device's shared cache rather than invented here, so that a runtime filling+ * them in is transcribing a descriptor and not translating between two models.+ *+ * MTLFunctionStitchingGraph -> air_stitch_graph (functionName, nodes, outputNode)+ * MTLFunctionStitchingFunctionNode -> air_stitch_node (name, arguments)+ * MTLFunctionStitchingInputNode -> AIR_STITCH_INPUT (argumentIndex)+ * MTLFunctionStitchingInputTexture -> AIR_STITCH_TEXTURE (bindIndex), and so+ * ...InputSampler, ...InputBuffer on for the other SPI input nodes+ * ...BuiltinThreadPositionInGrid -> AIR_STITCH_GRID_POSITION+ *+ * What is NOT here is the stitching ABI's four functions (load_argument,+ * copy_from_buffer, store_return_value, destroy). They are in the modules, with+ * bodies, and the graph says nothing about them: the frontend reads them from+ * the metadata of each function it stitches (docs/air.md section 5, "The+ * stitching ABI, before any of it is built").+ */+enum air_stitch_arg_kind {+ AIR_STITCH_NODE, /* another node's result; index is the node */+ AIR_STITCH_INPUT, /* the stitched function's own argument i */+ AIR_STITCH_TEXTURE, /* a texture bound at index */+ AIR_STITCH_SAMPLER, /* a sampler bound at index */+ AIR_STITCH_BUFFER, /* a constant buffer bound at index */+ AIR_STITCH_GRID_POSITION, /* thread_position_in_grid */+};++struct air_stitch_arg {+ enum air_stitch_arg_kind kind;+ unsigned index;+};++struct air_stitch_node {+ const char *function; /* the air.visible function this node calls */+ unsigned num_args;+ const struct air_stitch_arg *args;+};++struct air_stitch_graph {+ const char *name; /* the stitched function's name */+ unsigned num_nodes;+ const struct air_stitch_node *nodes; /* in dependency order */+ unsigned output_node; /* whose result the function returns */+};++/* One function available to a graph: a module of the library, by name. The+ * nodes of a graph name functions that live in DIFFERENT modules of the same+ * .metallib -- every one of CoreImage's 135 is a module of its own -- so a+ * stitched shader is built out of several bitcode blobs, which is why this+ * exists at all.+ */+struct air_stitch_function {+ const char *name;+ const void *bitcode;+ size_t size;};+/* A graph written as text, and the functions it names, for the tools that have+ * to type one: "f(in0); g(n0)" (air_stitch.c says the whole syntax). The error+ * buffer is the caller's, and gets one sentence with no prefix.+ */+const struct air_stitch_graph *air_stitch_parse(void *mem_ctx, const char *text,+ char *error, size_t size);++/* Every distinct function a graph names, found in an MTLB container by name:+ * the nodes of one graph live in different modules of one .metallib. Returns+ * how many, or 0 with *error set.+ */+unsigned air_stitch_functions_of(void *mem_ctx, const struct air_stitch_graph *g,+ const void *data, size_t size,+ struct air_stitch_function **out,+ char *error, size_t esize);+struct air_to_nir_options {const struct air_constant_value *constants;unsigned num_constants;@@ -112,6 +530,35 @@ struct air_to_nir_options {* entry point of the corpus and is counted separately.*/bool library_wrapper;++ /* Stitch a graph of air.visible functions into one entry point, instead of+ * translating the module handed to air_to_nir as a shader of its own. The+ * graph's nodes name functions of `stitch_functions`; the module passed to+ * air_to_nir must be one of them (its own visible function is then just+ * another node, and nothing else about it is used).+ *+ * The entry point generated is the library wrapper's, extended: argument i+ * of the STITCHED function is read from device buffer 0 at byte 16*i and the+ * output node's result is written to device buffer 1. That is deliberate and+ * not a convenience -- a one-node graph then has to produce exactly the+ * words `library_wrapper` produces for the same function, which is a+ * reference that already exists for 78 of the 135 and is what makes the+ * graphs above it decidable (docs/air.md section 5).+ */+ const struct air_stitch_graph *stitch;+ const struct air_stitch_function *stitch_functions;+ unsigned num_stitch_functions;++ /* Read the entry point's interface -- its outputs, its inputs and every+ * member of its argument buffers, with the bindings allocated as a full+ * translation allocates them -- and stop there: *info is complete, the body+ * is not translated and the NIR returned has none. For what a runtime has+ * to answer before any pipeline exists: MTLFunction's+ * -newArgumentEncoderWithBufferIndex: is asked of functions whose bodies+ * this frontend cannot translate yet (Resident Evil 4's culling shaders,+ * air.simd_ballot), and the layout of a set is in the metadata alone.+ */+ bool interface_only;};/* Translates one AIR module (one function of a .metallib). entry_point maydiff --git a/tools/mesa/ir3-shift-split.patch b/tools/mesa/ir3-shift-split.patchnew file mode 100644index 0000000..9b49588--- /dev/null+++ b/tools/mesa/ir3-shift-split.patch@@ -0,0 +1,84 @@+ir3: moving a shift inside an add changes the number when the constant is+negative.++create_shift() in ir3_nir_lower_io_offsets.c turns the byte offset of an SSBO+access into an element offset. When the offset has the shape `iadd constant,+foo` it rewrites `(C + x) >> s` as `(x >> s) + (C >> s)`, so that+nir_opt_offsets can fold the constant into the instruction's immediate field.++That identity does not hold for a negative constant. `buf[i - 1].field` is+ordinary code, and LLVM hands it over as `iadd 0xfffffffc, i*24` -- a+subtraction. `(i*24 - 4) >> 2` borrows from the bits above the two it drops;+`(i*24 >> 2) + (0xfffffffc >> 2)` adds 0x3fffffff instead of 0xffffffff and+does not. The two differ by 2^30, which puts the element offset a gigabyte past+the end of the buffer, and an out-of-range `ldib` returns zero without saying+so. It also drops the low bits of a constant that is not a whole number of+elements, which is the same mistake in a smaller way.++Measured on the phone's Adreno 7c+ Gen 3. `fbs_bistochastize_iter` of+PortraitFilters.metallib reads `grid_buf[id - 1].n` through exactly this shape+and sums `id + 1` terms when the read returns 0 instead of `id`. With input+built from the module's own metadata (tools/air/kinput.py, docs/air.md section+5) its four outputs should all be 1/sqrt(3) -- one term each -- and on the+device they were:++ word device 1/sqrt(n) Apple's own LLVM on the host+ 0 0x3f13cd3a 1/sqrt(3) 0x3f13cd3a+ 1 0x3f000000 1/sqrt(4) 0x3f13cd3a+ 2 0x3ee4f92e 1/sqrt(5) 0x3f13cd3a+ 3 0x3ed105ec 1/sqrt(6) 0x3f13cd3a++1/sqrt(3+i) in word i is the signature of the read returning 0 for every+thread, and 0 is what that address returns. The three fbs_* kernels of the+corpus carry this shape; the eight MetalFaceMask_* ones do not.++So split the constant out only when it survives being shifted and cannot be a+subtraction. Everything else keeps the shift outside the add, which is what the+code did before the folding and is always right.++--- a/src/freedreno/ir3/ir3_nir_lower_io_offsets.c++++ b/src/freedreno/ir3/ir3_nir_lower_io_offsets.c+@@ -126,6 +126,32 @@+ return new_offset;+ }+++/* Whether `(c + x) >> shift` and `(x >> shift) + (c >> shift)` are the same++ * number for every x, which is what moving the shift inside the add assumes.++ *++ * They are not, twice over. A constant whose low `shift` bits are set loses++ * them to `c >> shift`, so the offset comes out short. Worse, a negative++ * constant -- a struct field reached through `buf[i - 1]`, which is ordinary++ * code -- is a subtraction, and `(x - 4) >> 2` borrows from the bits above++ * while `(x >> 2) + (0xfffffffc >> 2)` adds 0x3fffffff and does not: the two++ * differ by 2^30 and the SSBO read lands outside the buffer, where a6xx++ * returns zero without saying so.++ *++ * That was measured: fbs_bistochastize_iter reads grid_buf[id - 1].n with++ * exactly this shape, and on an Adreno 7c+ Gen 3 it read 0 for every thread,++ * which turned 1/sqrt(3) in all four of its outputs into 1/sqrt(3), 1/sqrt(4),++ * 1/sqrt(5), 1/sqrt(6) (docs/air.md section 5).++ *++ * So split only a constant that is a whole number of elements and cannot be a++ * subtraction. Everything else keeps the shift outside the add, which is what++ * the code did before the folding was added and is always right.++ */++static bool++shift_splits_over_iadd(uint32_t c, int shift)++{++ return (c & BITFIELD_MASK(shift)) == 0 && c < (1u << 31);++}+++ static nir_def *+ create_shift(nir_builder *b, nir_def *offset, int shift)+ {+@@ -138,7 +164,9 @@+ nir_alu_instr *offset_instr = nir_instr_as_alu(offset->parent_instr);++ if (offset_instr->op == nir_op_iadd &&+- nir_src_is_const(offset_instr->src[0].src)) {++ nir_src_is_const(offset_instr->src[0].src) &&++ shift_splits_over_iadd(++ nir_src_as_const_value(offset_instr->src[0].src)->u32, shift)) {+ nir_def *new_shift = ir3_nir_try_propagate_bit_shift(+ b, offset_instr->src[1].src.ssa, -shift);+diff --git a/tools/mesa/ir3-ssbo-subdword.patch b/tools/mesa/ir3-ssbo-subdword.patchnew file mode 100644index 0000000..c8fce59--- /dev/null+++ b/tools/mesa/ir3-ssbo-subdword.patch@@ -0,0 +1,121 @@+ir3: an SSBO is a buffer of dwords to the Gallium driver, so keep sub-dword+access off it.++An SSBO on a6xx/a7xx is reached through an IBO descriptor, and `ldib`/`stib`+take an offset in *that descriptor's* texels -- the hardware multiplies the+index by the texel size the descriptor declares. ir3 does the dividing on the+other side, in ir3_nir_lower_io_offsets.c's lower_offset_for_ssbo(), and it+divides by the size of the *instruction*: shift 2 for a 32-bit access, 1 for+16-bit, 0 for 8-bit. The two only agree when the descriptor's texel is as wide+as the access.++Turnip keeps them agreeing by writing a descriptor per width and picking one:+write_buffer_descriptor_addr() lays down FMT6_16_UINT, FMT6_32_UINT and+FMT6_8_UINT descriptors next to each other, and tu_lower_io()'s+`descriptor_idx + 1` selects the 8-bit one for an 8-bit load or store.++The Gallium driver writes one descriptor per SSBO slot and it is always+PIPE_FORMAT_R32_UINT (fd6_image.cc, fd6_ssbo_descriptor()), with nothing that+selects another. So on that driver a 16-bit SSBO access addresses twice the+byte offset it was given and an 8-bit one four times, and an 8-bit store is+worse than mis-addressed: ir3_a6xx.c emits it as `stib.b.typed...u16.4`, which+is the blob's encoding for an imageBuffer of VK_FORMAT_R8 and means nothing+against a 32-bit descriptor. The driver also leaves+ir3_compiler_options::storage_16bit and storage_8bit false, which is already+how ir3 knows the difference elsewhere -- ir3_nir_lower_ssbo_size() shifts the+buffer length by 1 or 2 depending on exactly that, "a byte length divided by+IBO_0_FMT".++Nothing in GL reaches it, which is why it has stood: a GLSL SSBO member is at+least 32 bits and at least 4-byte aligned, so ir3_mem_access_size_align()'s+`(bytes & 1)` and `(bytes & 2)` cases never fire. A Metal shader is a different+matter -- `ushort` and `uchar` fields in a device buffer are ordinary there,+and the AIR frontend (src/compiler/air) translates them to 16- and 8-bit+load_ssbo/store_ssbo.++Measured on the phone's Adreno 7c+ Gen 3, two of Apple's compute kernels from+PortraitFilters.metallib, given input built from each module's own metadata+(docs/air.md section 5):++ MetalFaceMask_AddSeedPoint one `uchar` store, to byte 28 of the span+ table. The dispatch finished and byte 28 was+ unchanged; the CPU reference over the same+ bytes writes 0x5a there.+ MetalFaceMask_PropagateParents walks a linked list of spans, leaving when+ `nextSpanID` is 0xffff. The dispatch never+ finished. Its `nextSpanID` reads land four+ bytes apart instead of two, which turns the+ lists 0, 1, 2, 3 of the input into the cycle+ 0 -> 1 -> 3 -> 0, and that loop has no end.++The fix is to ask for dwords whenever the descriptor is one, in+ir3_mem_access_size_align(), where every other size decision is already made.+nir_lower_mem_access_bit_sizes then does the work: a narrow load becomes a dword+load at the masked offset plus a shift, and a narrow store becomes the iand/ior+pair of 32-bit SSBO atomics the pass already keeps for this, which is why+may_lower_unaligned_stores_to_atomics has to be set. Atomics rather than a plain+read-modify-write matter here: these kernels have sixteen threads writing `uchar`+fields that share a dword.++Turnip is unaffected where it sets the options, and is fixed by the same change+on a device where it cannot (a6xx without 16-bit storage, where an unaligned+32-bit access is split into 16-bit ones against a 32-bit descriptor).++cb_data was unset at both call sites, so it carries the compiler now; turnip's+other call (push constants, tu_shader.cc) leaves it null and never sees an SSBO+intrinsic, and a null is read as "no sub-dword descriptor" either way.++--- a/src/freedreno/ir3/ir3_nir.c++++ b/src/freedreno/ir3/ir3_nir.c+@@ -953,6 +953,8 @@+ uint32_t align_offset, bool offset_is_const,+ enum gl_access_qualifier access, const void *cb_data)+ {++ const struct ir3_compiler *compiler = cb_data;+++ align = nir_combined_align(align, align_offset);+ assert(util_is_power_of_two_nonzero(align));++@@ -970,6 +972,29 @@+ if (intrin == nir_intrinsic_load_ubo)+ bit_size = 32;+++ /* ldib/stib scale the offset by the texel size of the IBO descriptor, while++ * lower_offset_for_ssbo() scales it by the size of the access. A driver with++ * no narrower descriptor to point them at must therefore stay on dwords:++ * otherwise a 16-bit access reads and writes at twice the byte offset it was++ * given and an 8-bit one at four times -- and an 8-bit store is emitted as a++ * typed R8 store, which a 32-bit descriptor does not describe at all.++ *++ * nir_lower_mem_access_bit_sizes turns what is left into a dword load plus a++ * shift, or a store into the pair of 32-bit atomics it keeps for this, so the++ * narrow access still happens -- it happens through a descriptor that exists.++ */++ if ((intrin == nir_intrinsic_load_ssbo || intrin == nir_intrinsic_store_ssbo) &&++ ((bit_size == 16 && (!compiler || !compiler->options.storage_16bit)) ||++ (bit_size == 8 && (!compiler || !compiler->options.storage_8bit)))) {++ bit_size = 32;++ /* Below a dword of alignment the pass reads with a mask and a shift and++ * writes with a pair of atomics, and both of those work a dword at a++ * time -- so ask for one, however many narrow components there were.++ */++ if (align < 4)++ bytes = MIN2(bytes, 4);++ }+++ return (nir_mem_access_size_align){+ .num_components = MAX2(1, MIN2(bytes / (bit_size / 8), 4)),+ .bit_size = bit_size,+@@ -1092,6 +1117,12 @@+ nir_var_mem_global | nir_var_mem_shared |+ nir_var_function_temp | nir_var_mem_ssbo,+ .callback = ir3_mem_access_size_align,++ /* so the callback can ask which descriptors this driver writes */++ .cb_data = (void *)so->compiler,++ /* a narrow store the callback widened becomes an iand/ior pair of 32-bit++ * atomics; threads that share a dword then do not lose each other's bytes++ */++ .may_lower_unaligned_stores_to_atomics = true,+ };++ progress |= OPT(s, nir_lower_mem_access_bit_sizes, &mem_bit_size_options);