Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

tools/mesa: vcvkk's AIR front end and two ir3 fixes, the ones hakios-metal's compiler is written against

shimoku7409ebcf8a9354
diff --git a/tools/mesa/air-ir3.patch b/tools/mesa/air-ir3.patch
index 823d288..2a496df 100644
--- a/tools/mesa/air-ir3.patch
+++ b/tools/mesa/air-ir3.patch
@@ -29,7 +29,7 @@ dimensions -- and hides on the rest, which is presumably why it survived.
#include "pipe/p_context.h"
-@@ -247,10 +248,62 @@
+@@ -247,10 +248,69 @@
return nir;
}
@@ -68,9 +68,16 @@ dimensions -- and hides on the rest, which is presumably why it survived.
+
+ struct air_constant_value *fc;
+ unsigned num_fc = air_module_function_constants(NULL, bitcode, bitcode_size, &fc);
++ /* An air.visible library function is not an entry point; the wrapper makes
++ * one that calls it, so that these 135 modules can be compiled for the
++ * Adreno at all (nir_air.h, docs/air.md section 5). Always on here: this
++ * tool exists to answer "does ir3 accept it", and a module that is not an
++ * entry point has no other way to be asked.
++ */
+ const struct air_to_nir_options options = {
+ .constants = fc,
+ .num_constants = num_fc,
++ .library_wrapper = true,
+ };
+ struct air_shader_info info;
+ nir_shader *nir = air_to_nir(NULL, bitcode, bitcode_size, NULL, &options,
diff --git a/tools/mesa/air/air2nir.c b/tools/mesa/air/air2nir.c
index c4f89ea..f719cf4 100644
--- a/tools/mesa/air/air2nir.c
+++ b/tools/mesa/air/air2nir.c
@@ -10,6 +10,8 @@
* air2nir --ll FILE [FUNCTION] print what the bitcode reader read, as LLVM assembly
* air2nir --list FILE.metallib the functions of a container
* air2nir --report FILE... one line per module, for tools/air/corpus.py
+ * air2nir --interface FILE [FUNC] what the shader expects of the runtime
+ * (air_shader_info), key=value per line
*
* FILE is a .metallib (then FUNCTION picks one; without it, every function)
* or one module's bitcode, wrapped or not. --report forks for every module,
@@ -111,6 +113,20 @@ stage_name(int stage)
}
}
+/* --stitch: a graph of air.visible functions compiled into one entry point.
+ *
+ * The graph is Metal's own (MTLFunctionStitchingGraph and its nodes) and is
+ * typed on the command line the way air_stitch_parse reads it:
+ *
+ * "_pf_invertRed(in0); _pf_redContour(n0); _pf_textureDiff(n0, n1)"
+ *
+ * The functions it names are looked up in the container by name, because the
+ * nodes of one graph live in different modules of one .metallib.
+ */
+static const struct air_stitch_graph *stitch = NULL;
+static struct air_stitch_function *stitch_fns = NULL;
+static unsigned num_stitch_fns = 0;
+
/* -- evaluating a shader by folding it -------------------------------------
*
* For the class of shader that only reads buffers and computes -- 256 of the
@@ -151,7 +167,13 @@ library_byte(unsigned i)
static uint8_t
pattern_byte(unsigned i)
{
- return library_wrapper ? library_byte(i) : (i * 7 + 13) & 0xff;
+ /* A stitched graph reads the wrapper's argument slots, so it is filled the
+ * wrapper's way. Not a detail: with the byte pattern a one-node graph gave
+ * `1 0 0 1` where the wrapper gave `4.5 0 0 1` for the same function, which
+ * reads exactly like a mistranslation and was two harnesses handing the
+ * same shader different numbers.
+ */
+ return (library_wrapper || stitch) ? library_byte(i) : (i * 7 + 13) & 0xff;
}
struct eval_state {
@@ -293,6 +315,55 @@ print_buffer_stores(nir_shader *nir, const struct air_shader_info *info)
return printed;
}
+/* THE CONSTANTS A PIPELINE WOULD HAVE SUPPLIED, for `--fc INDEX=VALUE`.
+ *
+ * Without this the only constant set this tool can translate against is the
+ * one air_module_function_constants builds -- every declared constant, value
+ * zero, `supplied` false -- and that is not any pipeline's. It matters because
+ * which blocks a shader even HAS now depends on them (plan_global_folding in
+ * air_to_nir.c), so "does this module translate" is a question about a
+ * configuration and not about the module alone: fixed_frag_lph_cph stops in
+ * three different places under three different constant sets.
+ *
+ * air_run has had this option for a while and is the wrong place to reproduce
+ * a translation from: it runs the shader, so a module softpipe cannot execute
+ * -- which this one cannot -- aborts in the Gallium driver before anything is
+ * printed, and the translation's own verdict is lost with the buffer.
+ *
+ * 64 and not 32 because fixed_frag_lph_cph declares 58 (the same bound as
+ * fc_set in air_run.c and RW_MAX_CONSTANTS in air_metal.h).
+ */
+static struct {
+ unsigned index;
+ uint64_t bits;
+} fc_set[64];
+static unsigned num_fc_set;
+
+/* An index the module does not declare is refused rather than ignored: an
+ * option that silently sets nothing would make every run agree with every
+ * other one for the wrong reason. Mirrors air_run's apply_fc_values.
+ */
+static bool
+apply_fc_set(struct air_constant_value *fc, unsigned num_fc)
+{
+ for (unsigned i = 0; i < num_fc_set; i++) {
+ unsigned j;
+ for (j = 0; j < num_fc; j++) {
+ if (fc[j].index == fc_set[i].index) {
+ fc[j].bits = fc_set[i].bits;
+ fc[j].supplied = true;
+ break;
+ }
+ }
+ if (j == num_fc) {
+ fprintf(stderr, "air2nir: this module declares no function constant %u "
+ "(it declares %u)\n", fc_set[i].index, num_fc);
+ return false;
+ }
+ }
+ return true;
+}
+
static int
eval_module(const struct module_ref *mod)
{
@@ -300,8 +371,13 @@ eval_module(const struct module_ref *mod)
const char *err = NULL;
struct air_constant_value *fc;
unsigned num_fc = air_module_function_constants(ctx, mod->bitcode, mod->size, &fc);
+ if (!apply_fc_set(fc, num_fc))
+ return false;
struct air_to_nir_options options = { .constants = fc, .num_constants = num_fc,
- .library_wrapper = library_wrapper };
+ .library_wrapper = library_wrapper,
+ .stitch = stitch,
+ .stitch_functions = stitch_fns,
+ .num_stitch_functions = num_stitch_fns };
struct air_shader_info info;
nir_shader *nir = air_to_nir(ctx, mod->bitcode, mod->size, NULL, &options,
&default_options, &info, &err);
@@ -324,8 +400,30 @@ eval_module(const struct module_ref *mod)
struct eval_state st = { .info = &info };
nir_shader_intrinsics_pass(nir, fold_buffer_loads, nir_metadata_control_flow, &st);
- /* fold what the loads made foldable, and nothing else: no algebraic
- * rewriting, so the arithmetic is IEEE and comparable with LLVM's
+ /* Fold what the loads made foldable, and nothing else: no algebraic
+ * rewriting, so the arithmetic is IEEE and comparable with LLVM's.
+ *
+ * Control flow is simplified, though, and that is not a rewriting of any
+ * arithmetic: once a comparison has folded, the branch under it is dead and
+ * the phi below it has one live source. Without this, 20 of the corpus's
+ * library functions had no folded value at all -- every one of them is
+ * straight-line arithmetic behind one `if` -- and a shader with a branch
+ * could never be compared against LLVM, which has no such trouble.
+ *
+ * Dead vector components are dropped for the same reason, and it is the
+ * same kind of reason: an `insertelement undef, x, 0` arrives as
+ * `vec3 x, undef.y, undef.z`, and a vector with one undefined component in
+ * it is not a constant, so the fold stopped there and took everything
+ * downstream with it -- although nothing ever reads that component.
+ * nir_opt_shrink_vectors removes exactly the components nothing reads and
+ * rewrites the swizzles; the lanes that remain are computed by the same
+ * instruction over the same bits, so no arithmetic is touched. Four more of
+ * the library functions reduce to a value with it: the LLVM comparison of
+ * the exactly-defined ones went from 54 of 57 to 57 of 57 (and the three
+ * missing sides were ours, not LLVM's), and the softpipe comparison from
+ * 120 to 124 -- with the 255 entry points that already agreed with LLVM
+ * still agreeing, bit for bit, which is what says the pass changed no
+ * number anywhere.
*/
bool progress;
unsigned rounds = 0;
@@ -335,6 +433,9 @@ eval_module(const struct module_ref *mod)
NIR_PASS(progress, nir, nir_opt_constant_folding);
NIR_PASS(progress, nir, nir_opt_dce);
NIR_PASS(progress, nir, nir_lower_vars_to_ssa);
+ NIR_PASS(progress, nir, nir_opt_dead_cf);
+ NIR_PASS(progress, nir, nir_opt_if, nir_opt_if_optimize_phi_true_false);
+ NIR_PASS(progress, nir, nir_opt_shrink_vectors, false);
} while (progress && ++rounds < 16);
printf("eval: the outputs this shader stores:\n");
@@ -369,6 +470,87 @@ one_line(const char *s, FILE *fp)
fputc(*s == '\n' || *s == '\t' ? ' ' : *s, fp);
}
+/* --interface: what air_shader_info says the shader expects of the runtime,
+ * which nir_print_shader cannot show because it is not in the NIR. Its own
+ * mode rather than a line added to the others: the report's lines are parsed
+ * by tools/air/corpus.py and the translated NIR by tools/air/fbfetchcheck.py,
+ * and a field added to either would move numbers that are a baseline.
+ *
+ * One `key=value` per line, tab-separated, so a check can read it.
+ */
+static int
+interface_module(const struct module_ref *mod)
+{
+ void *ctx = ralloc_context(NULL);
+ const char *err = NULL;
+ struct air_constant_value *fc;
+ unsigned num_fc = air_module_function_constants(ctx, mod->bitcode, mod->size, &fc);
+ if (!apply_fc_set(fc, num_fc))
+ return false;
+ struct air_to_nir_options options = { .constants = fc, .num_constants = num_fc,
+ .library_wrapper = library_wrapper,
+ .stitch = stitch,
+ .stitch_functions = stitch_fns,
+ .num_stitch_functions = num_stitch_fns };
+ struct air_shader_info info;
+ nir_shader *nir = air_to_nir(ctx, mod->bitcode, mod->size, NULL, &options,
+ &default_options, &info, &err);
+ if (!nir) {
+ fprintf(stderr, "air2nir: %s: %s\n", mod->name, err);
+ ralloc_free(ctx);
+ return 1;
+ }
+
+ printf("entry=%s\tstage=%s\n", info.entry_point, gl_shader_stage_name(info.stage));
+ printf("inputs_read=0x%" PRIx64 "\n", (uint64_t)nir->info.inputs_read);
+ printf("vertex_attributes=%u\tvertex_attributes_read=0x%x\n",
+ info.num_vertex_attributes, info.vertex_attributes_read);
+ for (unsigned i = 0; i < info.num_vertex_attributes; i++) {
+ const struct air_vertex_attribute *a = &info.vertex_attributes[i];
+ printf("attribute\tindex=%u\tlocation=%s\tdriver_location=%d\tcomponents=%u"
+ "\tbits=%u\ttype=%s\tname=%s\tdecl=%s\tused=%s\tfc=%s\n",
+ a->index, gl_vert_attrib_name((gl_vert_attrib)a->location),
+ a->driver_location == ~0u ? -1 : (int)a->driver_location, a->components,
+ a->bit_size,
+ a->base_type == AIR_ATTRIB_FLOAT ? "float" :
+ a->base_type == AIR_ATTRIB_UINT ? "uint" : "int",
+ a->name ? a->name : "-", a->type_name ? a->type_name : "-",
+ a->used ? "yes" : "no",
+ a->function_constant ? a->function_constant : "-");
+ }
+ for (unsigned i = 0; i < info.num_varyings; i++)
+ printf("varying\tslot=%u\tcomponents=%u\tname=%s\n", info.varyings[i].slot,
+ info.varyings[i].components,
+ info.varyings[i].name ? info.varyings[i].name : "-");
+ for (unsigned i = 0; i < info.num_resources; i++) {
+ const struct air_resource *r = &info.resources[i];
+ /* set and id are the Metal name of an argument-buffer member, which is
+ * the one case where index is this frontend's number and not Metal's --
+ * so a check can see both halves of the mapping and not just the one it
+ * would have guessed (tools/air/argbufcheck.py)
+ *
+ * write and ir_write are two readings of one fact and are printed apart
+ * for the same reason: write is what Metal declares the shader may do to
+ * the buffer, ir_write is what the translated IR does to it, and a
+ * runtime that serves an unaligned `device` buffer by uploading a copy
+ * needs the second one with the first beside it as a cross-check.
+ */
+ printf("resource\tkind=%d\tindex=%u\tsize=%u\tname=%s\tset=%d\tid=%d"
+ "\tread=%d\twrite=%d\tir_write=%d\n",
+ r->kind, r->index, r->size, r->name ? r->name : "-",
+ r->argbuf_set == ~0u ? -1 : (int)r->argbuf_set,
+ r->argbuf_id == ~0u ? -1 : (int)r->argbuf_id, r->read, r->write,
+ r->ir_write);
+ }
+ for (unsigned i = 0; i < info.num_notes; i++) {
+ fputs("note\t", stdout);
+ one_line(info.notes[i], stdout);
+ fputc('\n', stdout);
+ }
+ ralloc_free(ctx);
+ return 0;
+}
+
/* tools/air/corpus.py reads these lines: tab-separated key=value */
static void
report_one(const struct module_ref *mod)
@@ -400,8 +582,13 @@ report_one(const struct module_ref *mod)
*/
struct air_constant_value *fc;
unsigned num_fc = air_module_function_constants(ctx, mod->bitcode, mod->size, &fc);
+ if (!apply_fc_set(fc, num_fc))
+ return;
struct air_to_nir_options options = { .constants = fc, .num_constants = num_fc,
- .library_wrapper = library_wrapper };
+ .library_wrapper = library_wrapper,
+ .stitch = stitch,
+ .stitch_functions = stitch_fns,
+ .num_stitch_functions = num_stitch_fns };
if (num_fc)
printf("\tfc=%u", num_fc);
@@ -423,7 +610,18 @@ report_one(const struct module_ref *mod)
instrs++;
}
}
- printf("\ttranslate=ok\tnir_instrs=%u\n", instrs);
+ printf("\ttranslate=ok\tnir_instrs=%u", instrs);
+ /* What was translated with a caveat: the shader was accepted and something
+ * other than exactly what it asked for was done (air_shader_info.notes).
+ * On the report line rather than on stderr, because the report is the only
+ * output anything parses, and a caveat nobody can count is the silence this
+ * channel exists to end.
+ */
+ for (unsigned i = 0; i < info.num_notes; i++) {
+ fputs(i ? ";" : "\tnotes=", stdout);
+ one_line(info.notes[i], stdout);
+ }
+ printf("\n");
ralloc_free(nir);
ralloc_free(ctx);
}
@@ -431,10 +629,35 @@ report_one(const struct module_ref *mod)
int
main(int argc, char **argv)
{
- enum { TRANSLATE, LL, LIST, REPORT, EVAL } mode = TRANSLATE;
+ enum { TRANSLATE, LL, LIST, REPORT, EVAL, INTERFACE } mode = TRANSLATE;
int i = 1;
- while (i < argc && !strcmp(argv[i], "--library"))
- library_wrapper = true, i++;
+ const char *stitch_text = NULL;
+ while (i < argc && (!strcmp(argv[i], "--library") ||
+ (!strcmp(argv[i], "--fc") && i + 1 < argc) ||
+ (!strcmp(argv[i], "--stitch") && i + 1 < argc))) {
+ if (!strcmp(argv[i], "--library")) {
+ library_wrapper = true, i++;
+ } else if (!strcmp(argv[i], "--fc")) {
+ unsigned idx = 0;
+ unsigned long long val = 0;
+ if (sscanf(argv[i + 1], "%u=0x%llx", &idx, &val) != 2 &&
+ sscanf(argv[i + 1], "%u=%llu", &idx, &val) != 2) {
+ fprintf(stderr, "air2nir: --fc wants INDEX=VALUE, decimal or 0x\n");
+ return 2;
+ }
+ if (num_fc_set >= ARRAY_SIZE(fc_set)) {
+ fprintf(stderr, "air2nir: more than %u function constants\n",
+ (unsigned)ARRAY_SIZE(fc_set));
+ return 2;
+ }
+ fc_set[num_fc_set].index = idx;
+ fc_set[num_fc_set].bits = val;
+ num_fc_set++;
+ i += 2;
+ } else {
+ stitch_text = argv[i + 1], i += 2;
+ }
+ }
if (i < argc && !strcmp(argv[i], "--ll"))
mode = LL, i++;
else if (i < argc && !strcmp(argv[i], "--list"))
@@ -443,10 +666,15 @@ main(int argc, char **argv)
mode = REPORT, i++;
else if (i < argc && !strcmp(argv[i], "--eval"))
mode = EVAL, i++;
+ else if (i < argc && !strcmp(argv[i], "--interface"))
+ mode = INTERFACE, i++;
if (i >= argc) {
fprintf(stderr,
- "usage: air2nir [--library] [--ll | --list | --report | --eval] "
- "FILE [FUNCTION]\n");
+ "usage: air2nir [--library] [--stitch GRAPH] "
+ "[--ll | --list | --report | --eval | --interface] "
+ "FILE [FUNCTION]\n"
+ " GRAPH is \"f(in0); g(n0)\": one node per `;', arguments inN, nN, "
+ "texN, sampN, bufN\n");
return 2;
}
@@ -454,6 +682,28 @@ main(int argc, char **argv)
void *ctx = ralloc_context(NULL);
int status = 0;
+ /* The module translated is the OUTPUT node's: a stitched graph is still
+ * handed to air_to_nir as one module plus a graph, and that module has to
+ * be one of the graph's own functions.
+ */
+ const char *only = NULL;
+ if (stitch_text) {
+ char why[256] = { 0 };
+ size_t size = 0;
+ const uint8_t *data = map_file(argv[i], &size);
+ stitch = air_stitch_parse(ctx, stitch_text, why, sizeof(why));
+ if (stitch && data)
+ num_stitch_fns = air_stitch_functions_of(ctx, stitch, data, size, &stitch_fns,
+ why, sizeof(why));
+ if (!stitch || !data || !num_stitch_fns) {
+ fprintf(stderr, "air2nir: --stitch: %s\n", why[0] ? why : "no functions");
+ ralloc_free(ctx);
+ glsl_type_singleton_decref();
+ return 2;
+ }
+ only = stitch->nodes[stitch->output_node].function;
+ }
+
if (mode == REPORT) {
setvbuf(stdout, NULL, _IOLBF, 0);
for (; i < argc; i++) {
@@ -482,7 +732,8 @@ main(int argc, char **argv)
}
struct module_ref *mods;
- unsigned n = modules_of(ctx, argv[i], i + 1 < argc ? argv[i + 1] : NULL, &mods);
+ unsigned n = modules_of(ctx, argv[i],
+ only ? only : i + 1 < argc ? argv[i + 1] : NULL, &mods);
if (n == 0)
return 1;
@@ -492,6 +743,10 @@ main(int argc, char **argv)
status |= eval_module(&mods[k]);
continue;
}
+ if (mode == INTERFACE) {
+ status |= interface_module(&mods[k]);
+ continue;
+ }
if (mode == LIST) {
printf("%-60s %-12s %zu\n", mods[k].name,
air_metallib_type_name(mods[k].container_type), mods[k].size);
@@ -509,8 +764,13 @@ main(int argc, char **argv)
}
struct air_constant_value *fc;
unsigned num_fc = air_module_function_constants(ctx, mods[k].bitcode, mods[k].size, &fc);
+ if (!apply_fc_set(fc, num_fc))
+ return 1;
struct air_to_nir_options options = { .constants = fc, .num_constants = num_fc,
- .library_wrapper = library_wrapper };
+ .library_wrapper = library_wrapper,
+ .stitch = stitch,
+ .stitch_functions = stitch_fns,
+ .num_stitch_functions = num_stitch_fns };
struct air_shader_info info;
nir_shader *nir = air_to_nir(ctx, mods[k].bitcode, mods[k].size, NULL, &options,
&default_options, &info, &err);
diff --git a/tools/mesa/air/air_intrinsics.c b/tools/mesa/air/air_intrinsics.c
index b9b5e31..11e26ce 100644
--- a/tools/mesa/air/air_intrinsics.c
+++ b/tools/mesa/air/air_intrinsics.c
@@ -28,16 +28,31 @@
/* the dot-separated parts of an air.* name, after "air." */
#define AIR_MAX_PARTS 8
+/* 48 and not 32: air.normalize_function_constant_predicate's first part is 37
+ * characters, and at 32 split_name refused the whole name rather than the
+ * operation -- "a name this cannot take apart", which reads like a malformed
+ * module and was a buffer. It is the longest part in Apple's own vocabulary
+ * list (section 3.2); the longest multi-part name there,
+ * air.next_intersection_query.instancing.triangle_data, has no part over 14.
+ */
struct air_name {
unsigned num;
- char part[AIR_MAX_PARTS][32];
+ char part[AIR_MAX_PARTS][48];
};
static bool
split_name(const char *name, struct air_name *out)
{
out->num = 0;
+ out->part[0][0] = 0;
const char *p = name + 4; /* past "air." */
+ /* "air." with nothing after it names no operation. Without this the loop
+ * below never runs, split_name returns true all the same, and the
+ * dispatcher compares a part[0] nothing wrote -- which gcc points out as
+ * soon as the first comparison is made unconditionally.
+ */
+ if (!*p)
+ return false;
while (*p && out->num < AIR_MAX_PARTS) {
const char *dot = strchr(p, '.');
size_t len = dot ? (size_t)(dot - p) : strlen(p);
@@ -59,6 +74,109 @@ is_signed_part(const char *s)
return !strcmp(s, "s");
}
+/* Metal's fmod: the remainder of x/y, with the SIGN OF x, correctly rounded.
+ *
+ * nir_frem is not that function. Its constant expression is, literally,
+ *
+ * src0 - src1 * truncf(src0 / src1) (nir_opcodes.py)
+ *
+ * and ir3 lowers it to the same three instructions (ir3_compiler.c sets
+ * .lower_fmod). Two things are wrong with the formula, and
+ * tools/air/intrincheck.py counted both: of 264 cases of air.fmod.f32 it
+ * reported 47 wrong, 25 of them a sign and 22 of them a number.
+ *
+ * - THE SIGN OF A ZERO IS LOST. fmod(-0, 5) is -0 and fmod(-1.5, 0.5) is
+ * -0, and the subtraction gives +0 for both: -0 - (0.5 * -0) is -0 - -0,
+ * which is +0. This is the shape of the air.sign bug recorded further
+ * down this file -- a sign of zero that a formula does not carry -- and
+ * it is the half of this that is not about precision at all.
+ * - THE PRODUCT IS ROUNDED BEFORE THE SUBTRACTION. y * trunc(x/y) needs up
+ * to 48 significant bits and a float has 24, so the subtraction is exact
+ * but of the wrong product: fmod(1.5, 0x1.fffffep-2) came out 1.19e-07
+ * against 8.94e-08, one whole ulp of the dividend, and fmod(1e8, 3) came
+ * out 0 where the answer is 1. A number, not a sign.
+ *
+ * Whose definition is followed here, and why it is Metal's: the Metal Shading
+ * Language Specification's table entry for fmod gives the same subtraction
+ * formula, but it inherits OpenCL's accuracy table along with it, where fmod
+ * is 0 ULP -- an exactly specified function, not a licensed approximation --
+ * and OpenCL's edge cases are C99 Annex F's, which say fmod(+-0, y) is +-0.
+ * Two references on this machine agree with that reading and none with the
+ * formula: this machine's libm answers -0 for fmodf(-0, 0.5) and for
+ * fmodf(-1.5, 0.5), and LLVM's own `frem` -- JIT-executed by lli, which is
+ * the second engine and not a reading of a document -- answers -0 for both
+ * and for frem(-2.5, -0.5). The corpus itself says nothing either way: not
+ * one of its 128 libraries contains a single `frem` instruction, because
+ * Apple's compiler always emits the call.
+ *
+ * So:
+ *
+ * q = trunc(x / y)
+ * r = fma(-y, q, x) one rounding, not two
+ * if (|r| >= |y|) r -= copysign(|y|, r) q was one too small
+ * if (sign r != sign x) r += copysign(|y|, x) q was one too large
+ * if (r == 0) r = copysign(0, x) a zero keeps x's sign
+ *
+ * The fma is the whole of the precision argument: when q is an exact integer
+ * the true remainder x - q*y is itself exactly representable (that is why
+ * IEEE 754 can require `remainder` to be exact), so ONE rounding of it is no
+ * rounding at all. The two corrections are there because fl(x/y) can round
+ * across an integer and move trunc by one either way, and because ir3 does
+ * not divide: it lowers fdiv to fmul(a, frcp(b)) and frcp is approximate, so
+ * a quotient off by one has to be survivable rather than assumed away.
+ *
+ * WHAT THIS STILL DOES NOT DO, measured and not guessed: q must be exactly
+ * representable, which means |x/y| < 2^24 for a float and 2^11 for a half.
+ * Above that no correction of a bounded size can help -- fmod(1e8, 1e-8)
+ * wants a quotient of 1e16 -- and the exact answer needs an iterated
+ * reduction: scale y up by 2^k so the partial quotient stays under 2^23,
+ * subtract, repeat. Each pass removes only 23 - 1 bits of the exponent
+ * difference, that difference reaches 253 for two normal floats, so twelve
+ * passes of about ten instructions each would be the price, and its exactness
+ * ON THE ADRENO could not be established from this machine at all for the
+ * frcp reason above. It is not done here; section 4 of docs/air.md carries
+ * the condition and the measurement.
+ *
+ * Over the reference pool this leaves 7 of 264 cases differing, every one of
+ * them with |x/y| >= 2^24; over 195 906 random pairs of normal floats every
+ * single disagreement with this machine's libm has |x/y| >= 2^24, and over
+ * 376 438 random pairs whose quotient IS representable there is none.
+ */
+static nir_def *
+air_fmod(nir_builder *b, nir_def *x, nir_def *y)
+{
+ uint64_t sign_bit = 1ull << (x->bit_size - 1);
+ /* The sign of a float is taken and planted with integer bit operations and
+ * never with a multiply by zero or a copysign built out of one: this
+ * builder runs with exact off (Metal's fast library), and
+ * nir_opt_algebraic's inexact rule (('~fmul', a, 0.0), 0.0) would throw
+ * away exactly the bit being carried. iand and ior cannot be rewritten by
+ * a float rule because they are not float operations.
+ */
+ nir_def *sx = nir_iand_imm(b, x, sign_bit);
+ nir_def *ay = nir_fabs(b, y);
+ nir_def *y_as_x = nir_ior(b, ay, sx); /* copysign(|y|, x) */
+
+ nir_def *q = nir_ftrunc(b, nir_fdiv(b, x, y));
+ nir_def *r = nir_ffma(b, nir_fneg(b, y), q, x);
+
+ /* |r| >= |y|: the quotient was one too small. A NaN fails this comparison
+ * (an ordered fge is false for one) and fails the feq against zero at the
+ * end; the middle correction may well fire for a NaN, and adding to a NaN
+ * gives a NaN, so a NaN comes out of all three unchanged.
+ */
+ nir_def *y_as_r = nir_ior(b, ay, nir_iand_imm(b, r, sign_bit));
+ r = nir_bcsel(b, nir_fge(b, nir_fabs(b, r), ay), nir_fsub(b, r, y_as_r), r);
+
+ /* r's sign is not x's: the quotient was one too large */
+ nir_def *wrong_sign = nir_ine(b, nir_iand_imm(b, r, sign_bit), sx);
+ r = nir_bcsel(b, nir_iand(b, wrong_sign, nir_fneu_imm(b, r, 0.0)),
+ nir_fadd(b, r, y_as_x), r);
+
+ /* and a zero result is the dividend's zero */
+ return nir_bcsel(b, nir_feq_imm(b, r, 0.0), sx, r);
+}
+
/* -- the families ------------------------------------------------------- */
static bool
@@ -130,6 +248,119 @@ math_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n,
r = nir_fsin(&c->b, a[0]);
} else if (!strcmp(op, "fast_cos") || !strcmp(op, "cos")) {
r = nir_fcos(&c->b, a[0]);
+ } else if (!strcmp(op, "fast_cospi") || !strcmp(op, "cospi") ||
+ !strcmp(op, "fast_sinpi") || !strcmp(op, "sinpi")) {
+ /* cos(pi*x) and sin(pi*x), which are their own operations in Apple's
+ * vocabulary because multiplying by pi is where the accuracy goes.
+ *
+ * APPLE'S OWN NAMES, and not the 137 of section 3.2: `air.cospi`,
+ * `air.sinpi`, `air.tanpi`, their three `air.fast_*` forms and
+ * `air.acospi`, `air.asinpi`, `air.atanpi`, `air.atan2pi` are C strings
+ * in the device's shared cache, in the AIR front end's own builtin table
+ * (dyld_shared_cache_arm64.17, `air.cospi` at file offset 95 195 301 --
+ * the same region carries AIRNTEmitDylibImage and Air.DylibScript). That
+ * table spells the whole name, prefix and all, which is why it holds 911
+ * single-segment entries where AGXCompilerCore's holds 137 suffixes.
+ * `tanpi` and the four inverse ones are deliberately NOT here: a
+ * declaration scan over every .metallib of the image finds only
+ * cospi.{f16,f32}, fast_cospi.f32 and sinpi.{f16,f32}, and tanpi's poles
+ * and zero signs are four more places to be silently wrong for no
+ * customer.
+ *
+ * STEP ONE, AND IT IS EXACT: r = x - 2*rint(x/2) lies in [-1, 1] and is
+ * exactly representable for EVERY x, which is why this is not fmul(x, pi)
+ * followed by a cosine. Above 2^(p+1) every float is an even integer, so
+ * r is zero; below it x and 2*rint(x/2) are within a factor of two of
+ * each other, where a subtraction is exact. An infinity reduces to
+ * inf - inf = NaN, which is what cospi(inf) is, so there is no edge case
+ * to add and none is added.
+ *
+ * STEP TWO IS A QUARTER AND NOT A HALF, and that is the part worth the
+ * instructions. Both functions are near a zero somewhere in [-1, 1], and
+ * a cosine evaluated near its own zero has no relative accuracy left:
+ * r*pi carries about 1e-7 of absolute error from pi's own rounding, which
+ * is nothing beside 1 and everything beside cospi(0.49999997). Measured,
+ * on the pool this suite already runs: the half-period form answers
+ * 7.55e-08 where the host libm's cospi says 9.36e-08 -- 2 551 410 f32 ulp,
+ * on the one input class the operation exists for. So |r| is folded into
+ * [0, 1/4], where neither function is near a zero, and which of sin and
+ * cos answers is chosen by the quarter it came from. Both are evaluated,
+ * because a bcsel is not a branch; the ARGUMENTS are selected first, so
+ * the cost is one sine and one cosine and not three.
+ *
+ * A HALF IS WIDENED TO f32 FIRST, measured and not assumed. pi rounded to
+ * f16 is 3.140625, a relative error of 3e-4, and near the zeros that is
+ * the whole answer: computed entirely at f16 the half-period form is
+ * bit-exact against the host libm's cospi on 50 064 of the 65 536 half
+ * inputs with 1 614 wrong by more than four half-ulp (sinpi 42 494) --
+ * which is a C model of that formula and not this code, since the
+ * un-widened lowering was never built.
+ * `tools/air/intrincheck.py --domain16` sweeps that whole domain -- the
+ * only width where "does it ever disagree" can be answered rather than
+ * sampled -- and this lowering is bit-exact on 65 536 of 65 536 for both.
+ *
+ * THE SIGN OF sinpi's ZERO is read off the argument's sign BIT and not
+ * compared, because -0 == 0. sinpi is zero at every integer with the sign
+ * of its argument, which the formula cannot produce on its own: for
+ * x = -2 the reduction gives r = +0 and for x = 3 it gives r = -1, so
+ * both the even and the odd integers need it. cospi needs no such case --
+ * at |r| = 1/2 the middle quarter evaluates sin(0), which is exactly
+ * zero.
+ */
+ bool is_sin = !strcmp(op, "sinpi") || !strcmp(op, "fast_sinpi");
+ nir_def *x = a[0];
+ bool narrow = a[0]->bit_size == 16;
+ if (narrow)
+ x = nir_f2f32(&c->b, x);
+ unsigned bits = x->bit_size;
+ nir_def *n = nir_fround_even(&c->b, nir_fmul_imm(&c->b, x, 0.5));
+ nir_def *red = nir_fsub(&c->b, x, nir_fmul_imm(&c->b, n, 2.0));
+ nir_def *u = nir_fabs(&c->b, red);
+ nir_def *one = nir_imm_floatN_t(&c->b, 1.0, bits);
+ /* 1/2 - |r| and 1 - |r| are both exact: Sterbenz, since |r| <= 1 */
+ nir_def *half_less = nir_fsub(&c->b, nir_imm_floatN_t(&c->b, 0.5, bits), u);
+ nir_def *one_less = nir_fsub(&c->b, one, u);
+ /* |r| <= 1/4 and |r| >= 3/4, spelled with the two comparisons the
+ * builder has: there is no nir_fle_imm, and `not (u > 1/4)` is not the
+ * same predicate as `u <= 1/4` for a NaN -- which is the right way round
+ * here, since a NaN must reach the cosine and come back a NaN rather than
+ * be routed by a comparison that answered false.
+ */
+ nir_def *lo = nir_inot(&c->b, nir_flt_imm(&c->b, nir_fneg(&c->b, u), -0.25));
+ nir_def *hi = nir_fge_imm(&c->b, u, 0.75);
+ if (is_sin) {
+ /* |r| in [0, 1/4] or (3/4, 1]: sin of the distance to the nearer
+ * integer; in between: cos of the distance to the nearer half
+ */
+ nir_def *mid = nir_inot(&c->b, nir_ior(&c->b, lo, hi));
+ nir_def *as = nir_bcsel(&c->b, lo, u, one_less);
+ nir_def *v =
+ nir_bcsel(&c->b, mid,
+ nir_fcos(&c->b, nir_fmul_imm(&c->b, half_less, M_PI)),
+ nir_fsin(&c->b, nir_fmul_imm(&c->b, as, M_PI)));
+ v = nir_bcsel(&c->b, nir_flt_imm(&c->b, red, 0.0), nir_fneg(&c->b, v), v);
+ r = nir_bcsel(&c->b,
+ nir_ior(&c->b, nir_feq_imm(&c->b, u, 0.0),
+ nir_feq_imm(&c->b, u, 1.0)),
+ nir_iand_imm(&c->b, x, 1ull << (bits - 1)), v);
+ } else {
+ /* |r| in [0, 1/4] or [3/4, 1]: cos of the distance to the nearer
+ * integer, negated past a half; in between: sin of the distance to
+ * the nearer half
+ */
+ nir_def *mid = nir_inot(&c->b, nir_ior(&c->b, lo, hi));
+ nir_def *ac = nir_bcsel(&c->b, lo, u, one_less);
+ nir_def *v =
+ nir_bcsel(&c->b, mid,
+ nir_fsin(&c->b, nir_fmul_imm(&c->b, half_less, M_PI)),
+ nir_fcos(&c->b, nir_fmul_imm(&c->b, ac, M_PI)));
+ /* `hi` alone and not `hi && !mid`: mid is the complement of
+ * `lo || hi`, so the two can never both hold
+ */
+ r = nir_bcsel(&c->b, hi, nir_fneg(&c->b, v), v);
+ }
+ if (narrow)
+ r = nir_f2f16(&c->b, r);
} else if (!strcmp(op, "fast_tan") || !strcmp(op, "tan")) {
r = nir_fdiv(&c->b, nir_fsin(&c->b, a[0]), nir_fcos(&c->b, a[0]));
} else if (!strcmp(op, "fast_atan2") || !strcmp(op, "atan2")) {
@@ -162,6 +393,106 @@ math_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n,
nir_def *at_origin =
nir_bcsel(&c->b, nir_ilt_imm(&c->b, y, 0), nir_fneg(&c->b, mag), mag);
r = nir_bcsel(&c->b, origin, at_origin, nir_atan2(&c->b, y, x));
+ } else if (!strcmp(op, "fast_asin") || !strcmp(op, "asin") ||
+ !strcmp(op, "fast_acos") || !strcmp(op, "acos")) {
+ /* The inverse sine and cosine, reduced to atan2, which this file
+ * already has and which Mesa implements:
+ *
+ * asin(x) = atan2(x, sqrt(1 - x*x))
+ * acos(x) = atan2(sqrt(1 - x*x), x)
+ *
+ * The reduction is exact rather than an approximation of its own, which
+ * is why it is preferred to the polynomial in Mesa's SPIR-V front end
+ * (build_asin in vtn_glsl450.c): that one's own comment says it "isn't
+ * precise enough to meet half-float precision requirements" and names
+ * this formula as the alternative it rejected for cost. Cost is the
+ * right thing to spend here -- air.fast_acos is called by 183 of
+ * Resident Evil 4's modules and the shape of a terminator's lighting
+ * depends on it.
+ *
+ * (1 - x)*(1 + x) and not 1 - x*x: the two are the same number in exact
+ * arithmetic and not in a float. Near |x| = 1 the subtraction of two
+ * close values loses most of the significand -- at x = 1 - 2^-20 the
+ * product form keeps the result to a few ulp where 1 - x*x has none of
+ * the low bits left -- and |x| near 1 is exactly where a normal dotted
+ * with a light direction spends its time.
+ *
+ * The edges fall out of atan2 rather than needing a case: the second
+ * argument is a square root and so never negative, both forms reach
+ * atan2(0, +-1) = {0, pi} at x = +-1, |x| > 1 takes the root of a
+ * negative and gives NaN as the domain demands, and neither form can
+ * reach atan2(0, 0) -- x and 1 - x*x are not both zero -- so the origin
+ * this file corrects for plain atan2 is unreachable from here.
+ */
+ nir_def *x = a[0];
+ nir_def *root = nir_fsqrt(&c->b,
+ nir_fmul(&c->b, nir_fsub(&c->b, nir_imm_floatN_t(&c->b, 1.0, x->bit_size), x),
+ nir_fadd_imm(&c->b, x, 1.0)));
+ bool is_asin = !strcmp(op, "fast_asin") || !strcmp(op, "asin");
+ r = is_asin ? nir_atan2(&c->b, x, root) : nir_atan2(&c->b, root, x);
+ } else if (!strcmp(op, "fast_atan") || !strcmp(op, "atan")) {
+ /* one argument: the arc tangent of a ratio already taken, which is
+ * Metal's atan(y_over_x) and GLSL's atan(y_over_x) both
+ */
+ r = nir_atan(&c->b, a[0]);
+ } else if (!strcmp(op, "fast_sinh") || !strcmp(op, "sinh")) {
+ /* 0.5 * (e^x - e^-x), which is Mesa's own lowering of GLSL sinh
+ * (vtn_glsl450.c), with the small-argument case taken out of it.
+ *
+ * A DIFFERENCE OF TWO EXPONENTIALS HAS NO LOW BITS LEFT NEAR ZERO: at
+ * x = 1e-8 both e^x and e^-x round to 1.0 in a float, so the formula
+ * answers 0 where sinh(1e-8) is 1e-8 -- not an imprecise answer but a
+ * wrong one, and the first thing tools/air/intrincheck.py said about
+ * this lowering. sinh(x) = x + x^3/6, so below the x where x^3/6 falls
+ * under the type's own precision the argument IS the answer, exactly;
+ * 2^-12 is that point for a float (x^2/6 < 2^-24) and 2^-5 for a half.
+ * Above it the formula keeps a few thousand ulp of relative error near
+ * the threshold, which is what a difference of exponentials costs and
+ * what the fast_ prefix licenses.
+ */
+ unsigned bits = a[0]->bit_size;
+ nir_def *big = nir_fge_imm(&c->b, nir_fabs(&c->b, a[0]),
+ bits > 16 ? 0x1p-12 : 0x1p-5);
+ r = nir_bcsel(&c->b, big,
+ nir_fmul_imm(&c->b,
+ nir_fsub(&c->b, nir_fexp(&c->b, a[0]),
+ nir_fexp(&c->b, nir_fneg(&c->b, a[0]))), 0.5),
+ a[0]);
+ } else if (!strcmp(op, "fast_cosh") || !strcmp(op, "cosh")) {
+ r = nir_fmul_imm(&c->b, nir_fadd(&c->b, nir_fexp(&c->b, a[0]),
+ nir_fexp(&c->b, nir_fneg(&c->b, a[0]))), 0.5);
+ } else if (!strcmp(op, "fast_tanh") || !strcmp(op, "tanh")) {
+ /* (e^x - e^-x) / (e^x + e^-x), with the two guards Mesa's own lowering
+ * of GLSL tanh carries and for the same reasons (vtn_glsl450.c):
+ *
+ * - the argument is clamped to +-10, above which e^x overflows while
+ * the answer has been 1.0 to the last bit of a float since |x| = 9;
+ * - and the clamp would turn a NaN into 10 and so a NaN into 1, so the
+ * result is taken only where the argument is large enough and handed
+ * back otherwise. That also keeps tanh(-0) = -0, which the division
+ * would give as +0... and this one is EXACTLY the shape of the
+ * air.sign bug recorded further up this file: a sign of zero that
+ * NIR's folder keeps and a backend need not.
+ *
+ * The threshold is 2^-12 (2^-5 for a half) and not zero, which is where
+ * this differs from Mesa's: tanh(x) = x - x^3/3, so below that point
+ * the argument is the answer to the last bit, and above it the
+ * difference of exponentials loses the low bits the same way sinh's
+ * does. With Mesa's threshold of zero, tanh(1e-8) came back 0.
+ */
+ unsigned bits = a[0]->bit_size;
+ nir_def *x = nir_fmin(&c->b, nir_fmax(&c->b, a[0],
+ nir_imm_floatN_t(&c->b, bits > 16 ? -10.0 : -4.2, bits)),
+ nir_imm_floatN_t(&c->b, bits > 16 ? 10.0 : 4.2, bits));
+ nir_def *ex = nir_fexp(&c->b, x), *enx = nir_fexp(&c->b, nir_fneg(&c->b, x));
+ bool was = c->b.exact;
+ c->b.exact = true;
+ nir_def *regular = nir_fge_imm(&c->b, nir_fabs(&c->b, a[0]),
+ bits > 16 ? 0x1p-12 : 0x1p-5);
+ c->b.exact = was;
+ r = nir_bcsel(&c->b, regular,
+ nir_fdiv(&c->b, nir_fsub(&c->b, ex, enx), nir_fadd(&c->b, ex, enx)),
+ a[0]);
} else if (!strcmp(op, "fast_exp") || !strcmp(op, "exp")) {
/* the natural one: Apple's compiler names exp and exp2 separately */
r = nir_fexp2(&c->b, nir_fmul_imm(&c->b, a[0], M_LOG2E));
@@ -181,7 +512,17 @@ math_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n,
r = nir_fpow(&c->b, a[0], a[1]);
} else if (!strcmp(op, "fast_fmod") || !strcmp(op, "fmod") ||
!strcmp(op, "fast_mod") || !strcmp(op, "mod")) {
- r = nir_frem(&c->b, a[0], a[1]);
+ /* One lowering for the fast form as well as the plain one. Metal's
+ * fast-math does license a lower-precision fmod and does license
+ * ignoring the sign of a zero -- 131 of the corpus's 139 fmod
+ * declarations are air.fast_fmod.f32, so the cheap form would be legal
+ * for nearly all of them -- but a reference that is right for one
+ * spelling and relaxed for another is a reference nobody can read, and
+ * the extra instructions are about a dozen. If the cost ever matters,
+ * the one-step form is nir_frem plus the zero's sign, and the condition
+ * under which the two differ is written in air_fmod.
+ */
+ r = air_fmod(&c->b, a[0], a[1]);
} else if (!strcmp(op, "fast_divide") || !strcmp(op, "divide")) {
r = nir_fdiv(&c->b, a[0], a[1]);
} else if (!strcmp(op, "saturate") || !strcmp(op, "fast_saturate")) {
@@ -219,6 +560,34 @@ math_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n,
} else if (!strcmp(op, "fast_fmax") || !strcmp(op, "fmax") || !strcmp(op, "max")) {
r = is_float ? nir_fmax(&c->b, a[0], a[1])
: (is_signed ? nir_imax(&c->b, a[0], a[1]) : nir_umax(&c->b, a[0], a[1]));
+ } else if (!strcmp(op, "fast_fmin3") || !strcmp(op, "fmin3") ||
+ !strcmp(op, "fast_fmax3") || !strcmp(op, "fmax3")) {
+ /* The least or the greatest of THREE, which Metal spells min3/max3 and
+ * the Adreno has an instruction for. Built out of the two-argument form
+ * above rather than given one of its own, which is exact and not merely
+ * convenient: min and max are associative over a total order, and the
+ * order the project's reference uses puts -0.0 below +0.0 and has no NaN
+ * to order (every module here is compiled nnan), so min(min(a,b),c) is
+ * the same value whichever way it is grouped. `clamp` just above is the
+ * same composition of the same two operations.
+ *
+ * 11 of RenderBox's 47 modules call air.fmax3.f16 -- it is how a shape
+ * function takes the widest of three coverage estimates -- and
+ * air.fmin3.f16 beside it. tools/air/intrincheck.py now carries the
+ * reference for both, so each is folded and drawn against it like every
+ * other operation here, and tools/air/airdefs.py emits an LLVM body so
+ * that a whole module calling one can still be compared on the CPU.
+ */
+ bool lo = strstr(op, "min") != NULL; /* fmin3 or fmax3 */
+ if (is_float)
+ r = lo ? nir_fmin(&c->b, nir_fmin(&c->b, a[0], a[1]), a[2])
+ : nir_fmax(&c->b, nir_fmax(&c->b, a[0], a[1]), a[2]);
+ else if (is_signed)
+ r = lo ? nir_imin(&c->b, nir_imin(&c->b, a[0], a[1]), a[2])
+ : nir_imax(&c->b, nir_imax(&c->b, a[0], a[1]), a[2]);
+ else
+ r = lo ? nir_umin(&c->b, nir_umin(&c->b, a[0], a[1]), a[2])
+ : nir_umax(&c->b, nir_umax(&c->b, a[0], a[1]), a[2]);
} else if (!strcmp(op, "fast_clamp") || !strcmp(op, "clamp")) {
if (is_float)
r = nir_fmin(&c->b, nir_fmax(&c->b, a[0], a[1]), a[2]);
@@ -276,6 +645,163 @@ math_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n,
r = nir_ball(&c->b, a[0]);
} else if (!strcmp(op, "any")) {
r = nir_bany(&c->b, a[0]);
+ } else if (!strcmp(op, "simd_sum") || !strcmp(op, "simd_min") ||
+ !strcmp(op, "simd_max") || !strcmp(op, "simd_prefix_exclusive_sum")) {
+ /* A REDUCTION OVER THE SIMD GROUP, given to every lane. NIR's reduce with
+ * cluster_size 0 is that, and ir3 has it (nir_intrinsic_reduce in
+ * ir3_compiler_nir.c, emit_intrinsic_reduce) and splits a vector into
+ * scalars on the way (nir_lower_subgroups with lower_to_scalar). The
+ * prefix scan is nir_exclusive_scan, which the same ir3 function emits
+ * from the same `scan` instruction's third destination.
+ *
+ * FOUR OF APPLE'S SEVENTEEN, and the four are the ones with a customer.
+ * This used to read "only the sum, because only the sum occurs", which
+ * was true of the corpus and never checked against the image: of the
+ * image's 10 242 modules, 9 call air.simd_sum, 4 air.simd_max, 3
+ * air.simd_min and 2 air.simd_prefix_exclusive_sum, and the last three
+ * names were the FIRST wall of 4 of them (Portrait.framework and
+ * MPSImage). The thirteen still missing -- product, and, or, xor, ballot,
+ * the inclusive scans, the prefix products -- are called by nothing in
+ * either, so each would still be code with no case behind it.
+ *
+ * What no amount of this can be checked against: a float reduction has no
+ * defined order in Metal or in NIR, so a sum over the group is not a
+ * number any reference can predict bit-exactly, on the phone or
+ * anywhere. Min and max are better off -- they are order-independent for
+ * every input that is not a NaN -- and the prefix sum is worse, because
+ * it is an order written into the operation's name over a group whose
+ * width is the hardware's.
+ */
+ nir_op rop = is_float ? nir_op_fadd : nir_op_iadd;
+ if (!strcmp(op, "simd_min"))
+ rop = is_float ? nir_op_fmin : is_signed ? nir_op_imin : nir_op_umin;
+ else if (!strcmp(op, "simd_max"))
+ rop = is_float ? nir_op_fmax : is_signed ? nir_op_imax : nir_op_umax;
+ air_note(c, "air.%s is taken over the SIMD group this implementation has, which "
+ "is not Metal's 32 lanes: ir3 runs 64 or 128 of them on this "
+ "hardware, so the result is a reduction over more lanes than the "
+ "shader was compiled for", op);
+ r = !strcmp(op, "simd_prefix_exclusive_sum")
+ ? nir_exclusive_scan(&c->b, a[0], .reduction_op = rop)
+ : nir_reduce(&c->b, a[0], .reduction_op = rop);
+ } else if (!strcmp(op, "normalize_function_constant_predicate")) {
+ /* Metal's compiler stores the predicates that say whether a dependent
+ * function constant applies in `__metal_implicit_fc_pred_N` bytes, and
+ * puts every one through this before storing it. The name is in Apple's
+ * own vocabulary (it sits between air.nextafter and air.pack.snorm1x8 in
+ * AGXCompilerCore's list of operation names, section 3.2), but no string
+ * or symbol in that binary says what it computes, so the reading here is
+ * C's: a byte is a predicate if it is not zero.
+ *
+ * THE CORPUS CANNOT TELL THAT APART FROM THE IDENTITY, and saying so is
+ * the point: every argument it is given in these modules is a
+ * `zext i1 ... to i8`, already 0 or 1, and any function that agrees with
+ * the identity on {0, 1} gives the same answer. A byte of 2 would
+ * discriminate, and nothing produces one here -- so this is a reading
+ * that no measurement available on this machine can confirm or refute,
+ * and a shader that fed it a wider byte would be the first test of it.
+ */
+ r = nir_b2iN(&c->b, nir_ine_imm(&c->b, a[0], 0),
+ air_nir_bit_size(c, in->base.type));
+ } else if (!strcmp(op, "unpack") && n->num >= 2 &&
+ !strcmp(n->part[1], "unorm4x8")) {
+ /* four unsigned bytes of a word, each divided by 255. Component 0 is the
+ * LOW byte, which is both GLSL's unpackUnorm4x8 and Metal's
+ * unpack_unorm4x8_to_float, so NIR's opcode is the operation and not an
+ * approximation of it. ir3 has no instruction for it and does not need
+ * one: it sets lower_unpack_unorm_4x8, and nir_opt_algebraic turns it
+ * into four extract_u8 and a divide.
+ */
+ r = nir_unpack_unorm_4x8(&c->b, a[0]);
+ if (air_nir_bit_size(c, in->base.type) == 16)
+ r = nir_f2f16(&c->b, r);
+ } else if (!strcmp(op, "pack") && n->num >= 3 &&
+ !strcmp(n->part[1], "unorm") && !strcmp(n->part[2], "rgb565")) {
+ /* COMPONENT 0 IS AT BIT 0, and that is Apple's own answer rather than a
+ * reading of the name. The module and its metadata fix the widths and
+ * that the 6-bit field is in the middle; which end component 0 is at is
+ * in neither, and the two readings exchange red and blue. A draw through
+ * air_run cannot separate them either: the pack and the number read back
+ * would both be ours. (docs/air.md 5e has what a draw on the DEVICE
+ * could and could not say, which is not what the refusal here used to
+ * claim.) What settles it is the lowering in AGXCompilerCore -- this
+ * device's own, ios165/20F66__iPhone10,3_6 -- read as instructions:
+ *
+ * AGCLLVMAirBuiltins::initializePackUnpackMap 0x1bbe5c0cc
+ * maps the name suffix "unorm.rgb565.v3f" to kind 2
+ * AGCLLVMAirBuiltins::buildPack 0x1bbe5be08
+ * kind 2 -> buildPackFormat(PackedType 0, normalized, unsigned)
+ * AGCLLVMTargetLowerer::buildPackFormat 0x1bbed0934
+ * PackedType 0..3 -> buildPackFloatFormat, which keeps the type
+ * AGCLLVMTargetLowerer::buildPackFloatFormat 0x1bbed3204
+ * indexes a 9-byte row of `formats` (0x1bbf1977f) by PackedType:
+ * one count byte, then four WIDTHS, then four SHIFTS. Row 0 is
+ * count 3, widths {5, 6, 5}, shifts {0, 5, 11}, and the loop is
+ * `acc |= packFromFloat(extractelement(v, i), width[i], shift[i])`
+ * from a constant 0, then truncateToSmall to the declared i16.
+ * AGCLLVMTargetLowerer::buildPackFromFloat 0x1bbed2e38
+ * fpext to f32; buildFClamp(0.0, 1.0), which for exactly those two
+ * bounds dispatches to buildSaturate (0x1bbe0e428); fmul by
+ * (1 << width) - 1; llvm.agx{1,2}.pack.rne.s32, whose prefix is the
+ * GPU generation's (Gen2's constructor stores agx1 and Gen3's agx2,
+ * both round to nearest even); shl by `shift`.
+ *
+ * THE SHIFTS ARE THE RUNNING SUM OF THE WIDTHS in all seven rows of that
+ * table -- 565 {0,5,11}, 4444 {0,4,8,12}, 1555 {0,1,6,11},
+ * 5551 {0,5,10,15}, rgb10 {0,10,20}, rgb10a2 {0,10,20,30} and a second
+ * copy of that last layout, rows 5 and 6 being the float and the uint
+ * variants of 1010102 -- so the fields are laid out from bit 0
+ * upward in component order and the 565 row cannot be read the other way
+ * round without breaking the six beside it. Each row's widths also sum
+ * to its word: 16 for the four 16-bit ones, 30 and 32 for the others.
+ *
+ * The inverse lowering is the second witness, and it reads the same row
+ * through the opposite argument order: buildUnpackFloatFormat
+ * (0x1bbe001ec) hands buildUnpackToFloat (0x1bbed2c10) the SHIFT first
+ * and the width second, where the pack side passes them the other way
+ * round, and that callee begins with `lshr` by its first argument and
+ * masks with (1 << width) - 1. A misreading of which byte-run of the row
+ * is widths and which is shifts would have to survive both call sites.
+ *
+ * So, for i in 0..2 with width {5,6,5} and shift {0,5,11}:
+ * acc |= rne(saturate(f32(v[i])) * ((1 << width[i]) - 1)) << shift[i]
+ *
+ * In f32 and not in f16, because Apple extends first, and that is
+ * measured and not asserted: 61 half values round differently if the
+ * product is computed at f16, two of them are in intrincheck.py's pool
+ * for this operation, and removing the extension here duly makes 2 of
+ * its 24 cases differ. nir_fsat IS saturate and nir_fround_even IS
+ * Apple's rne, so neither step here is an approximation of the one it
+ * stands for.
+ *
+ * ROUND-TO-NEAREST-EVEN IS NOT OBSERVABLE at this width, which is worth
+ * knowing rather than relying on: over all 65 536 half bit patterns the
+ * only exactly half-way products are 0.5*31 = 15.5 and 0.5*63 = 31.5,
+ * and nearest-even and half-away-from-zero both answer 16 and 32 there.
+ * So no input this operation can be given separates Apple's rounding
+ * mode from the other one, and it is spelled rne because that is what
+ * Apple's code names. Truncation does differ, on 7 551 of them, and that
+ * is the mistake the check can catch (tools/air/intrincheck.py, and
+ * tools/air/pack565.ll for four colours readable by eye).
+ */
+ static const unsigned pack565_width[3] = { 5, 6, 5 };
+ static const unsigned pack565_shift[3] = { 0, 5, 11 };
+ /* exact: a pack is a bit pattern, not a shader's arithmetic, and the
+ * fast-math this function turns on for everything else has no business
+ * reassociating the one multiply it contains
+ */
+ c->b.exact = true;
+ nir_def *acc = nir_imm_int(&c->b, 0);
+ for (unsigned k = 0; k < 3; k++) {
+ nir_def *x = nir_channel(&c->b, a[0], k);
+ if (x->bit_size != 32)
+ x = nir_f2f32(&c->b, x);
+ x = nir_fmul_imm(&c->b, nir_fsat(&c->b, x),
+ (double)((1u << pack565_width[k]) - 1));
+ nir_def *f = nir_f2u32(&c->b, nir_fround_even(&c->b, x));
+ acc = nir_ior(&c->b, acc, nir_ishl_imm(&c->b, f, pack565_shift[k]));
+ }
+ r = nir_u2uN(&c->b, acc, air_nir_bit_size(c, in->base.type));
} else if (!strcmp(op, "convert")) {
/* air.convert.<dst sign>.<dst type>.<src sign>.<src type> */
if (n->num < 5) {
@@ -291,7 +817,19 @@ math_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n,
nir_def *x = a[0];
if (!src_float && air_type_bit_size(air_type_scalar(st)) == 1)
x = src_signed ? nir_ineg(&c->b, nir_b2i32(&c->b, x)) : nir_b2i32(&c->b, x);
- if (dst_float && src_float)
+ /* A BOOL DESTINATION IS NOT A CONVERSION. C's rule, which MSL inherits,
+ * is `x != 0`, and in NIR a boolean is a 1-bit def -- so asking
+ * nir_f2uN for one bit produces f2u1, an opcode ir3 has no instruction
+ * for and no lowering of. That is the whole of why
+ * variable_blur_frag_{lpf,lph} were the only two modules of QuartzCore's
+ * library to reach the backend and die there: "Unhandled ALU op: f2u1",
+ * from air.convert.u.i1.f.f32. The comparison is also the right answer
+ * rather than a way round the backend: a conversion would have a
+ * rounding mode, and bool(0.5) is true, not false.
+ */
+ if (bits == 1)
+ r = src_float ? nir_fneu_imm(&c->b, x, 0.0) : nir_ine_imm(&c->b, x, 0);
+ else if (dst_float && src_float)
r = nir_f2fN(&c->b, x, bits);
else if (dst_float)
r = src_signed ? nir_i2fN(&c->b, x, bits) : nir_u2fN(&c->b, x, bits);
@@ -312,6 +850,53 @@ math_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n,
return true;
}
+/* -- function constants ------------------------------------------------- */
+
+/* air.is_function_constant_defined(ptr to the constant's global)
+ *
+ * Metal's companion to `[[function_constant(n)]]`: a shader may ask whether
+ * the pipeline supplied a value for a constant, and guard the code that reads
+ * it. The answer is known at translation time -- it is exactly whether the
+ * caller passed a value for that index in air_to_nir_options -- so this folds
+ * to a constant and the guarded branch folds away with it.
+ *
+ * It is handled here, before math_op, because its argument is a POINTER and
+ * math_op resolves every argument as a number: the four fixed_vert_* modules
+ * stopped at "a pointer where a number belongs" with no instruction named,
+ * because the failure was in reading the arguments and not in the operation.
+ */
+static bool
+fc_defined_op(struct air_ctx *c, const struct air_inst *in)
+{
+ if (in->num_operands != 2)
+ return air_error(c, "intrinsic: air.is_function_constant_defined of %u arguments",
+ in->num_operands - 1);
+ struct air_val *p = air_get(c, in->operands[1]);
+ if (!p)
+ return false;
+ if (p->kind != AIR_VAL_PTR || p->ptr.class != AIR_PTR_FUNCTION_CONSTANT)
+ return air_error(c, "intrinsic: air.is_function_constant_defined of something "
+ "that is not a function constant");
+ /*
+ * Defined means SUPPLIED BY THE PIPELINE, not "declared by the module".
+ *
+ * This loop used to answer true whenever the index was in the array, and the array is
+ * every constant the module declares -- so the answer was always true, and the value
+ * handed over with it was the zero nobody had filled in. The question these 80
+ * constants of the corpus ask is exactly the one that was being answered wrongly:
+ * each is guarded by this intrinsic and has a branch for not having been supplied.
+ */
+ bool defined = false;
+ for (unsigned i = 0; c->options && i < c->options->num_constants; i++) {
+ if (c->options->constants[i].index == p->ptr.binding) {
+ defined = c->options->constants[i].supplied;
+ break;
+ }
+ }
+ air_set(c, &in->base, air_val_ssa(c, nir_imm_bool(&c->b, defined)));
+ return true;
+}
+
/* -- atomics ------------------------------------------------------------ */
static bool
@@ -340,6 +925,38 @@ atomic_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n
nir_def *off = p.offset ? p.offset : nir_imm_int(&c->b, 0);
+ /* Compare-exchange, refused before the memory order is read -- because its
+ * arguments are in different places and reading them as the others' gave a
+ * sentence about the wrong thing. Its signature, decoded from the call
+ * sites so that implementing it needs no second reading:
+ *
+ * i32 air.atomic.<global|local>.cmpxchg.weak.i32(
+ * ptr object, ptr expected, i32 desired,
+ * i32 success_order, i32 failure_order, i32 scope, i1 volatile)
+ *
+ * so the order this used to read as argument 3 is the DESIRED VALUE, which
+ * is not a constant, and 4 modules of Resident Evil 4 were told "with a
+ * memory order that is not a constant" about an operation this does not
+ * implement at all.
+ *
+ * THE RESULT IS THE OBJECT'S OLD VALUE and not a boolean, which is read off
+ * what the callers do with it rather than from the name: Metal's
+ * atomic_compare_exchange_weak_explicit returns bool, and every call site
+ * here instead feeds the result to `icmp ne i32 %expected, %result` and
+ * then re-loads *expected -- the weak-CAS retry loop HLSL's
+ * InterlockedCompareExchange compiles to. A lowering is three operations
+ * (load *expected, nir_atomic_op_cmpxchg, store the old value back through
+ * `expected`, whose conditional-on-failure store may be unconditional
+ * because on success it writes the value already there), and what stops it
+ * is not the operation but the check: nothing in tools/air/atomiccheck.py
+ * can make a wrong compare-exchange show, and an atomic that is only
+ * compiled is the one kind this project will not claim.
+ */
+ if (!strcmp(op, "cmpxchg"))
+ return air_error(c, "intrinsic: air.atomic.%s.cmpxchg -- compare-exchange, whose "
+ "result is the old value and whose `expected` is a pointer "
+ "this does not write back yet", where);
+
/* The memory order, which this lowering is right for exactly one value of.
* That argument is Metal's memory_order, and every atomic in the corpus
* passes 0 -- memory_order_relaxed. A relaxed load or store of an aligned
@@ -403,11 +1020,40 @@ atomic_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n
/* -- barriers ----------------------------------------------------------- */
static bool
-barrier_op(struct air_ctx *c, const struct air_inst *in)
+barrier_op(struct air_ctx *c, const struct air_inst *in, bool simdgroup)
{
- /* air.wg.barrier(i32 flags, i32 scope): the flags say which memory is
- * ordered. 2 is threadgroup memory, 1 device, 4 texture; the corpus only
- * ever passes 2 with scope 1 (the threadgroup).
+ /* air.wg.barrier(i32 flags, i32 scope) and air.simdgroup.barrier(i32 flags,
+ * i32 scope). BOTH ARGUMENTS ARE NOW APPLE'S OWN WORDS rather than a
+ * census: the Metal headers shipped with this device's own shader compiler
+ * (GPUCompiler.framework/.../include/metal/metal_compute) define the two
+ * library functions as
+ *
+ * threadgroup_barrier(mem_flags f) { __metal_wg_barrier(int(f),
+ * __METAL_MEMORY_SCOPE_THREADGROUP__); }
+ * simdgroup_barrier(mem_flags f) { __metal_simdgroup_barrier(int(f),
+ * __METAL_MEMORY_SCOPE_THREADGROUP__); }
+ *
+ * so the second argument is the MEMORY scope and is the threadgroup in
+ * both -- the 1 the corpus passes, now sourced rather than observed -- and
+ * the only difference between the two operations is the EXECUTION scope:
+ * the threadgroup for one, the SIMD-group for the other. That is the whole
+ * of what `simdgroup` selects below.
+ *
+ * The flags are `enum class mem_flags` of the same header: mem_none,
+ * mem_device, mem_threadgroup, mem_threadgroup_imageblock, mem_texture,
+ * mem_object_data. The numbers are compiler built-ins and not in the
+ * header, so 1 = device, 2 = threadgroup, 4 = texture, 8 = imageblock stays
+ * a reading of the calls -- but `mem_threadgroup_imageblock` NAMES flag 8's
+ * memory as threadgroup memory, which is the thing the fallback below used
+ * to be guessing.
+ *
+ * The flag census of the corpus, kept because it is what a reader can
+ * check: 46 of the 852 modules call air.wg.barrier at all, and they pass 1
+ * in 3 of them (6 calls), 2 in 25 (43 calls) and 8 in 15 (15 calls). The
+ * fifteen are exactly the fifteen CMImaging modules with an explicit
+ * imageblock, which stop at the interface (air_to_nir.c) and so never
+ * arrive here. No module of the corpus calls air.simdgroup.barrier; 509 of
+ * the image's 10 242 do, and 106 of those stop here (docs/air.md 10).
*/
uint64_t flags = 2;
if (in->num_operands > 1)
@@ -420,12 +1066,568 @@ barrier_op(struct air_ctx *c, const struct air_inst *in)
modes |= nir_var_mem_shared;
if (flags & 4)
modes |= nir_var_image;
- if (!modes)
- modes = nir_var_mem_shared;
+ /* Flag 8 is mem_threadgroup_imageblock, and the header's name for it is why
+ * it gets threadgroup memory rather than the old catch-all fallback: an
+ * explicit imageblock is per-threadgroup storage and Apple calls it that.
+ * It still cannot reach this function -- an imageblock input is refused at
+ * the interface -- so this is a mode for a case that does not arrive, which
+ * is better than a default that answers for it silently.
+ */
+ if (flags & 8)
+ modes |= nir_var_mem_shared;
- nir_barrier(&c->b, .execution_scope = SCOPE_WORKGROUP,
- .memory_scope = SCOPE_WORKGROUP, .memory_semantics = NIR_MEMORY_ACQ_REL,
+ /* mem_none == 0 ORDERS NO MEMORY, and that is the one place the old code
+ * said something Apple's header contradicts. It read `if (!modes) modes =
+ * nir_var_mem_shared;` and called that right-for-the-wrong-reason about
+ * flag 8; but 0 is `mem_none`, a value the enumeration defines, and six
+ * calls in the image pass it (five in MLCompute, one in
+ * CameraColorProcessing, all of them to simdgroup.barrier). For those the
+ * fallback invented a threadgroup-memory fence the shader did not ask for.
+ * It was conservative and therefore not a wrong answer, but it was an
+ * answer to a question nobody asked, and a barrier that orders nothing is
+ * exactly what an execution-only barrier is.
+ */
+ nir_barrier(&c->b,
+ .execution_scope = simdgroup ? SCOPE_SUBGROUP : SCOPE_WORKGROUP,
+ .memory_scope = modes ? SCOPE_WORKGROUP : SCOPE_NONE,
+ .memory_semantics = modes ? NIR_MEMORY_ACQ_REL : 0,
.memory_modes = modes);
+
+ /* The SIMD-group's width is the hardware's and not Metal's 32 (simd_op
+ * below has the whole argument). For a barrier that costs less than it does
+ * for a shuffle -- a barrier over MORE lanes than the shader expects still
+ * orders the ones it expects -- but a shader that uses a simdgroup barrier
+ * INSTEAD of a threadgroup one because it knows its threadgroup is one
+ * 32-lane SIMD-group is relying on the width, and on this hardware the
+ * group is 64 or 128, so a threadgroup of 64 that Apple's compiler would
+ * have split into two SIMD-groups is one here. That changes which lanes the
+ * barrier covers, in the safe direction.
+ */
+ if (simdgroup)
+ air_note(c, "air.simdgroup.barrier is taken over the SIMD group this "
+ "implementation has, which is not Metal's 32 lanes: ir3 runs 64 or "
+ "128 of them on this hardware, so the barrier covers at least the "
+ "lanes the shader expects and possibly more");
+ return true;
+}
+
+/* -- fragment flow control ---------------------------------------------- */
+
+/* air.discard_fragment()
+ *
+ * NIR'S `demote` AND NOT NIR'S `terminate`, and that choice is the whole of
+ * this function. The two differ in what the thread does afterwards: terminate
+ * ends the invocation, so the lane stops contributing to its quad's
+ * derivatives and performs no further memory writes, while demote turns it
+ * into a HELPER invocation -- it keeps executing, its framebuffer and memory
+ * writes are dropped, and it still supplies the neighbour values that a
+ * derivative or an implicit-level-of-detail sample in the OTHER three lanes
+ * of the quad needs.
+ *
+ * It is a real distinction in NIR and in every backend that has one: ir3
+ * keeps the two apart in its own IR (ir3_compiler_nir.c emits ir3_DEMOTE
+ * against ir3_KILL, and they carry different scheduling barriers), and
+ * nir_to_tgsi sends them to TGSI's DEMOTE and KILL, which are separate
+ * opcodes. ONE PLACE COLLAPSES THEM, AND IT IS WORTH KNOWING ABOUT: the a6xx
+ * instruction encoder maps DEMOTE onto the hardware's KILL --
+ * `} else if (instr->opc == OPC_DEMOTE) { return OPC_KILL; }` in
+ * src/freedreno/isa/encode.c -- and the ISA description has no demote bitset
+ * at all (ir3-cat0.xml carries `kill` and nothing beside it). So on this GPU,
+ * with this Mesa, both come out as one instruction and the helper lanes Metal
+ * promises are not delivered by the hardware path today. That is a gap to
+ * name, not a reason to translate this as `terminate`: terminate would lose
+ * the same lanes and would be wrong on a backend that does tell them apart.
+ * What it costs is measurable only on the phone -- a shader that discards and
+ * then takes a derivative -- and that is written as a job rather than assumed.
+ *
+ * ONE THING IS NOT A DIFFERENCE BETWEEN THE TWO, and saying so is the
+ * correction of a sentence that used to stand here: stores. NIR's own
+ * definition of demote (nir_intrinsics.py) is "Any stores to memory after
+ * this instruction are suppressed and the fragment does not write outputs to
+ * the framebuffer. Unlike discard, demote needs to ensure that derivatives
+ * will still work for invocations that were not demoted." So demote drops the
+ * lane's later writes as well -- it differs from terminate in the derivatives
+ * and in the lane going on executing, not in the memory. Whether METAL drops
+ * them is a third question, and nothing on this machine answers it: no header
+ * on the device says, and a corpus cannot show it. If Metal keeps them, both
+ * of NIR's intrinsics are wrong about it by the same amount.
+ *
+ * Nothing in a corpus COUNT can decide it -- both translate, both validate,
+ * both compile -- so it is decided from two independent readings of what
+ * Metal means, and both say the thread lives:
+ *
+ * 1. The shaders themselves. Resident Evil 4's `PS_DrawByTiles` calls
+ * air.discard_fragment at the ninth instruction of a 1 800-line function
+ * and then calls `air.simd_is_helper_thread()` EIGHT TIMES further down
+ * it. A terminated thread cannot be asked whether it is a helper thread;
+ * a demoted one answers yes, which is the only reading under which that
+ * shader means anything. 24 of the library's 1 288 discard-calling modules
+ * call that function.
+ * 2. Apple's own declaration of it. `llvm-dis` of that module reads
+ * `declare void @air.discard_fragment() local_unnamed_addr #8` with
+ * `attributes #8 = { nounwind }` -- and NOT `noreturn`. LLVM is entitled
+ * to delete everything after a call it is told does not return, and
+ * Apple's compiler kept the rest of the function; the call is followed by
+ * `br` and not `unreachable` in 73 of 85 call sites sampled. A discard
+ * that killed the thread would be a noreturn call.
+ *
+ * And the cost of getting it wrong is not hypothetical here: of 85 sampled
+ * call sites, 45 are followed in the same function by an air.sample_* with an
+ * implicit level of detail and 6 by air.dfdx, both of which read the other
+ * lanes of the quad. Under `terminate` those neighbours would read a dead
+ * lane.
+ *
+ * uses_discard is set by hand because nir_shader_gather_info does not run on
+ * this path (ir3 calls it only for a binning variant), exactly as the
+ * framebuffer-fetch flags are set in air_to_nir.c. It is not bookkeeping: it
+ * is what nir_to_tgsi_info turns into `uses_kill`, which is what a Gallium
+ * driver reads before it allows an early depth test, and what freedreno reads
+ * before it writes LRZ. A discarding shader whose interface says it discards
+ * nothing gets its depth written for pixels it threw away.
+ */
+static bool
+discard_op(struct air_ctx *c, const struct air_inst *in)
+{
+ if (c->nir->info.stage != MESA_SHADER_FRAGMENT)
+ return air_error(c, "intrinsic: air.discard_fragment in a %s shader",
+ gl_shader_stage_name(c->nir->info.stage));
+ nir_demote(&c->b);
+ c->nir->info.fs.uses_discard = true;
+ return true;
+}
+
+/* -- SIMD groups and quads ----------------------------------------------- */
+
+/* air.simd_* and air.quad_*: Metal's cross-lane operations.
+ *
+ * THE SIMD-GROUP HERE IS THE IMPLEMENTATION'S, NOT A 32-LANE ONE, and that is
+ * the only way these can be translated honestly. Metal does not fix the width
+ * -- a shader asks for it with air.get_simd_width (this file answers with
+ * nir_load_subgroup_size) and a host asks MTLComputePipelineState for
+ * threadExecutionWidth -- and it is 32 on every Apple GPU, which is why a
+ * shader compiled for one may have the number built into it. This hardware's
+ * is 64 or 128: `threadsize_base` is 64 for the A635 this port targets and ir3
+ * doubles it per variant in the compute and fragment stages
+ * (ir3_shader_get_subgroup_size, IR3_SINGLE_OR_DOUBLE).
+ *
+ * So the width a translated shader is TOLD and the width its votes are taken
+ * over are the same number, whatever that number is: a shader that reads
+ * air.get_simd_width and works from it is right, and a shader with 32 written
+ * into it is wrong -- as it would be on any Metal implementation whose
+ * SIMD-group is not 32 wide. Each of these leaves a note saying so, because it
+ * is a thing the hardware cannot do exactly rather than a thing this frontend
+ * chose.
+ *
+ * One more thing decides whether that consistency holds, and it is a property
+ * of the GPU and not of this file: ir3 declares a fragment or vertex
+ * subgroup to be ONE LANE and lowers a vote to its own argument on a device
+ * without `has_getfiberid` (ir3_nir.c: `options.subgroup_size = 1`,
+ * `lower_vote_trivial`). The A635 has it -- it is an a6xx gen4, and
+ * freedreno_devices.py sets the flag for gen4 and above -- so the votes here
+ * are real in every stage, and air.get_simd_width answers with the real width
+ * in the same stages. On an a6xx below gen4 both would read 1, which is still
+ * self-consistent and still not Metal's 32.
+ *
+ * air.simd_ballot is a 64-bit mask with one bit per lane, and it is exact
+ * only while a group has at most 64 lanes. That is the driver's to promise,
+ * not this frontend's: hakios-metal builds every variant with zeroed
+ * shader_options, IR3_SINGLE_ONLY, so ir3 never doubles the threadsize and a
+ * group is threadsize_base, 64 lanes, in every stage -- the execution width
+ * the device reports. A driver that lets ir3 run 128 would get the low half
+ * of the mask; the note below says so where it is emitted.
+ *
+ * The QUAD operations are a different matter and are exact: a quad is four
+ * consecutive lanes on both machines -- it is what a fragment derivative is
+ * computed over -- so a swap inside one has the same meaning whatever the
+ * SIMD-group's width is, and nir_lower_subgroups leaves the quad intrinsics
+ * alone for ir3 (no lower_quad, and lower_vote_trivial does not reach them).
+ */
+static bool
+simd_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n,
+ bool *handled)
+{
+ const char *op = n->part[0];
+
+ /* THE NAME IS DECIDED BEFORE AN ARGUMENT IS TOUCHED, which is the same rule
+ * math_op states above it and fc_defined_op exists for -- and getting it
+ * wrong here cost 46 modules of the 456 for the length of one commit. This
+ * function used to resolve its first argument before asking whether the
+ * name was one of its own, and every texture operation passes a texture
+ * HANDLE as that argument: air.sample_texture_2d came back "instruction: a
+ * pointer where a number belongs", which reads like a broken module and
+ * names no operation at all. The count was the only visible symptom and it
+ * was briefly attributed to somebody else's commit.
+ */
+ if (strncmp(op, "simd_", 5) && strncmp(op, "quad_", 5) &&
+ strcmp(op, "get_sample_position") && strcmp(op, "is_uniform")) {
+ *handled = false;
+ return true;
+ }
+ *handled = true;
+
+ if (!strcmp(op, "simd_is_helper_thread")) {
+ /* Exact, and width-independent: a helper thread is one whose results
+ * are discarded, which is a property of the thread and not of a group.
+ * ir3 runs nir_lower_is_helper_invocation itself, and this is the other
+ * half of the discard above -- a demoted lane must answer yes here.
+ */
+ if (c->nir->info.stage != MESA_SHADER_FRAGMENT)
+ return air_error(c, "intrinsic: air.simd_is_helper_thread in a %s shader "
+ "(there are no helper threads outside the fragment "
+ "stage)", gl_shader_stage_name(c->nir->info.stage));
+ air_set(c, &in->base, air_val_ssa(c, nir_is_helper_invocation(&c->b, 1)));
+ return true;
+ }
+
+ nir_def *a0 = in->num_operands > 1 ? air_get_ssa(c, in->operands[1]) : NULL;
+ if (in->num_operands > 1 && !a0)
+ return false;
+
+ if (!strcmp(op, "simd_ballot")) {
+ /* Resident Evil 4's culling passes elect a lane with it (a ballot of the
+ * live lanes, then a popcount of the ones below this lane) */
+ if (!a0 || a0->bit_size != 1 || in->base.type->kind != AIR_TYPE_INT ||
+ in->base.type->bits != 64)
+ return air_error(c, "intrinsic: air.simd_ballot of something other than a "
+ "bool into a 64-bit mask");
+ air_note(c, "air.simd_ballot is a mask of the SIMD group this implementation "
+ "has, 64 lanes when the driver keeps ir3 to its single threadsize "
+ "(hakios-metal does), not Metal's 32: a shader that truncates it to "
+ "32 bits and indexes it by lane & 31 sees two groups of 32 in it");
+ air_set(c, &in->base, air_val_ssa(c, nir_ballot(&c->b, 1, 64, a0)));
+ return true;
+ }
+
+ if (!strcmp(op, "simd_all") || !strcmp(op, "simd_any")) {
+ bool all = !strcmp(op, "simd_all");
+ air_note(c, "air.%s is taken over the SIMD group this implementation has, "
+ "which is not Metal's 32 lanes: ir3 runs 64 or 128 of them on this "
+ "hardware. air.get_simd_width answers with the same number, so a "
+ "shader that reads it is right and one with 32 built into it is not",
+ n->part[0]);
+ air_set(c, &in->base,
+ air_val_ssa(c, all ? nir_vote_all(&c->b, 1, a0)
+ : nir_vote_any(&c->b, 1, a0)));
+ return true;
+ }
+
+ /* air.is_uniform (Metal's simd_is_uniform, mediaserverd's CNN kernels): whether
+ * every active lane holds the same value -- NIR's vote_ieq / vote_feq, over the
+ * group this hardware has (the note above). A 16-bit value is widened first,
+ * which keeps equality as it was.
+ */
+ if (!strcmp(op, "is_uniform")) {
+ if (!a0)
+ return air_error(c, "intrinsic: air.is_uniform with no value");
+ const bool fl = n->num > 1 && n->part[1][0] == 'f';
+ nir_def *v = a0->bit_size < 32 ? (fl ? nir_f2f32(&c->b, a0) : nir_u2u32(&c->b, a0)) : a0;
+ air_set(c, &in->base, air_val_ssa(c, fl ? nir_vote_feq(&c->b, 1, v) : nir_vote_ieq(&c->b, 1, v)));
+ return true;
+ }
+
+ if (!strcmp(op, "simd_broadcast_first")) {
+ air_note(c, "air.%s reads the first active lane of the SIMD group this "
+ "implementation has, which is not Metal's 32 lanes -- the value is "
+ "the same one Metal would give wherever it is uniform across the "
+ "group, and another lane's where it is not", n->part[0]);
+ air_set(c, &in->base, air_val_ssa(c, nir_read_first_invocation(&c->b, a0)));
+ return true;
+ }
+
+ if (!strcmp(op, "simd_shuffle")) {
+ nir_def *lane = air_get_ssa(c, in->operands[2]);
+ if (!lane)
+ return false;
+ air_note(c, "air.%s takes its lane number in the SIMD group this "
+ "implementation has, which is not Metal's 32 lanes", n->part[0]);
+ air_set(c, &in->base,
+ air_val_ssa(c, nir_shuffle(&c->b, a0, nir_u2u32(&c->b, lane))));
+ return true;
+ }
+
+ if (!strcmp(op, "simd_is_first")) {
+ /* simd_is_first(): true in exactly one lane of the SIMD group, the
+ * lowest-numbered ACTIVE one. NIR's `elect` is that operation and
+ * nothing else -- it is SPIR-V's OpGroupNonUniformElect -- and ir3 emits
+ * it directly (nir_intrinsic_elect in ir3_compiler_nir.c), with no
+ * lowering in between to disagree with.
+ *
+ * This is one of the few cross-lane operations whose MEANING does not
+ * depend on the group's width: "the first active lane" is the first
+ * active lane at 32, 64 or 128. What does depend on the width is what
+ * the shader then does with it, and in the two corpus modules that call
+ * this the dependence is total: CMISharpnessScore::sumScoreStep{1,2}
+ * elect one lane per SIMD-group to write that group's air.simd_sum into
+ * `simdGroupSums`, a threadgroup array declared `[32 x float]` and
+ * indexed by the SIMD-group's index in the threadgroup, and then read
+ * back at `simd_lane_id`. At 64 lanes that read is past the end of the
+ * array. So translating this operation makes those two modules COMPILE
+ * and does not make them right, and the note says which.
+ *
+ * The reading "lane 0" would agree with `elect` on every case those two
+ * modules can produce, because both calls sit in reconvergent control
+ * flow where lane 0 is active -- which is exactly the kind of test that
+ * passes for the wrong reason. The reason to prefer `elect` is not the
+ * corpus: it is that Apple's own header gates the function on
+ * __HAVE_SIMDGROUP_IS_FIRST__ beside simd_is_helper_thread, whose whole
+ * subject is which lanes are live, and that a lane-0 reading would need
+ * air.get_simd_lane_id, which the operation does not take.
+ */
+ /* NO SEMICOLON IN A NOTE. air2nir's --report joins the notes of a module
+ * with `;` and corpus.py splits them on it, so a note with one in it
+ * arrives in the report as two rows, the second of them starting
+ * mid-sentence. Found by reading the report after adding this one.
+ */
+ air_note(c, "air.simd_is_first elects one lane of the SIMD group this "
+ "implementation has, which is not Metal's 32 lanes: ir3 runs 64 or "
+ "128 of them on this hardware. The election itself is exact at any "
+ "width, but a shader that sizes a per-SIMD-group array at 32, or "
+ "indexes one by air.get_simd_lane_id, is relying on the width and is "
+ "wrong here however this operation is translated");
+ air_set(c, &in->base, air_val_ssa(c, nir_elect(&c->b, 1)));
+ return true;
+ }
+
+ if (!strcmp(op, "simd_broadcast")) {
+ /* simd_broadcast(data, lane): the value held by one named lane, given to
+ * every lane. NIR's read_invocation, which is the same operation; ir3
+ * lowers it to read_invocation_cond_ir3 (lower_read_invocation_to_cond)
+ * rather than refusing it.
+ */
+ nir_def *lane = air_get_ssa(c, in->operands[2]);
+ if (!lane)
+ return false;
+ air_note(c, "air.%s takes its lane number in the SIMD group this "
+ "implementation has, which is not Metal's 32 lanes", n->part[0]);
+ air_set(c, &in->base,
+ air_val_ssa(c, nir_read_invocation(&c->b, a0,
+ nir_u2u32(&c->b, lane))));
+ return true;
+ }
+
+ if (!strcmp(op, "simd_shuffle_up") || !strcmp(op, "simd_shuffle_down") ||
+ !strcmp(op, "simd_shuffle_xor")) {
+ /* simd_shuffle_{up,down,xor}(data, delta): the value held by the lane
+ * `lane - delta`, `lane + delta` or `lane ^ delta`. NIR has all three
+ * with the same definition, and in both languages the result is
+ * UNDEFINED when the computed lane is outside the group -- SPIR-V's
+ * OpGroupNonUniformShuffle{Up,Down,Xor} say so and Metal says so -- so
+ * the pair agrees on the defined cases and neither promises anything
+ * about the rest. That is why these three are translated and
+ * air.simd_shuffle_and_fill_* below is not.
+ *
+ * THE WIDTH IS WORSE HERE THAN FOR A VOTE, and the note says so. A
+ * reduction written as a shuffle ladder -- deltas 16, 8, 4, 2, 1 -- puts
+ * the sum of lanes 0..31 in lane 0 whatever the group's width is, because
+ * lane 0 never reads above lane 31; but every OTHER lane of a 64- or
+ * 128-wide group reads across the 32-lane window the shader believes in,
+ * so a shader that uses more than lane 0's answer gets a different number
+ * here than on an Apple GPU. Which of the two it is cannot be decided by
+ * this function: it is a property of the shader.
+ */
+ nir_def *delta = air_get_ssa(c, in->operands[2]);
+ if (!delta)
+ return false;
+ delta = nir_u2u32(&c->b, delta);
+ air_note(c, "air.%s steps by lanes of the SIMD group this implementation has, "
+ "which is not Metal's 32 lanes: ir3 runs 64 or 128 of them on this "
+ "hardware, so a lane whose source is outside Apple's 32-lane window "
+ "reads a real neighbour here where Metal leaves it undefined",
+ n->part[0]);
+ nir_def *r = !strcmp(op, "simd_shuffle_up")
+ ? nir_shuffle_up(&c->b, a0, delta)
+ : !strcmp(op, "simd_shuffle_down")
+ ? nir_shuffle_down(&c->b, a0, delta)
+ : nir_shuffle_xor(&c->b, a0, delta);
+ air_set(c, &in->base, air_val_ssa(c, r));
+ return true;
+ }
+
+ if (!strcmp(op, "simd_shuffle_and_fill_down") ||
+ !strcmp(op, "simd_shuffle_and_fill_up")) {
+ /* REFUSED BY NAME, and the refusal says where the answer is rather than
+ * that there is none -- which is the difference between a wall and a
+ * shrug, and this one has an address.
+ *
+ * The signature is Apple's: metal_simdgroup on this device declares
+ * `simd_shuffle_and_fill_down(T data, T filling_data, ushort delta,
+ * ushort modulo)` and a two-argument overload that passes
+ * `__metal_get_simdgroup_size()` as the modulo. So `modulo` is a WINDOW
+ * width, and the operation shuffles within a window of that many lanes
+ * and takes `filling_data` where the source lane would leave it. What the
+ * header does not state is which lane of `filling_data` it takes, and the
+ * two readings -- the lane `delta` from the window's start, or the same
+ * lane the shuffle would have had, counted on past the window -- give
+ * different answers for every call.
+ *
+ * And the modulo is not a formality that could be ignored: Apple's own
+ * calls pass a LITERAL. CMISharpnessScore::sharpnessScoreCalculatorFloat
+ * passes 32 twice and 4 once; FRC's modules pass 4, 8 and a runtime
+ * value. A literal 32 is Apple's group width written into the shader, and
+ * on a 64- or 128-lane group a plain shuffle crosses that window instead
+ * of filling at it, which is a wrong value and not an undefined one. A
+ * literal 4 is not the group width at all and would be wrong on an Apple
+ * GPU too if it were read as one.
+ *
+ * WHERE THE ANSWER IS: AGXCompilerCore in this device's own cache lowers
+ * it, and the symbols are not hidden --
+ * `AGCLLVMGen::TargetLowerer::buildShuffleAndFill` (`0x1bbefa8a4`),
+ * `...::buildShuffleAndFillCore` (`0x1bbefaa1c`) and
+ * `...::buildShuffleAndFillIfModuloIsAlpha` (`0x1bbefaf74`). The first
+ * reads one byte at offset 0x10 of the `modulo` operand -- where
+ * `llvm::Value` keeps its SubclassID -- and takes `...Core` for one value
+ * of it and `...IfModuloIsAlpha` for the rest, so a constant modulo has a
+ * path of its own, which is the path every call in this image takes. That
+ * binary also carries the string `AGC_SIMD_ROTATE_AS_SHUFFLE_AND_FILL`,
+ * which is why the rotate refusal below is this same wall and not another
+ * one. Reading `...Core` out is what would settle the fill's source lane;
+ * it has not been read, and inventing the formula instead is the thing
+ * this frontend does not do (docs/air.md 10.3).
+ */
+ /* INSIDE air_error's 255 BYTES, counted rather than hoped: the name is 30
+ * of them and the tail is 198, so the whole sentence arrives. The detail
+ * that does not fit is the comment above and docs/air.md 10.3.
+ */
+ return air_error(c, "intrinsic: air.%s -- the `modulo` lane window (Apple passes "
+ "32, 4 and 8). Which lane of `filling_data` fills is settled "
+ "by AGXCompilerCore buildShuffleAndFillCore 0x1bbefaa1c, "
+ "unread: docs/air.md 10.3", n->part[0]);
+ }
+
+ if (!strcmp(op, "simd_shuffle_rotate_down") || !strcmp(op, "simd_shuffle_rotate_up")) {
+ /* A rotation inside the SIMD group: `(lane + delta) mod modulo`, which
+ * names a different lane at 32 than at 64 in EVERY lane rather than only
+ * near the edge. NIR has nir_rotate and ir3 lowers it
+ * (lower_rotate_clustered_to_shuffle), so the operation exists on both
+ * sides; what does not exist here is the window to rotate within.
+ *
+ * It is the fill's wall and not a second one, which Apple's own compiler
+ * says rather than this file inferring it: AGXCompilerCore carries
+ * `AGC_SIMD_ROTATE_AS_SHUFFLE_AND_FILL`, so the rotate is built out of
+ * the operation above. No module of either the corpus or the image calls
+ * one, so this is a refusal with no customer rather than a gap in reach.
+ */
+ return air_error(c, "intrinsic: air.%s -- a rotation in the SIMD group, which "
+ "AGXCompilerCore builds out of shuffle-and-fill "
+ "(AGC_SIMD_ROTATE_AS_SHUFFLE_AND_FILL), so it is that "
+ "operation's wall: docs/air.md 10.3", n->part[0]);
+ }
+
+ if (!strcmp(op, "quad_shuffle") || !strcmp(op, "quad_broadcast")) {
+ /* Apple's two names for one operation, and the header says so:
+ * metal_quadgroup declares `quad_shuffle(T data, ushort quad_lane_id)`
+ * and `quad_broadcast(T data, ushort broadcast_lane_id)` -- an absolute
+ * lane WITHIN the quad in both cases. NIR's quad_broadcast is that, and
+ * it is exact at any SIMD-group width because a quad is four consecutive
+ * lanes on both machines.
+ *
+ * NIR inherits SPIR-V's requirement that the index be uniform across the
+ * quad, and so does Metal's own quad_broadcast; `quad_shuffle` is the
+ * name for the non-uniform reading, which is why both are here and only
+ * a constant or a quad-uniform index is honest. Nothing enforces that:
+ * the index is not inspected, because a value this frontend cannot prove
+ * uniform is not a value it can prove non-uniform either, and refusing
+ * every non-constant index would refuse the uniform ones with it.
+ */
+ nir_def *lane = air_get_ssa(c, in->operands[2]);
+ if (!lane)
+ return false;
+ air_set(c, &in->base,
+ air_val_ssa(c, nir_quad_broadcast(&c->b, a0, nir_u2u32(&c->b, lane))));
+ return true;
+ }
+
+ if (!strcmp(op, "quad_shuffle_down") || !strcmp(op, "quad_shuffle_up")) {
+ /* quad_shuffle_{down,up}(data, delta): the lane `delta` along inside the
+ * quad, undefined when that leaves the quad. NIR has no quad-restricted
+ * shuffle, so this is nir_shuffle_{down,up} over the SIMD group -- which
+ * gives the same lane for every case the quad contains, and reads the
+ * neighbouring quad for the cases Metal leaves undefined.
+ *
+ * That substitution is sound in exactly the way the `and_fill` one above
+ * is not: there, the out-of-window case has a DEFINED result that this
+ * would get wrong; here it has none, so a real neighbour's value is as
+ * conforming as anything else. What it is not is identical to Apple's
+ * hardware, and a shader written against that hardware's actual
+ * behaviour out of range would differ -- which no reading of the bitcode
+ * can detect, so it is a note and not a check.
+ */
+ nir_def *delta = air_get_ssa(c, in->operands[2]);
+ if (!delta)
+ return false;
+ delta = nir_u2u32(&c->b, delta);
+ air_note(c, "air.%s is emitted as a SIMD-group shuffle, which agrees with it "
+ "inside the quad and reads the neighbouring quad where Metal leaves "
+ "the result undefined", n->part[0]);
+ air_set(c, &in->base,
+ air_val_ssa(c, !strcmp(op, "quad_shuffle_down")
+ ? nir_shuffle_down(&c->b, a0, delta)
+ : nir_shuffle_up(&c->b, a0, delta)));
+ return true;
+ }
+
+ if (!strcmp(op, "quad_shuffle_xor")) {
+ /* quad_shuffle_xor(data, mask) hands each lane the value of the lane
+ * whose number is its own exclusive-ored with the mask. Within a quad
+ * that is one of three fixed swaps, which is exactly the three NIR has:
+ * 1 swaps the two columns, 2 the two rows, 3 the diagonal. A mask of 0
+ * is the identity.
+ *
+ * A mask that is not a constant, or is 4 or more, is refused rather
+ * than translated: the first would need a selection over the three
+ * swaps, and the second leaves the quad, where Metal's own definition
+ * stops. Every call in Resident Evil 4's three modules passes a
+ * constant.
+ */
+ nir_def *mask = air_get_ssa(c, in->operands[2]);
+ if (!mask)
+ return false;
+ if (!nir_src_is_const(nir_src_for_ssa(mask)))
+ return air_error(c, "intrinsic: air.%s with a mask that is not a constant",
+ n->part[0]);
+ uint64_t m = nir_src_as_uint(nir_src_for_ssa(mask));
+ switch (m) {
+ case 0: air_set(c, &in->base, air_val_ssa(c, a0)); return true;
+ case 1: air_set(c, &in->base,
+ air_val_ssa(c, nir_quad_swap_horizontal(&c->b, a0)));
+ return true;
+ case 2: air_set(c, &in->base,
+ air_val_ssa(c, nir_quad_swap_vertical(&c->b, a0)));
+ return true;
+ case 3: air_set(c, &in->base,
+ air_val_ssa(c, nir_quad_swap_diagonal(&c->b, a0)));
+ return true;
+ default:
+ return air_error(c, "intrinsic: air.%s with mask %u, which leaves the quad",
+ n->part[0], (unsigned)m);
+ }
+ }
+
+ /* air.get_sample_position(index, 0): the position of one sample within the
+ * pixel, which the hardware's sample pattern decides.
+ *
+ * REFUSED BY NAME, and the reason is half a pixel. NIR has the operation
+ * and the Adreno has the instruction -- nir_load_sample_pos_from_id, which
+ * ir3 emits as RGETPOS -- but that intrinsic's convention is an OFFSET
+ * from the pixel's centre, in [-0.5, 0.5]: ir3's own lowering of
+ * gl_SamplePosition adds (0.5, 0.5) to it and cites the dEQP case that
+ * fixes the difference (ir3_nir_lower_load_barycentric_at_sample.c).
+ * Metal's convention is not stated by anything on this machine -- not by
+ * the metal_graphics header on the device, which only names the function
+ * and its MSL section, and not by any string in the compiler libraries --
+ * and the two readings differ by exactly half a pixel, which is the kind
+ * of wrongness that looks like a slightly soft image rather than like a
+ * bug. 4 of Resident Evil 4's 7 879 modules call it, and the way to settle
+ * it is a measurement on the phone (draw one multisampled pixel, read the
+ * positions back) rather than a choice made here.
+ */
+ if (!strcmp(op, "get_sample_position"))
+ return air_error(c, "intrinsic: air.%s -- NIR's sample position is an offset "
+ "from the pixel centre and Metal's convention is not "
+ "attested anywhere on this machine; the two differ by half "
+ "a pixel, so this is left to a measurement", n->part[0]);
+
+ *handled = false;
return true;
}
@@ -436,6 +1638,10 @@ llvm_op(struct air_ctx *c, const struct air_inst *in, const char *name)
{
if (!strncmp(name, "llvm.lifetime.", 14))
return true; /* scope markers: nothing to emit */
+ /* a promise to the optimizer that its condition holds (MPS's CNN kernels
+ * carry them): it changes nothing a shader computes, so nothing either */
+ if (!strcmp(name, "llvm.assume"))
+ return true;
if (!strncmp(name, "llvm.memcpy.", 12)) {
/* a copy between two of our pointers, of a constant length */
struct air_val *dst = air_get(c, in->operands[1]);
@@ -470,17 +1676,87 @@ llvm_op(struct air_ctx *c, const struct air_inst *in, const char *name)
if (dst->kind != AIR_VAL_PTR)
return air_error(c, "intrinsic: memset of something that is not a pointer");
uint64_t bytes;
- if (!air_const_uint(in->operands[3], &bytes) || bytes % 4)
- return air_error(c, "intrinsic: memset of a length that is not a multiple of 4");
+ if (!air_const_uint(in->operands[3], &bytes))
+ return air_error(c, "intrinsic: memset of a run-time length");
if (!nir_src_is_const(nir_src_for_ssa(byte)))
return air_error(c, "intrinsic: memset of a run-time byte");
uint32_t b = nir_src_as_uint(nir_src_for_ssa(byte)) & 0xff;
uint32_t word = b | b << 8 | b << 16 | b << 24;
- for (uint64_t off = 0; off < bytes; off += 4) {
+ /* WHOLE WORDS, THEN WHATEVER IS LEFT. The length used to have to be a
+ * multiple of four, and the one llvm.memset in all 852 modules of the
+ * corpus is not: DepthProcessorV1's `kernelComputeStage1` zeroes EIGHTEEN
+ * bytes of a `device` buffer, which is four words and a short. So that
+ * rule never once applied to anything it allowed. There is nothing to
+ * decide here -- a memset's meaning is LLVM's own and the bytes it writes
+ * are not open to interpretation -- so the tail is written at the width it
+ * is, 16 bits and then 8, which air_store_ptr carries because a store is a
+ * store of whatever bit size the value has.
+ *
+ * Three stores and not a byte loop, because the length is a constant and
+ * a byte loop would emit eighteen of them. `llvm.memcpy` keeps its
+ * multiple-of-four refusal: a tail there needs a LOAD of the same odd
+ * width as well, nothing in the image asks for one, and a lowering with
+ * no customer is a lowering with no test.
+ */
+ uint64_t off = 0;
+ for (; off + 4 <= bytes; off += 4) {
struct air_ptr d = air_ptr_add(c, dst->ptr, nir_imm_int(&c->b, off));
if (!air_store_ptr(c, d, nir_imm_int(&c->b, word), 4))
return false;
}
+ if (bytes - off >= 2) {
+ struct air_ptr d = air_ptr_add(c, dst->ptr, nir_imm_int(&c->b, off));
+ if (!air_store_ptr(c, d, nir_imm_intN_t(&c->b, word & 0xffff, 16), 2))
+ return false;
+ off += 2;
+ }
+ if (bytes - off >= 1) {
+ struct air_ptr d = air_ptr_add(c, dst->ptr, nir_imm_int(&c->b, off));
+ if (!air_store_ptr(c, d, nir_imm_intN_t(&c->b, b, 8), 1))
+ return false;
+ }
+ return true;
+ }
+ /* llvm.fmuladd(a, b, c): a multiply-add that MAY be fused.
+ *
+ * Translated as a fused one -- nir_ffma, a single rounding -- and the
+ * licence for that is the intrinsic's own definition rather than the
+ * module's fast-math flag. LLVM's Language Reference says of llvm.fmuladd
+ * that the expression "is equivalent to a * b + c, except that it is
+ * unspecified whether rounding will be performed between the
+ * multiplication and addition steps", and that a caller who needs the
+ * fusion guaranteed must use llvm.fma instead. A producer that emitted
+ * llvm.fmuladd rather than `fmul` + `fadd` therefore accepted both answers
+ * when it chose the name, and the choice between them is the back end's.
+ *
+ * SO THE ABSENCE OF air.compile.fast_math_enable CHANGES NOTHING HERE, and
+ * that is worth saying because the measurement invites the opposite
+ * reading: 383 of a 394-module sample of Resident Evil 4 carry the flag, so
+ * contraction is permitted under the stricter reading too -- but the eleven
+ * that do not are not a different case for THIS intrinsic. A module that
+ * wanted two roundings would carry two instructions.
+ *
+ * Which answer the Adreno gives is then ir3's: nir_ffma is one rounding,
+ * and a backend that has no fused multiply-add would lower it. softpipe
+ * does not have one -- TGSI's own specification lets MAD round the product
+ * first and its interpreter does -- which is why tools/air/intrincheck.py
+ * sets air.fma aside on softpipe and leaves it to the phone. The same
+ * applies to every llvm.fmuladd this translates.
+ */
+ if (!strncmp(name, "llvm.fmuladd.", 13)) {
+ nir_def *a = air_get_ssa(c, in->operands[1]);
+ nir_def *b = air_get_ssa(c, in->operands[2]);
+ nir_def *d = air_get_ssa(c, in->operands[3]);
+ if (!a || !b || !d)
+ return false;
+ air_set(c, &in->base, air_val_ssa(c, nir_ffma(&c->b, a, b, d)));
+ return true;
+ }
+ if (!strncmp(name, "llvm.fabs.", 10)) {
+ nir_def *a = air_get_ssa(c, in->operands[1]);
+ if (!a)
+ return false;
+ air_set(c, &in->base, air_val_ssa(c, nir_fabs(&c->b, a)));
return true;
}
if (!strcmp(name, "llvm.trap"))
@@ -501,19 +1777,51 @@ llvm_op(struct air_ctx *c, const struct air_inst *in, const char *name)
*
* The signatures are measured from the corpus, not guessed:
*
- * sample_<kind>_<dim> (tex, samp, [plane,] coord, i1 has_offset, offset,
+ * sample_<kind>_<dim> (tex, samp, [plane,] coord, [i32 array,]
+ * [i1 has_offset, offset,]
* i1 lod_is_level, float lod_or_bias, float min_lod,
* i32 flags) -> { <4 x T>, i8 }
- * gather_<kind>_<dim> (tex, samp, coord, i1 has_offset, offset,
+ * sample_<kind>_<dim>_grad
+ * (tex, samp, [plane,] coord, [i32 array,]
+ * dPdx, dPdy, float min_lod,
+ * [i1 has_offset, offset,] i32 flags)
+ * gather_<kind>_<dim> (tex, samp, coord, [i32 array,]
+ * [i1 has_offset, offset,]
* i32 component, i32 flags) -> { <4 x T>, i8 }
- * read_<kind>_<dim> (tex, [plane,] coord, i32 lod_or_sample, i32 flags)
- * write_<kind>_<dim> (tex, coord, value, i32 lod, i32 flags)
+ * read_<kind>_<dim> (tex, [plane,] coord, [i32 array,]
+ * i32 lod_or_sample, i32 flags)
+ * write_<kind>_<dim> (tex, coord, [i32 array,] value, i32 lod, i32 flags)
* get_<what>_<kind>_<dim> (tex[, i32 lod]) -> i32
*
+ * Three things in those brackets, and none of them is a guess. They are read
+ * off Apple's own metal_stdlib on this device --
+ * GPUCompiler.framework/.../include/metal/metal_texture, where every
+ * `texture2d_array<T>::sample` and its fifty neighbours are one-line
+ * forwarders to the builtin, so the header states the argument ORDER for all
+ * nine dimensionalities and six operations -- and then confirmed against the
+ * LLVM declaration in the modules that call them. The two agree everywhere:
+ *
+ * AN ARRAY INDEX IS ITS OWN ARGUMENT, straight after the coordinate, in
+ * every one of sample, gather, read and write. It is not inside the
+ * coordinate vector, which is what this file used to require of it:
+ * air.sample_texture_2d_array takes a <2 x float> and an i32, and the
+ * refusal it used to earn said "with a 2-component coordinate, wanting 3".
+ *
+ * THE GRADIENT FORM REORDERS WHAT FOLLOWS. dPdx, dPdy and the minimum level
+ * come before the offset pair, where the plain form has the offset pair
+ * first and the level last. A reader that assumed one order for both would
+ * take a gradient for an offset flag.
+ *
+ * A CUBE HAS NO OFFSET PAIR AT ALL -- Metal defines no offset for cube
+ * sampling -- so air.sample_texture_cube goes from the coordinate straight
+ * to the level flag. Reading the 2d shape there is what made 621 modules
+ * stop at "argument 5 is not a constant": the float minimum level was being
+ * read as the i1 that says whether the next float is a level.
+ *
* The i1 before the level is read as "the float is a level, not a bias", so
- * (false, 0.0) is a plain sample -- which is what every one of the corpus's
- * sample calls passes. The level and bias forms are written here but no
- * module exercises them.
+ * (false, 0.0) is a plain sample -- which is what every one of the system
+ * corpus's sample calls passes. Resident Evil 4 exercises both: 23 823
+ * level-form calls against 5 961 bias-form.
*
* The second field of the returned struct is Metal's sparse-residency byte.
* No module in the corpus reads it (every extractvalue of a texture result
@@ -527,6 +1835,8 @@ struct air_tex_name {
bool depth; /* a depth texture rather than a colour one */
enum glsl_sampler_dim dim;
bool is_array, is_ms, is_buffer;
+ bool grad; /* the _grad form: explicit dPdx and dPdy */
+ bool no_offset; /* this dimensionality takes no offset argument */
};
static bool
@@ -592,27 +1902,41 @@ parse_tex_name(const char *name, struct air_tex_name *t)
else if (!eat(&s, "texture_"))
return false;
- /* the dimensionality, longest first so that 2d_ms_array beats 2d */
+ /* The gradient form appends _grad to the DIMENSIONALITY and not to the
+ * operation: Apple's builtin is __metal_sample_texture_2d_array_t_grad, so
+ * the air.* name is air.sample_texture_2d_array_grad. Taking it off here
+ * rather than giving the table nine more rows is what keeps one name for
+ * one dimensionality -- and the exact comparison below is why 858 modules
+ * used to stop at "a texture name this cannot take apart".
+ */
+ size_t len = strlen(s);
+ if (len > 5 && !strcmp(s + len - 5, "_grad")) {
+ t->grad = true;
+ len -= 5;
+ }
+
+ /* the dimensionality, and whether it has an offset argument at all */
static const struct {
const char *suffix;
enum glsl_sampler_dim dim;
- bool array, ms;
+ bool array, ms, no_offset;
} dims[] = {
- { "cube_array", GLSL_SAMPLER_DIM_CUBE, true, false },
- { "2d_ms_array", GLSL_SAMPLER_DIM_MS, true, true },
- { "2d_array", GLSL_SAMPLER_DIM_2D, true, false },
- { "1d_array", GLSL_SAMPLER_DIM_1D, true, false },
- { "2d_ms", GLSL_SAMPLER_DIM_MS, false, true },
- { "cube", GLSL_SAMPLER_DIM_CUBE, false, false },
- { "1d", GLSL_SAMPLER_DIM_1D, false, false },
- { "2d", GLSL_SAMPLER_DIM_2D, false, false },
- { "3d", GLSL_SAMPLER_DIM_3D, false, false },
+ { "cube_array", GLSL_SAMPLER_DIM_CUBE, true, false, true },
+ { "2d_ms_array", GLSL_SAMPLER_DIM_MS, true, true, false },
+ { "2d_array", GLSL_SAMPLER_DIM_2D, true, false, false },
+ { "1d_array", GLSL_SAMPLER_DIM_1D, true, false, false },
+ { "2d_ms", GLSL_SAMPLER_DIM_MS, false, true, false },
+ { "cube", GLSL_SAMPLER_DIM_CUBE, false, false, true },
+ { "1d", GLSL_SAMPLER_DIM_1D, false, false, false },
+ { "2d", GLSL_SAMPLER_DIM_2D, false, false, false },
+ { "3d", GLSL_SAMPLER_DIM_3D, false, false, false },
};
for (unsigned i = 0; i < ARRAY_SIZE(dims); i++) {
- if (!strcmp(s, dims[i].suffix)) {
+ if (strlen(dims[i].suffix) == len && !strncmp(s, dims[i].suffix, len)) {
t->dim = dims[i].dim;
t->is_array = dims[i].array;
t->is_ms = dims[i].ms;
+ t->no_offset = dims[i].no_offset;
return true;
}
}
@@ -756,17 +2080,192 @@ all_zero(nir_def *def)
return true;
}
+/* The coordinate, with the array index that follows it folded into it.
+ *
+ * NIR wants one coordinate with the layer as its last component; AIR passes
+ * the two separately (see the signatures above). Which way the layer is
+ * converted follows the coordinate's own type, which is also the only place
+ * the answer can come from: a sample's coordinate is floating point and NIR's
+ * array layer is then a float like the rest of it, while a read's is an
+ * integer and the layer stays one.
+ *
+ * The float conversion is exact and not a rounding to be worried about. Metal
+ * hands over an integer layer; u2f of an integer below 2^24 is exact in a
+ * float, and 2^24 layers is 16 million slices of a texture array whose limit
+ * on this hardware is 2 048. What a backend does with a *fractional* layer --
+ * Vulkan says round to even, GLSL says nearest -- therefore never arises
+ * here, because the value handed to it is always a whole number.
+ *
+ * An integer coordinate is widened to 32 bits first so that the layer, which
+ * arrives as i32, does not have to be narrowed to meet it: air.read_texture_2d
+ * has a 16-bit coordinate variant and the layer in those signatures is 32-bit
+ * all the same.
+ */
+static nir_def *
+tex_coord_and_layer(struct air_ctx *c, const struct air_inst *in,
+ const struct air_tex_name *t, const struct air_name *n,
+ unsigned *arg, unsigned ncoord)
+{
+ if (in->num_operands <= *arg) {
+ air_error(c, "intrinsic: air.%s without a coordinate", n->part[0]);
+ return NULL;
+ }
+ const struct air_value *cv = in->operands[*arg];
+ nir_def *coord = air_get_ssa(c, cv);
+ if (!coord)
+ return NULL;
+ (*arg)++;
+ bool is_float = air_type_is_float(air_type_scalar(cv->type));
+ if (!is_float)
+ coord = nir_u2u32(&c->b, coord);
+
+ if (t->is_array) {
+ if (in->num_operands <= *arg) {
+ air_error(c, "intrinsic: air.%s without the array index its signature has",
+ n->part[0]);
+ return NULL;
+ }
+ nir_def *layer = air_get_ssa(c, in->operands[(*arg)++]);
+ if (!layer)
+ return NULL;
+ if (layer->num_components != 1) {
+ air_error(c, "intrinsic: air.%s with a %u-component array index",
+ n->part[0], layer->num_components);
+ return NULL;
+ }
+ layer = is_float ? nir_u2fN(&c->b, layer, coord->bit_size)
+ : nir_u2u32(&c->b, layer);
+
+ nir_def *comps[4];
+ unsigned nc = MIN2(coord->num_components, 3);
+ for (unsigned i = 0; i < nc; i++)
+ comps[i] = nir_channel(&c->b, coord, i);
+ comps[nc] = layer;
+ coord = nir_vec(&c->b, comps, nc + 1);
+ }
+
+ if (coord->num_components != ncoord) {
+ air_error(c, "intrinsic: air.%s with a %u-component coordinate, wanting %u",
+ n->part[0], coord->num_components, ncoord);
+ return NULL;
+ }
+ return coord;
+}
+
static bool
texture_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *n)
{
+ /* THE NULL TEXTURE, named before the parser gets to shrug at it. 26 calls of
+ * air.get_null_texture_2d and 8 of air.get_null_texture_3d in the system
+ * corpus, all of them in QuartzCore's two YCbCr uber shaders -- and
+ * fixed_frag_lph_cph is the module this driver is asked for more than any
+ * other (104 of 105 pipeline refusals in one boot), so this is the last
+ * thing between it and a pipeline.
+ *
+ * WHAT APPLE'S COMPILER DOES WITH IT, read rather than remembered.
+ * AGCLLVMUserObject::replaceTexturesAndSamplers (0x1bbde9754) walks the
+ * module's function list (body 0x1bbde9ed8, latch 0x1bbdea168) and looks
+ * each name up with llvm::StringMapImpl::FindKey (0x1bbde9f04, libLLVM
+ * 0x1bd5ddba0) in getGetNullTextureFunctionSet (0x1bbdf34a0, object
+ * 0x1e29980c0), whose 16 names are "air.get_null_" (0x1bbf4867f) joined to
+ * each suffix of the table at 0x1d9274398 -- texture_buffer_1d, texture_1d,
+ * texture_1d_array, texture_2d, texture_2d_array, texture_3d, texture_cube,
+ * texture_cube_array, texture_2d_ms, texture_2d_ms_array, depth_2d,
+ * depth_2d_array, depth_cube, depth_cube_array, depth_2d_ms,
+ * depth_2d_ms_array. On a hit it takes the function's return type and
+ * replaces every call of it with a CONSTANT:
+ *
+ * with a texture heap ConstantInt::get(intTy, [this+0x84c], false)
+ * (0x1bbde9f70, libLLVM 0x1bce62ec8) then
+ * ConstantExpr::getIntToPtr(.., retTy, false)
+ * (0x1bbde9f7c, libLLVM 0x1bce63694)
+ * without one Constant::getNullValue(retTy)
+ * (0x1bbdea178, libLLVM 0x1bce62bf4)
+ *
+ * and [this+0x84c] is written at 0x1bbe988b4 in
+ * createTextureHeapIndexArray, from the SmallVector's own Size -- so the
+ * heap index is ONE PAST THE LAST real texture.
+ *
+ * SO THERE IS NO RULE HERE FOR WHAT SAMPLING ONE YIELDS. Apple's compiler
+ * substitutes a handle and leaves the answer to the descriptor that handle
+ * names; the behaviour is the hardware's. Folding the sample to a value at
+ * translation time therefore cannot be justified out of this -- it would
+ * need the MSL specification or a measurement on an Apple GPU, and neither
+ * is in reach, so it is NOT the road and that is a reading and not a
+ * shrug.
+ *
+ * What the read does give is the other road's precedent, in Apple's own
+ * structure: where textures are indices into a table, a null texture is a
+ * RESERVED INDEX in that table. This frontend's textures are indices too
+ * (Metal's own `[[texture(n)]]`, resolved while translating), so the
+ * analogue is exact -- a zero-filled 1x1 texture the runtime binds at an
+ * index past the module's last. docs/air.md 3 costs it: for
+ * fixed_frag_lph_cph that index is 15, because it declares 0..14, and
+ * RWA_MAX_TEXTURES is 16, so it fits with nothing to spare. The frontend's
+ * half is small and the runtime's is not this file's.
+ */
+ if (!strncmp(n->part[0], "get_null_texture", 16)) {
+ /* ROAD B, and the frontend's whole half of it: a reserved slot, flagged,
+ * for the runtime to bind a 1x1 texture of zeros at. The sample through
+ * it is an ordinary sample at an ordinary index, which is why nothing
+ * else in this file changes -- and the shape the runtime must make it is
+ * derived from that sample like any other slot's (slot_shapes), so a
+ * `get_null_texture_3d` reaches the driver as a 3D slot without this
+ * function saying so.
+ */
+ int slot = air_null_texture_slot(c);
+ if (slot < 0)
+ return false; /* null_texture_slot has said why */
+ struct air_ptr p = { .class = AIR_PTR_TEXTURE, .binding = (unsigned)slot };
+ air_set(c, &in->base, air_val_ptr(c, p));
+ return true;
+ }
+ if (!strncmp(n->part[0], "is_null_texture", 15)) {
+ /* Exact, and a constant, because the question is one this frontend has
+ * already answered: a texture is resolved to an index while translating,
+ * so "is this the null one" is "is this the reserved slot". Folding it
+ * here is not an optimisation -- an argument that did not resolve to a
+ * texture never reaches this line, because tex_handle refuses it first.
+ */
+ struct air_val *t = air_get(c, in->operands[1]);
+ if (!t)
+ return false;
+ if (t->kind != AIR_VAL_PTR || t->ptr.class != AIR_PTR_TEXTURE)
+ return air_error(c, "intrinsic: air.%s of something this frontend did not "
+ "resolve to a texture", n->part[0]);
+ bool is_null = c->null_texture_index >= 0 &&
+ t->ptr.binding == (unsigned)c->null_texture_index;
+ air_set(c, &in->base, air_val_ssa(c, nir_imm_bool(&c->b, is_null)));
+ return true;
+ }
+
struct air_tex_name t;
if (!parse_tex_name(n->part[0], &t))
return air_error(c, "intrinsic: air.%s (a texture name this cannot take apart)",
n->part[0]);
if (t.is_buffer)
return air_error(c, "intrinsic: air.%s (texture buffers, not yet)", n->part[0]);
- if (t.compare)
- return air_error(c, "intrinsic: air.%s (compare sampling, not yet)", n->part[0]);
+ /* A comparison belongs to a depth texture and to a sample: Apple's header declares
+ * sample_compare and gather_compare on depth2d and depth2d_array and on nothing
+ * else, so a name claiming either of the other two shapes is a misparse and not a
+ * gap. Said by name rather than translated into a plausible instruction.
+ */
+ if (t.compare && !t.depth)
+ return air_error(c, "intrinsic: air.%s (a comparison of a colour texture, which "
+ "Metal declares on depth textures only)", n->part[0]);
+ if (t.compare && !strcmp(t.op, "gather"))
+ return air_error(c, "intrinsic: air.%s (gather_compare: four comparisons in one "
+ "instruction, which nothing measured here attests -- no module "
+ "of the 456 and none of Resident Evil 4's 7 879 calls one -- so "
+ "there is nothing to check an answer against)", n->part[0]);
+ /* Only a sample has a gradient form -- Apple's header has
+ * __metal_sample_*_t_grad and no gather, read or write counterpart -- and a
+ * name that claimed one would be read by the sample branch's rules below,
+ * which is the kind of silent misparse this file refuses instead.
+ */
+ if (t.grad && strcmp(t.op, "sample"))
+ return air_error(c, "intrinsic: air.%s (a gradient form of %s, which Metal has no "
+ "signature for)", n->part[0], t.op);
struct air_ptr tex;
if (!tex_handle(c, in, 1, AIR_PTR_TEXTURE, &tex))
@@ -832,10 +2331,26 @@ texture_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *
}
if (!strcmp(t.op, "write")) {
- nir_def *coord = air_get_ssa(c, in->operands[arg++]);
+ nir_def *coord = tex_coord_and_layer(c, in, &t, n, &arg, ncoord);
+ if (!coord)
+ return false;
+ /* The store's type is the VALUE's, not the call's: a write returns void,
+ * so dest_type above (from the return type) can only say "uint", and ir3
+ * would then emit stib.b.typed.*.u16/u32 for a half or float value --
+ * a typed store converts its source as the type says, so the texture
+ * would receive the bit pattern of 1.0h read as the integer 15360 (that
+ * much is the ISA's rule, not a measurement: hakios-metal's dispatch
+ * refuses a uint-typed kernel over a float format before it runs, and
+ * that refusal is what the old typing hit on the phone with Apple's
+ * MetalFaceMask_Clear, a texture2d<half, write>). With the store typed
+ * by its value, hakios-metal's tex_fill_h.ll landed 2048 of 2048 exact
+ * half bit patterns in an RGBA16Float image on 29 October (hakios
+ * logs/012127-run.log). Signedness still comes from the name (.u/.s). */
+ unsigned value_arg = arg;
nir_def *value = air_get_ssa(c, in->operands[arg++]);
- if (!coord || !value)
+ if (!value)
return false;
+ nir_alu_type src_type = tex_dest_type(c, in->operands[value_arg]->type, n);
uint64_t lod;
if (!tex_flag(c, in, arg, &lod))
return false;
@@ -850,17 +2365,14 @@ texture_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *
nir_imm_int(&c->b, 0), nir_pad_vector(&c->b, value, 4),
nir_imm_int(&c->b, 0),
.image_dim = t.dim, .image_array = t.is_array,
- .src_type = dest_type);
+ .src_type = src_type);
return true;
}
/* sample, gather, read: a texture instruction, or an image load */
- nir_def *coord = air_get_ssa(c, in->operands[arg++]);
+ nir_def *coord = tex_coord_and_layer(c, in, &t, n, &arg, ncoord);
if (!coord)
return false;
- if (coord->num_components != ncoord)
- return air_error(c, "intrinsic: air.%s with a %u-component coordinate, wanting %u",
- n->part[0], coord->num_components, ncoord);
if (!strcmp(t.op, "read") && tex.is_image) {
uint64_t extra;
@@ -876,59 +2388,158 @@ texture_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *
return true;
}
- nir_tex_src srcs[6];
+ /* The value a depth texel is compared against, which Metal passes straight after
+ * the coordinate AND AFTER THE ARRAY INDEX -- __metal_sample_compare_depth_2d_t
+ * takes (tex, samp, coord, compare_value, ...) and the _2d_array form takes (tex,
+ * samp, coord, array, compare_value, ...), so the layer comes first in exactly the
+ * place tex_coord_and_layer has already consumed it. Read here and not in the
+ * branches below because it is before the offset pair in the plain form and before
+ * the gradients in the _grad one: one position, not two.
+ *
+ * At the AIR level there is also the aspect selector the depth branch above ate, as
+ * there is in air.sample_depth_2d -- Apple's header has no such argument and the
+ * declaration in texturedFragment2DDepth does:
+ * { float, i8 } air.sample_depth_2d.f32(ptr as(1), ptr as(2), i32, <2 x float>,
+ * i1, <2 x i32>, i1, float, float, i32)
+ * which is where the `i32` between the sampler and the coordinate comes from, and
+ * the only reason the compare form's shape could be read off the header at all.
+ */
+ nir_def *compare = NULL;
+ if (t.compare) {
+ if (in->num_operands <= arg)
+ return air_error(c, "intrinsic: air.%s without the value its signature compares "
+ "against", n->part[0]);
+ if (!(compare = air_get_ssa(c, in->operands[arg++])))
+ return false;
+ if (compare->num_components != 1)
+ return air_error(c, "intrinsic: air.%s compares against %u components, and a "
+ "depth comparison takes one", n->part[0],
+ compare->num_components);
+ /* The texel and the value compared with it go into one register tuple on the
+ * Adreno -- ir3 puts the comparison straight after the coordinates in src0 --
+ * so a 16-bit comparison value beside 32-bit coordinates is a half register in a
+ * full tuple. Metal's own signature is `float`, so this never fires on anything
+ * Apple's compiler emits; it is here so that if something does, it is widened
+ * and not mixed.
+ */
+ if (compare->bit_size != coord->bit_size)
+ compare = nir_f2fN(&c->b, compare, coord->bit_size);
+ }
+
+ /* Seven and not six: a compare sample in the gradient form carries a coordinate, a
+ * comparison value, an offset, two derivatives and a minimum level at once, which is
+ * exactly six, and the next source anyone adds should not be the one that overflows
+ * this silently.
+ */
+ nir_tex_src srcs[7];
unsigned nsrc = 0;
srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_coord,
- .src = nir_src_for_ssa(
- air_type_is_float(air_type_scalar(
- in->operands[arg - 1]->type))
- ? coord : nir_u2u32(&c->b, coord)) };
+ .src = nir_src_for_ssa(coord) };
+ if (compare)
+ srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_comparator,
+ .src = nir_src_for_ssa(compare) };
nir_texop op;
if (!strcmp(t.op, "sample")) {
- uint64_t has_offset, is_level;
- if (!tex_flag(c, in, arg, &has_offset))
- return false;
- arg++;
- nir_def *offset = air_get_ssa(c, in->operands[arg++]);
- if (!offset)
- return false;
- if (has_offset && !all_zero(offset))
+ /* The offset pair, for the dimensionalities that have one. A cube does
+ * not, and reading one there consumed the level flag and the level and
+ * left the minimum level to be read as the flags word -- which is the
+ * refusal 621 of Resident Evil 4's modules used to earn, "argument 5 is
+ * not a constant", from a float that indeed is not an integer constant.
+ *
+ * In the gradient form this pair comes AFTER the gradients, so the
+ * reading is a small state machine over the signature rather than one
+ * fixed order: see the table at the head of this section.
+ */
+ uint64_t has_offset = 0, is_level = 0;
+ nir_def *offset = NULL, *ddx = NULL, *ddy = NULL, *min_lod = NULL, *level = NULL;
+
+ if (t.grad) {
+ ddx = air_get_ssa(c, in->operands[arg++]);
+ ddy = air_get_ssa(c, in->operands[arg++]);
+ min_lod = air_get_ssa(c, in->operands[arg++]);
+ if (!ddx || !ddy || !min_lod)
+ return false;
+ if (!t.no_offset) {
+ if (!tex_flag(c, in, arg, &has_offset))
+ return false;
+ arg++;
+ if (!(offset = air_get_ssa(c, in->operands[arg++])))
+ return false;
+ }
+ } else {
+ if (!t.no_offset) {
+ if (!tex_flag(c, in, arg, &has_offset))
+ return false;
+ arg++;
+ if (!(offset = air_get_ssa(c, in->operands[arg++])))
+ return false;
+ }
+ if (!tex_flag(c, in, arg, &is_level))
+ return false;
+ arg++;
+ level = air_get_ssa(c, in->operands[arg++]);
+ min_lod = air_get_ssa(c, in->operands[arg++]);
+ if (!level || !min_lod)
+ return false;
+ }
+
+ if (offset && has_offset && !all_zero(offset))
srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_offset,
.src = nir_src_for_ssa(nir_u2u32(&c->b, offset)) };
- if (!tex_flag(c, in, arg, &is_level))
- return false;
- arg++;
- nir_def *level = air_get_ssa(c, in->operands[arg++]);
- nir_def *min_lod = air_get_ssa(c, in->operands[arg++]);
- if (!level || !min_lod)
- return false;
- bool level_is_zero = nir_src_is_const(nir_src_for_ssa(level)) &&
- nir_src_as_float(nir_src_for_ssa(level)) == 0.0;
- if (is_level) {
- op = nir_texop_txl;
- srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_lod,
- .src = nir_src_for_ssa(level) };
- } else if (!level_is_zero) {
- op = nir_texop_txb;
- srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_bias,
- .src = nir_src_for_ssa(level) };
+
+ if (t.grad) {
+ /* An explicit pair of derivatives of the coordinate, which is what
+ * Metal's sample(..., gradient2d(dPdx, dPdy)) is for: a shader that
+ * knows better than the quad's own differences -- a ray-marched or
+ * reprojected coordinate, or one sampled inside a branch the quad
+ * does not take together. NIR has the operation (txd) and so does
+ * the Adreno; ir3 emits it and softpipe's interpreter has TXD, so
+ * unlike most of this family it can be both compiled and run here.
+ *
+ * The gradients carry the coordinate's components WITHOUT the array
+ * layer, which is what NIR asks for too (nir_tex_instr_src_size:
+ * coord_components minus is_array for ddx and ddy) and what AIR
+ * passes -- <2 x float> for a 2d_array whose coordinate is also
+ * <2 x float>. So nothing is added or dropped here, and that
+ * agreement is the reason this needs no reshaping.
+ */
+ op = nir_texop_txd;
+ srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_ddx,
+ .src = nir_src_for_ssa(ddx) };
+ srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_ddy,
+ .src = nir_src_for_ssa(ddy) };
} else {
- op = nir_texop_tex;
+ bool level_is_zero = nir_src_is_const(nir_src_for_ssa(level)) &&
+ nir_src_as_float(nir_src_for_ssa(level)) == 0.0;
+ if (is_level) {
+ op = nir_texop_txl;
+ srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_lod,
+ .src = nir_src_for_ssa(level) };
+ } else if (!level_is_zero) {
+ op = nir_texop_txb;
+ srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_bias,
+ .src = nir_src_for_ssa(level) };
+ } else {
+ op = nir_texop_tex;
+ }
}
+
if (!(nir_src_is_const(nir_src_for_ssa(min_lod)) &&
nir_src_as_float(nir_src_for_ssa(min_lod)) == 0.0))
srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_min_lod,
.src = nir_src_for_ssa(min_lod) };
} else if (!strcmp(t.op, "gather")) {
- uint64_t has_offset;
- if (!tex_flag(c, in, arg, &has_offset))
- return false;
- arg++;
- nir_def *offset = air_get_ssa(c, in->operands[arg++]);
- if (!offset)
- return false;
- if (has_offset && !all_zero(offset))
+ uint64_t has_offset = 0;
+ nir_def *offset = NULL;
+ if (!t.no_offset) {
+ if (!tex_flag(c, in, arg, &has_offset))
+ return false;
+ arg++;
+ if (!(offset = air_get_ssa(c, in->operands[arg++])))
+ return false;
+ }
+ if (offset && has_offset && !all_zero(offset))
srcs[nsrc++] = (nir_tex_src){ .src_type = nir_tex_src_offset,
.src = nir_src_for_ssa(nir_u2u32(&c->b, offset)) };
op = nir_texop_tg4;
@@ -952,7 +2563,15 @@ texture_op(struct air_ctx *c, const struct air_inst *in, const struct air_name *
tex_instr->op = op;
tex_instr->sampler_dim = t.dim;
tex_instr->is_array = t.is_array;
- tex_instr->is_shadow = false;
+ /* is_new_style_shadow because Metal's sample_compare returns ONE value, which the
+ * declaration says too -- air.sample_compare_depth_2d.f32 returns { float, i8 }, not
+ * { <4 x float>, i8 } the way a colour sample does. NIR's old-style shadow
+ * replicates the comparison into four components and would leave the shader reading
+ * component 0 of four where the signature has one, which is the same answer by
+ * accident; this says what the operation is.
+ */
+ tex_instr->is_shadow = t.compare;
+ tex_instr->is_new_style_shadow = t.compare;
tex_instr->dest_type = dest_type;
tex_instr->coord_components = ncoord;
tex_instr->texture_index = tex.binding;
@@ -1005,12 +2624,26 @@ air_translate_call(struct air_ctx *c, const struct air_inst *in, const char *nam
if (!split_name(name, &n))
return air_error(c, "intrinsic: %s (a name this cannot take apart)", name);
+ if (!strcmp(n.part[0], "is_function_constant_defined"))
+ return fc_defined_op(c, in);
if (n.num >= 2 && !strcmp(n.part[0], "wg") && !strcmp(n.part[1], "barrier"))
- return barrier_op(c, in);
+ return barrier_op(c, in, false);
+ if (n.num >= 2 && !strcmp(n.part[0], "simdgroup") && !strcmp(n.part[1], "barrier"))
+ return barrier_op(c, in, true);
if (!strcmp(n.part[0], "atomic"))
return atomic_op(c, in, &n);
+ if (!strcmp(n.part[0], "discard_fragment"))
+ return discard_op(c, in);
bool handled = false;
+ /* before math_op, which resolves every argument as a number: these take a
+ * lane number or nothing at all, and air.simd_sum falls through to it
+ */
+ if (!simd_op(c, in, &n, &handled))
+ return false;
+ if (handled)
+ return true;
+
if (!math_op(c, in, &n, &handled))
return false;
if (handled)
@@ -1022,5 +2655,62 @@ air_translate_call(struct air_ctx *c, const struct air_inst *in, const char *nam
strstr(op, "_lod_"))
return texture_op(c, in, &n);
+ /* The rest of Apple's pack family, named as absent rather than as undecided.
+ *
+ * air.pack.unorm.rgb565 is translated above, off AGXCompilerCore's lowering.
+ * The same reading gave the whole of that binary's `formats` table, so the
+ * field layout of air.pack.{s,u}norm.rgb10a2, air.pack.unorm.rgb10 and the
+ * 4444/1555/5551 rows the driver's own blit shaders use is known too -- but
+ * no module of either corpus calls one. The rest take a different path in
+ * that binary and it has not been read: the {s,u}norm NxM family that
+ * initializePackUnpackMap builds out of its own component-count and width
+ * tables (air.pack.unorm4x8.v4f32 and its relatives) goes to buildPackNorm,
+ * and air.pack.unorm4x8.srgb, air.pack.unorm.rg11b10f and
+ * air.pack.unorm.rgb9e5 go to buildPackSRGB, buildPack111110 and
+ * buildPack9995. So this is "not implemented", with the places to read
+ * named, and not "cannot be decided".
+ */
+ /* air.pack.{u,s}norm{4x8,2x16}.v<N>f{16,32} and air.unpack the other way:
+ * Metal's pack_float_to_unorm4x8 and its kin, which are GLSL's packUnorm4x8
+ * and its kin -- clamp, scale, round to nearest, component 0 in the low
+ * bits -- and so NIR's pack_unorm_4x8 and the rest, over 32-bit floats (a
+ * half vector is widened first, exactly). VideoToolbox's scaler
+ * (VTMTSComputeFunction, the live wallpaper's) writes its pixels so.
+ */
+ if ((!strcmp(op, "pack") || !strcmp(op, "unpack")) && n.num >= 2) {
+ const char *f = n.part[1];
+ const bool un = !strcmp(f, "unorm4x8") || !strcmp(f, "unorm2x16");
+ const bool sn = !strcmp(f, "snorm4x8") || !strcmp(f, "snorm2x16");
+ if (un || sn) {
+ const bool x8 = strstr(f, "4x8") != NULL;
+ if (!strcmp(op, "pack")) {
+ nir_def *v = in->num_operands > 1 ? air_get_ssa(c, in->operands[1]) : NULL;
+ if (!v)
+ return false;
+ if (v->bit_size != 32)
+ v = nir_f2f32(&c->b, v);
+ nir_def *r = x8 ? (un ? nir_pack_unorm_4x8(&c->b, v) : nir_pack_snorm_4x8(&c->b, v))
+ : (un ? nir_pack_unorm_2x16(&c->b, v) : nir_pack_snorm_2x16(&c->b, v));
+ air_set(c, &in->base, air_val_ssa(c, r));
+ return true;
+ }
+ nir_def *w = in->num_operands > 1 ? air_get_ssa(c, in->operands[1]) : NULL;
+ if (!w)
+ return false;
+ nir_def *r = x8 ? (un ? nir_unpack_unorm_4x8(&c->b, w) : nir_unpack_snorm_4x8(&c->b, w))
+ : (un ? nir_unpack_unorm_2x16(&c->b, w) : nir_unpack_snorm_2x16(&c->b, w));
+ const unsigned want = air_nir_bit_size(c, in->base.type);
+ if (want == 16)
+ r = nir_f2f16(&c->b, r);
+ air_set(c, &in->base, air_val_ssa(c, r));
+ return true;
+ }
+ }
+
+ if (!strcmp(op, "pack"))
+ return air_error(c, "intrinsic: %s -- of Apple's pack family only "
+ "unorm.rgb565 is translated, because only it occurs; "
+ "the rest are unread lowerings, not undecided ones", name);
+
return air_error(c, "intrinsic: %s", name);
}
diff --git a/tools/mesa/air/air_layout.c b/tools/mesa/air/air_layout.c
index 313d498..60e194c 100644
--- a/tools/mesa/air/air_layout.c
+++ b/tools/mesa/air/air_layout.c
@@ -5,7 +5,7 @@
/*
* Sizes and alignments from the module's datalayout string, by LLVM's rules
- * (LangRef, "Data Layout"). Every AIR module of Darwin userland 16.5 has the same layout,
+ * (LangRef, "Data Layout"). Every AIR module of iOS 16.5 has the same layout,
* but it is read rather than assumed: GEP offsets are computed from it, and
* a wrong offset is a wrong read.
*/
diff --git a/tools/mesa/air/air_metallib.c b/tools/mesa/air/air_metallib.c
index 05020b6..f94492e 100644
--- a/tools/mesa/air/air_metallib.c
+++ b/tools/mesa/air/air_metallib.c
@@ -6,7 +6,7 @@
/*
* The MTLB container of a .metallib: a header, a function list of tagged
* fields, and one LLVM module per function. Worked out against the files of
- * Darwin userland 16.5 (docs/metal.md, section 9.1; tools/metallib.py is the same in
+ * iOS 16.5 (docs/metal.md, section 9.1; tools/metallib.py is the same in
* Python).
*/
diff --git a/tools/mesa/air/air_module.c b/tools/mesa/air/air_module.c
index d908885..534c80a 100644
--- a/tools/mesa/air/air_module.c
+++ b/tools/mesa/air/air_module.c
@@ -851,12 +851,33 @@ read_metadata_kind_block(struct reader *r)
/* -- module records ----------------------------------------------------- */
+/* Module version 1 (LLVM before 5) has no string table: a GLOBALVAR or FUNCTION
+ * record is the version 2 one without its first two operands (the name's place in
+ * the table), and the names come from the module's own symbol table instead
+ * (read_module_vst). Apple's compiler service still writes version 1 for some of
+ * the MSL it compiles at run time (sokol-samples' debugtext and drawex), so such a
+ * record is read as the version 2 one with a name of no length, named later.
+ */
+static const struct air_bs_entry *
+as_version2(struct reader *r, const struct air_bs_entry *e, struct air_bs_entry *copy)
+{
+ if (r->m->version >= 2)
+ return e;
+ uint64_t *ops = ralloc_array(r->m, uint64_t, e->num_ops + 2);
+ ops[0] = ops[1] = 0;
+ memcpy(ops + 2, e->ops, e->num_ops * sizeof(uint64_t));
+ *copy = *e;
+ copy->ops = ops;
+ copy->num_ops = e->num_ops + 2;
+ return copy;
+}
+
static bool
-read_globalvar(struct reader *r, const struct air_bs_entry *e)
+read_globalvar(struct reader *r, const struct air_bs_entry *e1)
{
struct air_module *m = r->m;
- if (r->m->version < 2)
- return fail(r, "module version %u: only bitcode with a string table is read", m->version);
+ struct air_bs_entry copy;
+ const struct air_bs_entry *e = as_version2(r, e1, &copy);
if (e->num_ops < 8)
return fail(r, "GLOBALVAR too short");
@@ -901,11 +922,11 @@ read_globalvar(struct reader *r, const struct air_bs_entry *e)
}
static bool
-read_function_record(struct reader *r, const struct air_bs_entry *e)
+read_function_record(struct reader *r, const struct air_bs_entry *e1)
{
struct air_module *m = r->m;
- if (m->version < 2)
- return fail(r, "module version %u: only bitcode with a string table is read", m->version);
+ struct air_bs_entry copy;
+ const struct air_bs_entry *e = as_version2(r, e1, &copy);
if (e->num_ops < 8)
return fail(r, "FUNCTION too short");
@@ -1289,6 +1310,45 @@ read_instruction(struct reader *r, const struct air_bs_entry *e, struct air_bloc
return fail(r, "br of %u operands", e->num_ops);
}
break;
+ case FUNC_CODE_INST_SWITCH: {
+ /* [ type of the condition, condition, default block,
+ * (case constant, block)* ]
+ *
+ * The condition is a RELATIVE value number and each case constant is an
+ * ABSOLUTE one, because that is what the writer does -- pushValue for
+ * the condition, getValueID for each case value. Reading them the same
+ * way gives the wrong constant for every case in any function that has
+ * more than a handful of values, which is every function here.
+ *
+ * The blocks are block numbers, as br's are, not value numbers.
+ */
+ if (e->num_ops < 3 || (e->num_ops - 3) % 2)
+ return fail(r, "a switch of %u operands", e->num_ops);
+ unsigned ty = take(&rc);
+ /* LLVM 3.0 wrote switches over ranges, with the case count and a bit
+ * width packed into the first operand under this magic instead of a type
+ * id. Nothing Apple's compiler emits uses it, and reading it as a type
+ * would silently take the wrong path, so it is named and refused.
+ */
+ if ((ty >> 16) == 0x4B5)
+ return fail(r, "a switch in LLVM 3.0's range form (SWITCH_INST_MAGIC)");
+ unsigned cond = take_value(r, &rc);
+ unsigned deflt = take(&rc);
+ unsigned n = (e->num_ops - 3) / 2;
+ in = new_inst(r, AIR_OP_SWITCH, NULL, 1 + n);
+ in->num_targets = 1 + n;
+ in->target_ids = ralloc_array(m, unsigned, 1 + n);
+ /* target 0 and operand 0 are the default edge and the condition; the
+ * cases follow, so case k is operand k+1 and target k+1
+ */
+ in->target_ids[0] = deflt;
+ in->operand_ids[0] = cond;
+ for (unsigned k = 0; k < n; k++) {
+ in->operand_ids[1 + k] = take(&rc);
+ in->target_ids[1 + k] = take(&rc);
+ }
+ break;
+ }
case FUNC_CODE_INST_UNREACHABLE:
in = new_inst(r, AIR_OP_UNREACHABLE, NULL, 0);
break;
@@ -1651,6 +1711,29 @@ read_function_block(struct reader *r)
/* -- the module block --------------------------------------------------- */
+/* A version 1 module's symbol table: ENTRY [value id, name...] for a global or a
+ * declaration, FNENTRY [value id, offset of its body, name...] for a function with
+ * a body. Version 2 keeps the same names in the string table and has no use for it.
+ */
+static bool
+read_module_vst(struct reader *r)
+{
+ for (;;) {
+ struct air_bs_entry e = air_bs_next(&r->bs);
+ if (e.kind == AIR_BS_END_BLOCK)
+ return true;
+ if (e.kind != AIR_BS_RECORD)
+ return check_bs(r) && fail(r, "module symbol table ends early");
+ unsigned skip = e.id == VST_CODE_ENTRY ? 1 : e.id == VST_CODE_FNENTRY ? 2 : 0;
+ if (!skip || e.num_ops < skip)
+ continue;
+ struct air_value *v = value_by_id(r, e.ops[0]);
+ if (!v)
+ return false;
+ v->name = air_bs_ops_to_string(r->m, e.ops + skip, e.num_ops - skip);
+ }
+}
+
static bool
read_module_block(struct reader *r)
{
@@ -1692,9 +1775,18 @@ read_module_block(struct reader *r)
air_bs_enter(&r->bs, e.id);
ok = ok && read_function_block(r);
break;
+ case VALUE_SYMTAB_BLOCK_ID:
+ /* a version 1 module's names (read_module_vst); version 2's are the string table's */
+ if (m->version < 2) {
+ air_bs_enter(&r->bs, e.id);
+ ok = read_module_vst(r);
+ } else {
+ ok = air_bs_skip_block(&r->bs);
+ }
+ break;
default:
- /* attributes, symbol table, operand bundle tags, sync scopes,
- * use lists: nothing the translation depends on
+ /* attributes, operand bundle tags, sync scopes, use lists: nothing
+ * the translation depends on
*/
ok = air_bs_skip_block(&r->bs);
break;
@@ -1870,12 +1962,14 @@ air_module_read(void *mem_ctx, const void *data, size_t size, const char **error
goto out;
}
- /* the string table comes after the module, so names come last */
- for (unsigned i = 0; i < m->num_globals; i++)
+ /* the string table comes after the module, so names come last (a version 1
+ * module named them from its symbol table already, read_module_vst)
+ */
+ for (unsigned i = 0; m->version >= 2 && i < m->num_globals; i++)
if (!(m->globals[i]->base.name =
strtab_name(&r, m->globals[i]->strtab_offset, m->globals[i]->strtab_size)))
goto out;
- for (unsigned i = 0; i < m->num_functions; i++)
+ for (unsigned i = 0; m->version >= 2 && i < m->num_functions; i++)
if (!(m->functions[i]->base.name =
strtab_name(&r, m->functions[i]->strtab_offset, m->functions[i]->strtab_size)))
goto out;
diff --git a/tools/mesa/air/air_module.h b/tools/mesa/air/air_module.h
index b0aede9..7ea7c95 100644
--- a/tools/mesa/air/air_module.h
+++ b/tools/mesa/air/air_module.h
@@ -6,7 +6,7 @@
/*
* The part of LLVM IR that AIR uses, read from bitcode into plain C
* structures. It is a reader, not an IR library: nothing here is modified
- * after reading, and only what the 456 modules of Darwin 16.5's system
+ * after reading, and only what the 456 modules of iOS 16.5's system
* libraries contain (plus a little margin) is understood. Anything else is an
* error that names the record, never a silent skip, because a skipped
* instruction is a wrong shader.
diff --git a/tools/mesa/air/air_print.c b/tools/mesa/air/air_print.c
index 5654a10..bc70239 100644
--- a/tools/mesa/air/air_print.c
+++ b/tools/mesa/air/air_print.c
@@ -637,6 +637,20 @@ print_inst(struct printer *p, const struct air_inst *in)
print_block_ref(p, in->targets[1]);
}
break;
+ case AIR_OP_SWITCH:
+ fputs("switch ", fp);
+ print_typed(p, in->operands[0]);
+ fputs(", ", fp);
+ print_block_ref(p, in->targets[0]);
+ fputs(" [", fp);
+ for (unsigned k = 1; k < in->num_targets; k++) {
+ fputs("\n ", fp);
+ print_typed(p, in->operands[k]);
+ fputs(", ", fp);
+ print_block_ref(p, in->targets[k]);
+ }
+ fputs("\n ]", fp);
+ break;
case AIR_OP_UNREACHABLE:
fputs("unreachable", fp);
break;
diff --git a/tools/mesa/air/air_private.h b/tools/mesa/air/air_private.h
index e8760dd..1ebb930 100644
--- a/tools/mesa/air/air_private.h
+++ b/tools/mesa/air/air_private.h
@@ -29,6 +29,32 @@ enum air_ptr_class {
AIR_PTR_FUNCTION_CONSTANT, /* a function constant: .binding is its index */
AIR_PTR_TEXTURE, /* a texture handle; offset unused */
AIR_PTR_SAMPLER, /* a sampler handle */
+
+ /* An ARGUMENT BUFFER (`air.indirect_buffer`): .binding is the set's number
+ * in air_ctx.argbufs and .offset the byte offset into it. Nothing is ever
+ * loaded from such a pointer as a value -- a load through it RESOLVES to
+ * the member that byte offset names (argbuf_load), which is the whole of
+ * why argument buffers need no addresses here.
+ */
+ AIR_PTR_ARGBUF,
+
+ /* A LOCAL, OR A READ-ONLY GLOBAL, THAT HOLDS A POINTER -- kept as a
+ * translation-time slot and not as memory, because a pointer in this
+ * frontend is a resolution and has no bytes to put anywhere. .binding is
+ * the slot's number in air_ctx.slots, and a slot holds a whole air_val, so
+ * what comes back out of one may itself be a texture or a sampler.
+ *
+ * Only a whole load or store at offset 0 reaches a slot. That is not a
+ * simplification: the two shapes this exists for are `alloca ptr
+ * addrspace(2)` followed by a store of a sampler argument, and
+ * `constant %"struct.metal::sampler" { ptr addrspace(2) @__air_sampler_state }`
+ * read through a getelementptr whose every index is zero -- and offset 0 of
+ * a struct is offset 0 of its first member, transitively, which is the only
+ * byte arithmetic a tree of values can answer. Anything else is refused by
+ * name rather than resolved to whichever member the offset happened to land
+ * on (air_to_nir.c, slot_at_zero).
+ */
+ AIR_PTR_SLOT,
};
struct air_ptr {
@@ -44,6 +70,21 @@ enum air_val_kind {
AIR_VAL_SSA, /* a scalar or vector */
AIR_VAL_AGG, /* a struct or array, as a tree of values */
AIR_VAL_PTR,
+
+ /* A POINTER as a value rather than a resolution. air_ptr is *(where, byte
+ * offset)* decided while translating, which is what lets this frontend emit
+ * load_ubo at Metal's own index with no descriptor indirection; a pointer
+ * CHOSEN at run time has no such resolution, so the choice itself is kept
+ * and made where the pointer is used. A pointer STORED to memory is the
+ * other half of the same problem and has no form here at all -- see
+ * docs/air.md section 3 for what that costs.
+ */
+
+ /* `select i1 %c, ptr %a, ptr %b`: .def is the condition and .elems[0..1]
+ * are the two pointers. A LOAD through it loads BOTH and selects the
+ * values, which is the only form NIR has; that is why it leaves a note.
+ */
+ AIR_VAL_PTR_SELECT,
};
struct air_val {
@@ -54,6 +95,43 @@ struct air_val {
struct air_val **elems;
};
+/* One member of an argument buffer, as `air.struct_type_info` declares it and
+ * the module's own struct type confirms it.
+ *
+ * Keyed on `offset` and not on the member number, which is what makes the
+ * three shapes SPIRV-Cross emits one case: a buffer member is the pointer
+ * itself, while a texture or sampler member is a one-field
+ * `metal::texture2d` / `metal::sampler` wrapper whose field sits at byte 0 of
+ * it -- so the extra `getelementptr` into the wrapper adds nothing to the
+ * offset and resolves by the same lookup (docs/air.md section 8.2).
+ */
+struct air_argbuf_member {
+ uint64_t offset; /* bytes from the start of the set: the KEY */
+ enum air_resource_kind kind;
+ unsigned metal_id; /* the member's own air.location_index: Metal's [[id(n)]] */
+ unsigned index; /* the flat per-kind index this frontend allocated */
+
+ /* A buffer member: 1 device, 2 constant. The metadata does not say -- not
+ * one of the 82 048 members of Resident Evil 4's 7 857 argument buffers
+ * carries `air.address_space` -- so this comes from the module's own struct
+ * type, and every load through the member checks the pointer it loads
+ * against it (argbuf_load). 0 for a texture or a sampler.
+ */
+ unsigned addrspace;
+ bool read, write;
+ const char *name, *type_name;
+ const struct air_md *node; /* the member's own air.indirect_argument descriptor */
+};
+
+struct air_argbuf {
+ unsigned metal_index; /* the SET's own Metal [[buffer(n)]] */
+ unsigned size; /* air.buffer_size: eight bytes per member */
+ unsigned param; /* the entry point's parameter the set arrives as */
+ const char *name, *type_name;
+ unsigned num_members;
+ struct air_argbuf_member *members;
+};
+
struct air_ctx {
void *mem_ctx;
const struct air_module *m;
@@ -72,6 +150,19 @@ struct air_ctx {
struct hash_table *phi_cursors;
/* phi -> nir register decl */
struct hash_table *phi_regs;
+ /* Where this function's constants are built: the top of its entry block,
+ * which dominates every use of them. air_get caches a constant the first
+ * time it is asked for, so the def has to be somewhere every later block
+ * can see; built at the cursor of the first use, a constant shared by a
+ * branch and the block after it is a non-dominating use, and nir_repair_ssa
+ * turns that into undef on the path that misses the branch.
+ */
+ nir_cursor const_cursor;
+ /* set while a constant is being built there, so that the elements of an
+ * aggregate constant -- which come through air_get too -- keep appending
+ * after it instead of each restoring the cursor to where the aggregate
+ * itself will go, which put a vec before the components it reads */
+ bool in_const;
/* A call of another function in the module becomes a nir_call, and
* nir_inline_functions folds it in afterwards. These two carry the state
@@ -85,6 +176,19 @@ struct air_ctx {
struct hash_table *fns;
nir_deref_instr *ret_deref;
+ /* ... and, for a callee whose return type is a STRUCT, one out-parameter per
+ * member instead of one for the whole value: a NIR function parameter
+ * carries a number, and a struct is a tree of values in this translator and
+ * never a NIR value at all. ret_fields is indexed by the AIR member number
+ * and is NULL for a member that is not a number -- the `[8 x i8]` tail
+ * padding that Metal's own vertex-output structs carry, which takes no
+ * parameter because nothing may read or write it. num_ret_fields is
+ * non-zero only while such a callee is being translated, and that is how
+ * emit_return tells this case from the scalar one.
+ */
+ nir_deref_instr **ret_fields;
+ unsigned num_ret_fields;
+
/* What each module global resolves to, decided once (plan_globals) and
* turned into a value in whichever function needs it (materialise_globals).
* The two are separate because the deciding half has side effects that must
@@ -101,12 +205,53 @@ struct air_ctx {
unsigned offset; /* bytes, when has_offset */
bool has_offset;
bool skip; /* llvm.* lists, and anything that failed */
+
+ /* A PROGRAM-SCOPE VARIABLE THAT IS A VALUE, not memory. Set by
+ * plan_global_folding for a scalar integer global that the module only
+ * ever loads and stores WHOLE, whose single store is in its
+ * air.static_init constructor, and whose address never escapes. `known`
+ * and `bits` are then filled when that store is translated and the value
+ * turns out to be a constant -- which it is when it came from a function
+ * constant -- and every later load answers with the constant instead of
+ * a load_scratch. The scratch slot is still allocated and still written:
+ * the fold is an answer on top of memory that is there, not a promise
+ * that nothing reads it (air_to_nir.c, AIR_OP_LOAD).
+ */
+ bool fold;
+ bool known;
+ uint64_t bits;
+ unsigned bit_size;
} *global_plan;
+ /* The translation-time slots: locals and read-only globals that hold a
+ * POINTER. Grown as allocas are met; the read-only globals reserve theirs
+ * in plan_globals, before any body is translated, and materialise_globals
+ * fills each one from the global's initialiser in whatever builder is
+ * asking -- the same rule the offsets follow, and for the same reason.
+ */
+ struct air_slot {
+ struct air_val *val;
+ const struct air_type *type; /* what a whole load of it yields */
+ const struct air_value *init; /* a read-only global's initialiser, or NULL */
+ } *slots;
+ unsigned num_slots;
+
+ /* The reserved texture slot the null texture lives in, or -1 before anything
+ * has asked for one. Allocated once per shader and shared by every
+ * air.get_null_texture_* call in it, because a module with fifteen of them
+ * (fixed_frag_lph_cph) wants one zero texture and not fifteen.
+ */
+ int null_texture_index;
+
/* what the entry point's parameters are */
unsigned num_params;
struct air_val **params;
+ /* the argument buffers the entry point declares, in the order its input
+ * metadata lists them, which is the order their members are numbered in */
+ unsigned num_argbufs;
+ struct air_argbuf *argbufs;
+
/* outputs, in the order of the returned struct's fields */
unsigned num_outputs;
struct {
@@ -117,6 +262,7 @@ struct air_ctx {
unsigned scratch_size;
unsigned shared_size;
+ unsigned tg_param_plus1; /* the [[threadgroup(0)]] argument's parameter + 1; 0 none */
/* module-scope variables, by their air_global */
struct hash_table *globals;
@@ -139,7 +285,24 @@ nir_def *air_get_ssa(struct air_ctx *c, const struct air_value *v);
unsigned air_nir_bit_size(struct air_ctx *c, const struct air_type *t);
unsigned air_nir_components(const struct air_type *t);
+/* Records that the shader was translated with a caveat: what it asked for and
+ * what was done instead (air_shader_info.notes). Not an error -- translation
+ * continues -- and not a warning about our own code: it is for a thing the
+ * hardware or this frontend cannot do exactly, where doing something else is
+ * still correct. Refusing is air_error; passing over it in silence is the one
+ * option this is here to remove.
+ */
+void air_note(struct air_ctx *c, const char *fmt, ...) PRINTFLIKE(2, 3);
+
/* memory, by the pointer's class */
+/* The reserved texture slot a null texture lives in, allocated on the first
+ * air.get_null_texture_* of a shader and shared by the rest of them. Negative
+ * when it does not fit, with the reason already in c->error. Across the file
+ * boundary because the interface half decides it and the INTRINSIC half asks
+ * (air_intrinsics.c).
+ */
+int air_null_texture_slot(struct air_ctx *c);
+
nir_def *air_load_ptr(struct air_ctx *c, struct air_ptr ptr, const struct air_type *type,
unsigned align);
bool air_store_ptr(struct air_ctx *c, struct air_ptr ptr, nir_def *value, unsigned align);
diff --git a/tools/mesa/air/air_run.c b/tools/mesa/air/air_run.c
index 3007534..46af530 100644
--- a/tools/mesa/air/air_run.c
+++ b/tools/mesa/air/air_run.c
@@ -19,9 +19,12 @@
* (osmesa_create_gpu_screen), and nothing else about it changes.
*
* usage: air_run FILE[.metallib] [FUNCTION]
+ * air_run --library FILE.metallib FUNCTION one air.visible function, wrapped
+ * air_run --stitch "f(in0); g(n0)" FILE.metallib a graph of them, stitched
*/
#include <stdio.h>
+#include <ctype.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
@@ -55,7 +58,7 @@
#include <mach-o/dyld.h>
#include <signal.h>
/* <sys/ucontext.h>, not <ucontext.h>: the latter refuses to be included
- * without _XOPEN_SOURCE on the Darwin SDK, and userland/src/inject.c takes the
+ * without _XOPEN_SOURCE on the iOS SDK, and userland/src/inject.c takes the
* same route for the same reason.
*/
#include <sys/ucontext.h>
@@ -104,7 +107,7 @@ texel_value(unsigned slot, unsigned x, unsigned y, unsigned z, float out[4])
* print the fault before dying. These two functions are userland/src/inject.c's
* say_the_fault()/watch_for_faults() (branch sileo2, 6683521), with the slide
* taken from this program's own image rather than from the shared cache, since
- * the pc of interest is in air_run or in libOSMesa and not in Darwin.
+ * the pc of interest is in air_run or in libOSMesa and not in iOS.
*/
static void
say_the_fault(int signal_number, siginfo_t *info, void *context_in)
@@ -156,6 +159,34 @@ static void watch_for_faults(void) {}
/* set by --library: the fill below is then floats, for the reason given there */
static bool library_fill = false;
+/* set by --stitch: a function-stitching graph, as air_stitch.c reads it */
+static const char *stitch_text = NULL;
+
+/* set by --zero: every buffer the shader reads is zeroed instead of patterned.
+ *
+ * For the 14 kernels whose loops leave on a count or a sentinel read out of a
+ * buffer, the pattern is not an input at all -- the loop never ends and the GPU
+ * runs forever (docs/air.md section 5, tools/air/kstruct.py). Zeros are a
+ * degenerate input but a *defined* one: a count of zero means the loop runs no
+ * times, so the kernel reaches its end and the hardware answers. What that
+ * proves is that the translated shader runs on the Adreno without hanging; it
+ * proves nothing about what the kernel computes, and the harness says so rather
+ * than printing the zeros as a result.
+ */
+static bool zero_fill = false;
+
+/* set by --ones: every byte is 0xff.
+ *
+ * The other half of the same idea. A loop that counts up to a value from a
+ * buffer ends at once when that value is zero; a loop that walks until it meets
+ * a sentinel -- MetalFaceMask's span tables leave on -1 -- never meets it in a
+ * buffer of zeros, and ends at once in a buffer of 0xff. Neither fill is an
+ * input the kernel was written for, and between them they let most of these
+ * kernels reach their end instead of running forever. Which of the two a kernel
+ * terminates under is itself a fact about its shape.
+ */
+static bool ones_fill = false;
+
/* The argument buffer of a wrapped library function holds floats, not the byte
* pattern.
*
@@ -187,9 +218,98 @@ library_byte(unsigned i)
static uint8_t
pattern_byte(unsigned i)
{
+ if (zero_fill)
+ return 0;
+ if (ones_fill)
+ return 0xff;
return library_fill ? library_byte(i) : (i * 7 + 13) & 0xff;
}
+/* --buffer N@OFF=HEX puts those bytes into buffer N at that offset, over
+ * whatever fill the rest of it keeps.
+ *
+ * It exists because the kernels that never finish do not need a whole buffer
+ * chosen for them: they need a span table in one slot and a row index in
+ * another, and the value their exit test compares against is a field or two
+ * (tools/air/kinput.py works out which, and prints these arguments). A file is
+ * one fill for every buffer, so it cannot say "this slot and not that one".
+ * Offsets are decimal so that a job line stays readable; init.c splits a line
+ * on blanks and knows no variables, so each kernel has to be one line.
+ */
+struct buffer_patch {
+ unsigned index;
+ unsigned off;
+ unsigned len;
+ uint8_t *bytes;
+};
+static struct buffer_patch patches[64];
+static unsigned num_patches;
+
+/* Parses "N@OFF=HEX". Returns false when the argument is not that shape, so
+ * that --buffer keeps meaning a file name and a typo is not read as one.
+ */
+static bool
+add_buffer_patch(const char *spec)
+{
+ char *end;
+ unsigned long index = strtoul(spec, &end, 10);
+ if (end == spec || *end != '@')
+ return false;
+ const char *p = end + 1;
+ unsigned long off = strtoul(p, &end, 10);
+ if (end == p || *end != '=')
+ return false;
+
+ const char *hex = end + 1;
+ size_t n = strlen(hex);
+ if (!n || n % 2)
+ return false;
+ for (size_t i = 0; i < n; i++)
+ if (!isxdigit((unsigned char)hex[i]))
+ return false;
+
+ if (num_patches == ARRAY_SIZE(patches)) {
+ fprintf(stderr, "air_run: more than %zu --buffer patches\n", ARRAY_SIZE(patches));
+ exit(2);
+ }
+ struct buffer_patch *q = &patches[num_patches++];
+ q->index = index;
+ q->off = off;
+ q->len = n / 2;
+ q->bytes = malloc(q->len);
+ for (unsigned i = 0; i < q->len; i++) {
+ char byte[3] = { hex[2 * i], hex[2 * i + 1], 0 };
+ q->bytes[i] = strtoul(byte, NULL, 16);
+ }
+ return true;
+}
+
+/* Writes the patches for one buffer and returns how many bytes they wrote.
+ *
+ * A patch that does not fit is refused out loud rather than clipped: these
+ * bytes are what makes a loop end, and a kernel that silently did not get its
+ * sentinel hangs the GPU, which the next boot pays for.
+ */
+static unsigned
+apply_buffer_patches(unsigned index, uint8_t *bits, unsigned bytes)
+{
+ unsigned written = 0;
+ for (unsigned i = 0; i < num_patches; i++) {
+ const struct buffer_patch *q = &patches[i];
+ if (q->index != index)
+ continue;
+ if (q->off + q->len > bytes) {
+ printf("air_run: --buffer %u@%u= is %u bytes past the end of this %u-byte "
+ "buffer and was NOT written\n", q->index, q->off,
+ q->off + q->len - bytes, bytes);
+ continue;
+ }
+ memcpy(bits + q->off, q->bytes, q->len);
+ written += q->len;
+ }
+ return written;
+}
+
/* Fill a buffer the shader is about to read.
*
* AIR_RUN_BUFFER=FILE puts that file's bytes there instead of the pattern,
@@ -205,11 +325,16 @@ pattern_byte(unsigned i)
* rest as the pattern.
*/
static unsigned
-fill_buffer(uint8_t *bits, unsigned bytes)
+fill_buffer(unsigned index, uint8_t *bits, unsigned bytes)
{
for (unsigned k = 0; k < bytes; k++)
bits[k] = pattern_byte(k);
+ unsigned patched = apply_buffer_patches(index, bits, bytes);
+ if (patched)
+ printf("air_run: buffer at index %u, %u of its %u bytes from --buffer %u@...\n",
+ index, patched, bytes, index);
+
const char *path = getenv("AIR_RUN_BUFFER");
if (!path || !*path)
return 0;
@@ -221,6 +346,10 @@ fill_buffer(uint8_t *bits, unsigned bytes)
}
size_t got = fread(bits, 1, bytes, f);
fclose(f);
+ /* the file is the coarser statement -- one fill for every buffer -- so the
+ * per-slot patches go on top of it and not under it
+ */
+ apply_buffer_patches(index, bits, bytes);
return got;
}
@@ -274,8 +403,42 @@ type_size_vec4(const struct glsl_type *type, bool bindless)
}
static void
-finalize_for_gallium(nir_shader *nir)
+finalize_for_gallium(nir_shader *nir, struct pipe_screen *screen)
{
+ /* Does this screen's own backend lower I/O from the variables? softpipe's
+ * does -- its shader state is built by nir_to_tgsi, which calls nir_lower_io
+ * with this very mask and this very flag -- and ir3's does not.
+ *
+ * Gated on the screen's NAME, as the scratch block at the end of this
+ * function is and for the same reason: there is no pipe cap that says "this
+ * driver lowers I/O itself". PIPE_SHADER_CAP_SUPPORTED_IRS cannot stand in
+ * for one, because freedreno advertises TGSI too (through tgsi_to_nir, which
+ * is the other direction), and get_compiler_options returns a
+ * nir_shader_compiler_options with no field for it. So the name is what
+ * there is, and naming it is honest where inferring it from a cap that means
+ * something else would not be.
+ */
+ const bool backend_lowers_io =
+ screen->get_name && !strcmp(screen->get_name(screen), "softpipe");
+
+ /* A fragment shader's [[position]]: the frontend hands it over as a system-value
+ * variable, because that is the one form both of these drivers can be served from
+ * (the long version is beside air.position in air_to_nir.c). freedreno sets
+ * pipe_caps.fs_position_is_sysval and wants the intrinsic; softpipe does not set it
+ * and reads (0,0,0,0) from the intrinsic, because nothing fills TGSI's system values
+ * for a fragment shader. For such a screen the variable becomes an input varying
+ * here -- and then needs a driver_location, which nothing else would give it.
+ */
+ if (nir->info.stage == MESA_SHADER_FRAGMENT && !screen->caps.fs_position_is_sysval) {
+ static const struct nir_lower_sysvals_to_varyings_options pos_opts = {
+ .frag_coord = true,
+ };
+ if (nir_lower_sysvals_to_varyings(nir, &pos_opts)) {
+ nir_foreach_shader_in_variable(var, nir)
+ if (var->data.location == VARYING_SLOT_POS)
+ var->data.driver_location = nir->num_inputs++;
+ }
+ }
NIR_PASS(_, nir, nir_lower_system_values);
/* An options struct of all-false and not NULL, which are not the same thing
* here and the difference is a trap. nir_lower_system_values *introduces*
@@ -293,8 +456,116 @@ finalize_for_gallium(nir_shader *nir)
*/
static const nir_lower_compute_system_values_options cs_options = { 0 };
NIR_PASS(_, nir, nir_lower_compute_system_values, &cs_options);
- NIR_PASS(_, nir, nir_lower_io, nir_var_shader_in | nir_var_shader_out,
- type_size_vec4, 0);
+ /*
+ * nir_lower_io_use_interpolated_input_intrinsics, and WITHOUT it every varying
+ * was FLAT on the Adreno while softpipe interpolated it. Mesa's own comment on the
+ * flag says exactly why: without it nir_lower_io "generates regular load_input
+ * intrinsics and interpolation information must be inferred from the list of input
+ * nir_variables" -- and the two consumers infer it from different places.
+ * nir_to_tgsi declares each FS input from its variable's data.interpolation (SMOOTH,
+ * as the frontend sets it), so softpipe was right by accident; ir3's setup_input
+ * does `so->inputs[n].flat = !coord` and only builds `coord` for
+ * load_interpolated_input, so a plain load_input is flat, full stop.
+ *
+ * What that cost: the one draw in this project whose varying actually varies --
+ * Apple's texturedVertex2D handing a texcoord to texturedFragment2D* -- read the
+ * provoking vertex's texcoord at every fragment on the phone, and the picture looked
+ * VERTICALLY FLIPPED for three boots (docs/metal.md 10.26, 10.32). It is not flipped.
+ * The quad's pixel (0,0) sits in the triangle whose last vertex carries uv (0,1), so
+ * flat shading hands it texel (0,3) -- `100 203 303 400`, bit for bit the number the
+ * device printed, and reproduced here on softpipe by forcing INTERP_MODE_FLAT.
+ *
+ * Why nothing caught it: the 255 fragment shaders compared pixel by pixel are drawn
+ * over the generated vertex stage, which gives every vertex the SAME texcoord -- flat
+ * and smooth are identical over a constant. And ir3_compiler, the tool the corpus's
+ * "compiles through ir3" column comes from, passes this flag already
+ * (tools/mesa/air-ir3.patch): the tool was lowering correctly while the two harnesses
+ * that actually draw were not.
+ *
+ * ... and it is skipped entirely for a backend that lowers I/O itself, which
+ * is what made this harness unable to draw a fragment shader that reads a
+ * varying. nir_to_tgsi lowers I/O a SECOND time, from the variables, with
+ * the same mask, the same flag and a type_size that is type_size_vec4 word
+ * for word -- and it opens with
+ * nir_remove_dead_variables(nir_var_shader_in). Lowering here first leaves a
+ * shader whose inputs are intrinsics and whose input variables are held up
+ * by nothing but the orphaned deref_var instructions nir_lower_io does not
+ * delete; the scratch block below then runs nir_opt_dce, those go, ntt
+ * deletes the variables, and ntt_setup_inputs sizes input_index_map by the
+ * driver_locations of the variables that are left -- none. The
+ * load_interpolated_input this pass already emitted then swizzles a
+ * TGSI_FILE_NULL register and tgsi_ureg asserts, before any format, texture
+ * or sampler is reached.
+ *
+ * That abort was read as softpipe's limit in two places (pixfmtcheck's 30
+ * SAMPLED cases, metalcheck's draw sweep) and it is this file's. Counted:
+ * 134 of the corpus's 400 fragment entry points have an input variable on
+ * this screen -- 125 read a [[stage_in]] varying and nine read only
+ * [[position]], which the block above turns into one -- and 87 of them are
+ * QuartzCore's, the library every pixel of the screen is drawn with. Two of
+ * the 134 drew before this and both drew the WRONG pixel: the register
+ * ntt swizzled was whatever the heap held under a zero-length
+ * input_index_map, so the abort was the loud outcome and not the only one.
+ * air_metal.c runs the same passes without the dce and draws those shaders
+ * through the same softpipe, which is what said the limit was not softpipe's.
+ *
+ * Not lowering is preferred over keeping the derefs alive past the dce
+ * because it leaves ONE lowering of I/O in the pipeline instead of one and a
+ * half: ntt also runs nir_lower_indirect_derefs(nir_var_shader_in) before it
+ * lowers, which is a no-op on a shader whose derefs are already gone and is
+ * there because ureg splits input declarations to elements.
+ */
+ if (!backend_lowers_io)
+ NIR_PASS(_, nir, nir_lower_io, nir_var_shader_in | nir_var_shader_out,
+ type_size_vec4, nir_lower_io_use_interpolated_input_intrinsics);
+
+ /* SCRATCH, for a screen that has none.
+ *
+ * An `alloca` becomes load_scratch and store_scratch, which ir3 compiles
+ * and nir_to_tgsi has no case for at all: softpipe printed
+ * "Unknown intrinsic: @store_scratch" and then asserted inside ureg_swizzle
+ * on the undefined register it had made nothing for.
+ *
+ * WHAT IT ACTUALLY UNLOCKS, measured and not assumed: tools/air/callptr.ll,
+ * which is the whole reason it is here -- a shader with an alloca in it and
+ * nothing else nir_to_tgsi lacks, so LLVM on this machine and this draw can
+ * be compared and they agree on all four lanes. It does NOT make Apple's own
+ * RenderBox modules draw here: all fourteen of the ones that hand a shape
+ * function a `ptr` still refuse on softpipe, now for a reason that has
+ * nothing to do with scratch (`u2u32` from 16 bits asserts `src_64` in
+ * ntt_emit_alu, and these shaders are full of halves). The scratch is off
+ * the list of reasons; the list is not empty.
+ *
+ * nir_lower_scratch_to_var puts the whole scratch space back into one local
+ * variable, which nir_lower_vars_to_ssa then turns into ordinary values.
+ * That is sound exactly where this harness runs it: one invocation per
+ * fragment, no addresses taken across a call (nir_inline_functions has
+ * already folded every callee in), and nothing else reading the space.
+ *
+ * Gated on the screen by NAME and not run unconditionally, because the
+ * point of this harness is to put the SAME NIR through two drivers: doing
+ * it for ir3 as well would mean the Adreno column measured a shader with
+ * the scratch lowered away, which is not the shader the phone runs. There
+ * is no pipe cap that says "this driver has no scratch", so the name is
+ * what there is, and it is named here rather than guessed at from a cap
+ * that means something else.
+ *
+ * The nir_opt_dce at the end of this block is the pass that used to orphan
+ * every [[stage_in]] variable, and it is kept: with I/O left unlowered above
+ * for exactly the screens this block runs on, a load_deref still references
+ * each one, so there is nothing dead to remove. That is a coupling between
+ * two gates that test the same screen for different reasons, and the thing
+ * that would notice it coming apart is a test rather than a comment --
+ * tools/air/pixfmtcheck.py's last case draws one of Apple's varying-reading
+ * shaders and fails if the abort comes back.
+ */
+ if (screen->get_name && !strcmp(screen->get_name(screen), "softpipe")) {
+ NIR_PASS(_, nir, nir_lower_scratch_to_var);
+ NIR_PASS(_, nir, nir_split_var_copies);
+ NIR_PASS(_, nir, nir_lower_var_copies);
+ NIR_PASS(_, nir, nir_lower_vars_to_ssa);
+ NIR_PASS(_, nir, nir_opt_dce);
+ }
}
/* -- compute ---------------------------------------------------------------
@@ -325,6 +596,17 @@ finalize_for_gallium(nir_shader *nir)
*/
#define GRID 4
+/* The threadgroup this harness dispatches. 4x4x1 by default, because a grid with more than
+ * one thread in two dimensions is what makes a wrong index show, and --block X,Y,Z because
+ * some kernels are written for a particular one and say so in their own code: both fbs_pcg_*
+ * kernels end with `if (tid == 0)` summing 128 entries of a threadgroup array, which is a
+ * 128x1x1 reduction. Under 4x4x1 four threads have tid == 0 -- Metal's scalar
+ * thread_position_in_threadgroup is the x component, not the flattened index -- so four of
+ * them run the whole reduction over the four entries this shape writes, and the word they
+ * add up is a property of the dispatch rather than of the kernel (docs/air.md section 5).
+ */
+static unsigned block[3] = { GRID, GRID, 1 };
+
#define MAX_IMG 16
/* The shape of each image a kernel touches, read off the intrinsics the
@@ -362,13 +644,40 @@ image_shapes(nir_shader *cs, enum pipe_format fmt[MAX_IMG], bool isf[MAX_IMG],
unsigned b = nir_src_as_uint(in->src[0]);
if (b >= MAX_IMG || seen[b])
continue;
+ /* The WIDTH as well as the base type.
+ *
+ * A 32-bit format for a 16-bit store is not a conversion, it is a
+ * descriptor that disagrees with the instruction: ir3 emits
+ * `stib.b.typed.2d.u16.4` from air.write_texture_2d.v4f16 and the
+ * image it is given says 32 bits per component. MetalFaceMask_DrawSpans
+ * is that case -- it declares texture2d<half, write> -- and it is the
+ * kernel whose dispatch took a whole boot with it in
+ * logs/070755-run.log, printing its bindings and then nothing at all.
+ * 22 kernels of the corpus emit a narrow typed image store, so this is
+ * not one shader's accident. GL has no 16-bit image store, which is why
+ * neither Mesa nor this harness had met it; the draw path already picks
+ * the narrow format when a fragment shader returns halves.
+ */
+ unsigned bits = nir_alu_type_get_type_size(t);
+
switch (nir_alu_type_get_base_type(t)) {
case nir_type_int:
- fmt[b] = PIPE_FORMAT_R32G32B32A32_SINT; isf[b] = false; break;
+ fmt[b] = bits == 16 ? PIPE_FORMAT_R16G16B16A16_SINT
+ : bits == 8 ? PIPE_FORMAT_R8G8B8A8_SINT
+ : PIPE_FORMAT_R32G32B32A32_SINT;
+ isf[b] = false;
+ break;
case nir_type_uint:
- fmt[b] = PIPE_FORMAT_R32G32B32A32_UINT; isf[b] = false; break;
+ fmt[b] = bits == 16 ? PIPE_FORMAT_R16G16B16A16_UINT
+ : bits == 8 ? PIPE_FORMAT_R8G8B8A8_UINT
+ : PIPE_FORMAT_R32G32B32A32_UINT;
+ isf[b] = false;
+ break;
default:
- fmt[b] = PIPE_FORMAT_R32G32B32A32_FLOAT; isf[b] = true; break;
+ fmt[b] = bits == 16 ? PIPE_FORMAT_R16G16B16A16_FLOAT
+ : PIPE_FORMAT_R32G32B32A32_FLOAT;
+ isf[b] = true;
+ break;
}
dim[b] = nir_intrinsic_image_dim(in);
seen[b] = true;
@@ -414,6 +723,452 @@ texture_shapes(nir_shader *cs, enum pipe_format fmt[MAX_IMG], bool isf[MAX_IMG],
}
}
+/* Wait for the work to finish, and say so when it does not.
+ *
+ * This matters more than it looks. On 20 October a dispatch hung the GPU --
+ * our own driver said "gpu: no answer: submission 2 is the last, the core has
+ * done 1" -- and every dispatch for the rest of that boot was accepted,
+ * returned, and did nothing. air_run did not wait on a fence, so it printed
+ * the buffer it had filled itself as though the GPU had answered: a page of
+ * numbers that were a statement about the harness and not about the shader.
+ * 21 of the 47 kernels of one run were vacuous that way before anyone noticed
+ * (docs/air.md section 5).
+ *
+ * The hang itself turned out to be this harness too: the kernels that did it
+ * loop until a buffer value they were never given, and the fill is a byte
+ * pattern. So the message below leads with that rather than with the GPU.
+ *
+ * So the fence is waited on with a timeout, and a timeout is reported as
+ * loudly as a crash. A dispatch nobody waited for is not a measurement.
+ */
+static bool
+finish_or_say_why(struct pipe_screen *screen, struct pipe_context *pipe, const char *what)
+{
+ struct pipe_fence_handle *fence = NULL;
+ pipe->flush(pipe, &fence, 0);
+ if (!fence) {
+ printf("air_run: the driver gave no fence for the %s, so nothing here waited for "
+ "the GPU and the numbers below may be the fill\n", what);
+ return true;
+ }
+ bool done = screen->fence_finish(screen, NULL, fence, 2000000000ull); /* 2 s */
+ screen->fence_reference(screen, &fence, NULL);
+ if (!done)
+ printf("air_run: THE GPU DID NOT FINISH the %s within two seconds. Everything below is "
+ "what this harness wrote, not what the shader computed. The likeliest reason is "
+ "this harness: a kernel whose buffers carry counts, indices or sentinels is being "
+ "given (i * 7 + 13) & 0xff, and a loop that leaves on a value it was never given "
+ "does not leave at all -- both kernels measured this way turned out to be exactly "
+ "that (docs/air.md section 5). And one timeout wedges this GPU for the rest of the "
+ "boot, so every later run in it is vacuous too.\n", what);
+ return done;
+}
+
+/* What a shader's own sampler says, in one line. Printed rather than assumed:
+ * the ten modules of the corpus that carry one have no reference to be checked
+ * against, so the only thing a reader can verify is that the harness bound
+ * what the word says.
+ */
+static void
+print_own_sampler(unsigned slot, const struct air_sampler_state *a, bool compares)
+{
+ static const char *const addr[] = { "clamp_to_zero", "clamp_to_edge", "repeat",
+ "mirrored_repeat", "clamp_to_border" };
+ static const char *const filt[] = { "nearest", "linear", "bicubic" };
+ static const char *const mip[] = { "none", "nearest", "linear" };
+ /* Metal's compare_func, in its own order: `none` at 0 and `never` at 8, which is
+ * the value Apple's default carries (docs/air.md 3.3).
+ */
+ static const char *const cmp[] = { "none", "less", "less_equal", "greater",
+ "greater_equal", "equal", "not_equal", "always",
+ "never" };
+ printf("air_run: sampler %u is the shader's own: %s/%s filtering, mip %s, %s "
+ "addressing, %s coordinates, compare_func::%s%s\n", slot,
+ filt[a->mag_filter % ARRAY_SIZE(filt)], filt[a->min_filter % ARRAY_SIZE(filt)],
+ mip[a->mip_filter % ARRAY_SIZE(mip)],
+ addr[a->address[0] % ARRAY_SIZE(addr)],
+ a->pixel_coord ? "pixel" : "normalized",
+ a->compare_func < ARRAY_SIZE(cmp) ? cmp[a->compare_func] : "?",
+ compares ? " APPLIED (a shadow sample reads this slot)"
+ : " not read (nothing samples this slot with a comparison)");
+}
+
+/*
+ * THE PACKED 16-BIT PROBE: one known word, both directions, on whatever screen this is.
+ *
+ * air_metal.c has to decide which pipe_format each of Metal's four packed 16-bit names
+ * means, and that decision has two halves. Which Metal name goes with which Gallium name
+ * is Apple's convention and only Apple's own compositor can settle it. Where the fields
+ * of a given GALLIUM name actually land is a fact about the driver and the hardware, and
+ * that is what this measures -- on the Adreno, which is the only place it matters and
+ * the one place `air_metal_test --pixel-formats` cannot run, because that program is not
+ * on the device and air_run is.
+ *
+ * TWO FLAGS AND NOT ONE, because a round trip measures nothing. Sampling a packed
+ * texture and rendering into a packed target of the same format is unpack then pack with
+ * the same layout, which is the identity whatever that layout is: it would catch a read
+ * path disagreeing with a write path and could never catch both being wrong together.
+ * So the two directions are separate runs with a known value at the end that is NOT
+ * packed:
+ *
+ * --packed16 FORMAT 0xWORD every texel of the bound texture is that 16-bit word,
+ * with one field set; the target is the ordinary float one,
+ * so the four numbers printed say which CHANNEL the field
+ * became. The SAMPLED direction.
+ * --packed16-rt FORMAT the colour target is that packed format and the shader
+ * supplies the colour, so the word printed says which FIELD
+ * each channel became. The WRITTEN direction, and it wants
+ * a module whose output is a constant -- four floats could
+ * not show it, because they would have come back out
+ * through the same unpacking the write used.
+ *
+ * FORMAT is one of the eight Gallium names the two readings of Metal's four can mean. A
+ * name and not a number, so the job line says which layout it is asking about and a
+ * reader needs nothing else.
+ */
+static const struct { const char *name; enum pipe_format fmt; } packed16_names[] = {
+ { "R5G6B5_UNORM", PIPE_FORMAT_R5G6B5_UNORM },
+ { "B5G6R5_UNORM", PIPE_FORMAT_B5G6R5_UNORM },
+ { "A1R5G5B5_UNORM", PIPE_FORMAT_A1R5G5B5_UNORM },
+ { "A1B5G5R5_UNORM", PIPE_FORMAT_A1B5G5R5_UNORM },
+ { "R5G5B5A1_UNORM", PIPE_FORMAT_R5G5B5A1_UNORM },
+ { "B5G5R5A1_UNORM", PIPE_FORMAT_B5G5R5A1_UNORM },
+ { "A4R4G4B4_UNORM", PIPE_FORMAT_A4R4G4B4_UNORM },
+ { "A4B4G4R4_UNORM", PIPE_FORMAT_A4B4G4R4_UNORM },
+ /* the control, and the census is why it is here: the window server makes a
+ * BGRA8 texture of exactly the shape it makes the packed one
+ */
+ { "B8G8R8A8_UNORM", PIPE_FORMAT_B8G8R8A8_UNORM },
+};
+static enum pipe_format packed16_fmt = PIPE_FORMAT_NONE; /* the sampled texture */
+static enum pipe_format packed16_rt = PIPE_FORMAT_NONE; /* ... or the colour target */
+static enum pipe_format miplevel_fmt = PIPE_FORMAT_NONE; /* ... or a mip pyramid */
+/*
+ * --clear: what a framebuffer fetch will read, PER RENDER TARGET.
+ *
+ * Softpipe's TGSI interpreter has no FBFETCH at all, so the only driver here that
+ * can answer a fetch is the one on the phone -- and there, a fetch in a single
+ * draw reads what the clear put down. Chosen numbers turn that into a
+ * measurement: a shader that hands the fetch back must return the clear, and
+ * zero is then distinguishable from a fetch that read nothing.
+ *
+ * Per target and not one colour for all of them, because the question that
+ * matters is WHICH ATTACHMENT was read. The window server's own passes carry two
+ * colour attachments and mask one of them off entirely; a fetch that read the
+ * other one would be invisible if both held the same numbers.
+ */
+#define MAX_CLEARS 8
+static bool halftex;
+static union pipe_color_union clear_colour[MAX_CLEARS];
+static unsigned clear_count;
+static bool clear_given;
+static const char *packed16_name;
+static unsigned packed16_word;
+
+/*
+ * THE MIP PYRAMID, at the shape the window server actually asks for.
+ *
+ * The texture census of a whole boot (userland/src/redwoodmetal.m's texture line)
+ * found exactly ONE packed texture in the entire system: MTLPixelFormatBGR5A1Unorm,
+ * 2D, shaderRead|renderTarget, TEN MIP LEVELS, 776x24. Nothing else packed is ever
+ * created -- not formats 40, 41 or 42, and SpringBoard asks for none at all. So a
+ * probe on a single-level 4x4 texture is a probe of something the system does not
+ * have, and a mip chain is where a two-byte format differs most from the
+ * four-byte one everything else uses.
+ *
+ * AND THE SAME CENSUS CARRIES ITS OWN CONTROL: a format 80 (BGRA8Unorm) texture of
+ * the IDENTICAL shape -- type 2, usage 0x5, mips 10, 776x24. One logical object in
+ * two formats. Run through this probe, BGRA8 is the line that says whether a
+ * 776x24 pyramid works at all on this GPU, so a failure confined to the packed
+ * formats cannot be blamed on the shape, the usage, the level count or the
+ * harness.
+ *
+ * THE LEVEL IS IN THE VALUE, which is this harness's rule everywhere else: level L
+ * is filled with a texel whose four channels are the binary code of L + 1, each
+ * channel either all-ones or all-zeros. Three things follow. A sample that read
+ * the wrong level names the level it actually read, rather than merely
+ * disagreeing. Every expectation is exactly 0.0 or exactly 1.0 whatever the field
+ * widths are -- a 5-bit field at 9/31 would not be, and an expectation needing a
+ * tolerance cannot tell a near miss from a hit. And L + 1 RATHER THAN L, so that
+ * no level's code is zero: with L the first level would read 0 0 0 0, which is
+ * also what an unwritten level and an untouched allocation read, and the one
+ * outcome this is built to catch would have been indistinguishable from the
+ * healthy one.
+ */
+/*
+ * --halftex: a 256x1 RGBA16Float lookup table, written as RAW HALF BIT PATTERNS.
+ *
+ * CoreAnimation's gradient shader -- axial_shading_frag_lpf, 47 NIR instructions
+ * and one air.sample_texture_2d -- does nothing but sample a table of this exact
+ * shape and write the result, and the census found it: format 115, 2D, 256x1,
+ * shaderRead only, so the 256 entries are built on the CPU and uploaded. That
+ * upload path's half conversion has never been measured here.
+ *
+ * THE WORDS ARE CHOSEN AS INTEGERS AND WRITTEN AS INTEGERS, with no float-to-half
+ * conversion anywhere in this program, because a test that samples a table it
+ * also converted would pass with both halves wrong. The expectation is decoded
+ * from the same integers by a different implementation of binary16 on the host,
+ * so the two sides share nothing but the bit patterns.
+ *
+ * Texel i carries four patterns chosen to span what binary16 can be and nothing
+ * it cannot:
+ *
+ * r (i << 7) & 0x7bff a NORMAL of rising exponent; masked so the exponent
+ * never reaches 31, which would be an infinity or a NaN
+ * g r | 0x8000 the same magnitude NEGATIVE, so a sign dropped shows
+ * b i a DENORMAL for every i, and exactly zero at i = 0
+ * a 0x3c00 exactly 1.0, so one channel is a constant a wrong
+ * channel order cannot hide in
+ */
+#define HALFTEX_W 256
+
+static void
+halftex_texel(unsigned i, uint16_t out[4])
+{
+ out[0] = (uint16_t)((i << 7) & 0x7bff);
+ out[1] = (uint16_t)(out[0] | 0x8000);
+ out[2] = (uint16_t)i;
+ out[3] = 0x3c00;
+}
+
+#define MIPTEX_W 776
+#define MIPTEX_H 24
+#define MIPTEX_LEVELS 10
+
+static void
+miplevel_texel(unsigned level, float out[4])
+{
+ for (unsigned c = 0; c < 4; c++)
+ out[c] = ((level + 1) >> c) & 1 ? 1.0f : 0.0f;
+}
+
+static enum pipe_format
+packed16_by_name(const char *want, const char **name)
+{
+ for (unsigned i = 0; i < ARRAY_SIZE(packed16_names); i++)
+ if (!strcmp(want, packed16_names[i].name)) {
+ *name = packed16_names[i].name;
+ return packed16_names[i].fmt;
+ }
+ fprintf(stderr, "air_run: --packed16 wants one of these names:");
+ for (unsigned i = 0; i < ARRAY_SIZE(packed16_names); i++)
+ fprintf(stderr, " %s", packed16_names[i].name);
+ fprintf(stderr, "\n");
+ return PIPE_FORMAT_NONE;
+}
+
+/* The depth texel a shadow sample reads, which is the only pattern that fits.
+ *
+ * A depth format is normalised to [0, 1] and this harness's colour pattern starts at
+ * 100, so the two cannot be the same rule. (x + TEX_SIZE*y) / 16 gives sixteen
+ * distinct values over a 4x4 texture, every one exactly representable in binary32 and
+ * every one a different answer -- so a comparison that read the wrong texel answers
+ * differently, and a reference landing between two of them is exact on both sides.
+ * tools/air/sampcheck.py computes its expectations from this same expression.
+ */
+static float
+depth_texel_value(unsigned x, unsigned y)
+{
+ return (float)(x + TEX_SIZE * y) / 16.0f;
+}
+
+/* Which sampler slots a shadow texture instruction reads, out of the translated
+ * shader itself.
+ *
+ * Metal decides AT THE SAMPLE CALL that a comparison happens: sample() and
+ * sample_compare() are two instructions over one sampler, and the sampler word
+ * only says WHICH comparison. Gallium decides it in the sampler state, through
+ * pipe_sampler_state.compare_mode, and softpipe compares on every sample of a
+ * sampler that has it set (sp_tex_sample.c: `if (compare_mode != NONE)
+ * sample_compare(...)`). So switching it on for a slot a plain sample() reads
+ * would turn every texel into a 0 or a 1, and leaving it off for a slot a
+ * sample_compare() reads would return the texel instead of the comparison.
+ * Exactly these slots, and the shader is the only thing that knows which.
+ *
+ * On the Adreno the same two lines mean something slightly different and the
+ * answer is the same: a6xx has no compare-enable bit at all, only
+ * A6XX_TEX_SAMP_1_COMPARE_FUNC, which fd6_sampler_state_create writes only when
+ * compare_mode is set -- and the hardware reads it only for the shadow form of
+ * the sample instruction, which ir3 selects from nir_tex_instr::is_shadow
+ * (IR3_INSTR_S). So the field has to be programmed for these slots, and
+ * programming it for the others would be harmless there and wrong here.
+ */
+static void
+compare_sampled_slots(nir_shader *s, bool *slot, unsigned count)
+{
+ nir_foreach_function_impl(impl, s) {
+ nir_foreach_block(block, impl) {
+ nir_foreach_instr(instr, block) {
+ if (instr->type != nir_instr_type_tex)
+ continue;
+ nir_tex_instr *t = nir_instr_as_tex(instr);
+ if (t->is_shadow && t->sampler_index < count)
+ slot[t->sampler_index] = true;
+ }
+ }
+ }
+}
+
+/* A comparison through a sampler whose own word says `compare_func::none`,
+ * refused by name and in the same words air_metal.c's bind_stage refuses it.
+ *
+ * MTLCompareFunction has eight values and no "not configured", so `none` --
+ * value 0 of the shader word's nine -- has no API spelling and arrives at a
+ * driver as Never, which is MTLSamplerDescriptor's own default and is right for
+ * the plain sample that is all either can do. A COMPARISON is where the two
+ * part company: AGX keeps its NONE and its NEVER apart (AGXCompilerCore's
+ * _AGCCompareFunction), and what it does differently is not readable from here.
+ * Answering such a sample as `never` would be a harness quietly agreeing with
+ * itself where the library this is a reference for says no.
+ */
+static bool
+compare_without_a_function(const struct air_shader_info *info,
+ const bool *compare_slot, unsigned count,
+ const char *name)
+{
+ for (unsigned i = 0; i < info->num_resources; i++) {
+ const struct air_resource *r = &info->resources[i];
+ if (r->kind != AIR_RESOURCE_SAMPLER || !r->inline_sampler)
+ continue;
+ if (r->index >= count || !compare_slot[r->index])
+ continue;
+ if (r->sampler.compare_func != 0)
+ continue;
+ fprintf(stderr, "air_run: %s samples sampler %u with a comparison and its own "
+ "word says compare_func::none, which is not one of "
+ "MTLCompareFunction's eight: a sampler that was not made to "
+ "compare\n", name, r->index);
+ return true;
+ }
+ return false;
+}
+
+/* The sampler state a shader declared for itself, as Gallium wants it.
+ *
+ * Metal's clamp_to_zero is a wrap mode and not a border colour: outside the
+ * texture it reads transparent black, in every channel including alpha, and
+ * the border colour is not configurable for it. Gallium has no such mode, so
+ * it becomes CLAMP_TO_BORDER with a zero border -- which is the same thing and
+ * is worth writing down, because the obvious reading of that enum value is
+ * clamp_to_border with a default border, and one of Apple's own binaries
+ * prints it that way (nir_air.h).
+ *
+ * Fields with no Gallium equivalent, or none this harness can honour, are left
+ * at the default and named by the caller rather than silently dropped:
+ * bicubic filtering and the reduction mode AGX itself does not read.
+ *
+ * The comparison function is honoured, and only for the slots a shadow
+ * instruction reads -- `compares`, from compare_sampled_slots above. Its own
+ * enumeration is not MTLCompareFunction's and not Gallium's: in the word 0 is
+ * `none` and `never` is 8, which is the value Apple's own default carries, so
+ * every `constexpr sampler` that names no compare_func arrives here with 8 and
+ * must mean no comparison rather than an unknown one.
+ */
+static struct pipe_sampler_state
+sampler_from_air(const struct air_sampler_state *a, bool compares)
+{
+ static const unsigned wrap[] = {
+ [AIR_SAMPLER_CLAMP_TO_ZERO] = PIPE_TEX_WRAP_CLAMP_TO_BORDER,
+ [AIR_SAMPLER_CLAMP_TO_EDGE] = PIPE_TEX_WRAP_CLAMP_TO_EDGE,
+ [AIR_SAMPLER_REPEAT] = PIPE_TEX_WRAP_REPEAT,
+ [AIR_SAMPLER_MIRRORED_REPEAT] = PIPE_TEX_WRAP_MIRROR_REPEAT,
+ [AIR_SAMPLER_CLAMP_TO_BORDER] = PIPE_TEX_WRAP_CLAMP_TO_BORDER,
+ };
+ static const unsigned filt[] = {
+ [AIR_SAMPLER_NEAREST] = PIPE_TEX_FILTER_NEAREST,
+ [AIR_SAMPLER_LINEAR] = PIPE_TEX_FILTER_LINEAR,
+ [AIR_SAMPLER_BICUBIC] = PIPE_TEX_FILTER_LINEAR,
+ };
+ static const unsigned mip[] = {
+ [AIR_SAMPLER_MIP_NONE] = PIPE_TEX_MIPFILTER_NONE,
+ [AIR_SAMPLER_MIP_NEAREST] = PIPE_TEX_MIPFILTER_NEAREST,
+ [AIR_SAMPLER_MIP_LINEAR] = PIPE_TEX_MIPFILTER_LINEAR,
+ };
+ struct pipe_sampler_state st = {
+ .wrap_s = wrap[a->address[0] % ARRAY_SIZE(wrap)],
+ .wrap_t = wrap[a->address[1] % ARRAY_SIZE(wrap)],
+ .wrap_r = wrap[a->address[2] % ARRAY_SIZE(wrap)],
+ .mag_img_filter = filt[a->mag_filter % ARRAY_SIZE(filt)],
+ .min_img_filter = filt[a->min_filter % ARRAY_SIZE(filt)],
+ .min_mip_filter = mip[a->mip_filter % ARRAY_SIZE(mip)],
+ .unnormalized_coords = a->pixel_coord,
+ .max_anisotropy = a->max_anisotropy > 1 ? a->max_anisotropy : 0,
+ .min_lod = _mesa_half_to_float(a->lod_min),
+ .max_lod = _mesa_half_to_float(a->lod_max),
+ };
+ /* The comparison, where the shader asked for one. The table is the one in
+ * air_metal.c's sampler_state_from_air -- MTLCompareFunction and
+ * PIPE_FUNC_x run in the same order, so one table serves both -- and it is
+ * written out here rather than cast because 2 `less_equal` would arrive as
+ * PIPE_FUNC_EQUAL and 3 `greater` as PIPE_FUNC_LEQUAL, which is the
+ * opposite comparison and still a picture.
+ */
+ if (compares) {
+ static const unsigned func[] = {
+ [0] = PIPE_FUNC_NEVER, /* none: no MTLCompareFunction has a name for it */
+ [1] = PIPE_FUNC_LESS, /* less */
+ [2] = PIPE_FUNC_LEQUAL, /* less_equal */
+ [3] = PIPE_FUNC_GREATER, /* greater */
+ [4] = PIPE_FUNC_GEQUAL, /* greater_equal */
+ [5] = PIPE_FUNC_EQUAL, /* equal */
+ [6] = PIPE_FUNC_NOTEQUAL, /* not_equal */
+ [7] = PIPE_FUNC_ALWAYS, /* always */
+ [8] = PIPE_FUNC_NEVER, /* never */
+ };
+ st.compare_mode = PIPE_TEX_COMPARE_R_TO_TEXTURE;
+ st.compare_func = func[a->compare_func % ARRAY_SIZE(func)];
+ }
+
+ /* Metal's border_color: 0 transparent black, 1 opaque black, 2 opaque white.
+ * clamp_to_zero does not read it -- it is transparent black by definition.
+ */
+ if (a->address[0] == AIR_SAMPLER_CLAMP_TO_BORDER ||
+ a->address[1] == AIR_SAMPLER_CLAMP_TO_BORDER ||
+ a->address[2] == AIR_SAMPLER_CLAMP_TO_BORDER) {
+ float v = a->border_color == 2 ? 1.0f : 0.0f;
+ st.border_color.f[0] = st.border_color.f[1] = st.border_color.f[2] = v;
+ st.border_color.f[3] = a->border_color == 0 ? 0.0f : 1.0f;
+ }
+ return st;
+}
+
+/* The state for sampler slot `idx`: the shader's own if it declared one there,
+ * and this harness's nearest/clamp-to-edge otherwise -- which is what the API
+ * would have bound and what every measurement before inline samplers were
+ * decoded was made with.
+ */
+static struct pipe_sampler_state
+sampler_for_slot(const struct air_shader_info *info, unsigned idx, bool unnormalized,
+ bool compares, const struct air_sampler_state **from_shader)
+{
+ for (unsigned i = 0; i < info->num_resources; i++) {
+ const struct air_resource *r = &info->resources[i];
+ if (r->kind == AIR_RESOURCE_SAMPLER && r->inline_sampler && r->index == idx) {
+ if (from_shader)
+ *from_shader = &r->sampler;
+ return sampler_from_air(&r->sampler, compares);
+ }
+ }
+ if (from_shader)
+ *from_shader = NULL;
+ /* The API's sampler, and a comparison only if the shader asked for one.
+ * MTLSamplerDescriptor's own compareFunction defaults to Never, so this is
+ * the API's default sampler said completely rather than left out.
+ */
+ return (struct pipe_sampler_state){
+ .wrap_s = PIPE_TEX_WRAP_CLAMP_TO_EDGE,
+ .wrap_t = PIPE_TEX_WRAP_CLAMP_TO_EDGE,
+ .wrap_r = PIPE_TEX_WRAP_CLAMP_TO_EDGE,
+ .min_img_filter = PIPE_TEX_FILTER_NEAREST,
+ .mag_img_filter = PIPE_TEX_FILTER_NEAREST,
+ .min_mip_filter = PIPE_TEX_MIPFILTER_NONE,
+ .unnormalized_coords = unnormalized,
+ .compare_mode = compares ? PIPE_TEX_COMPARE_R_TO_TEXTURE
+ : PIPE_TEX_COMPARE_NONE,
+ .compare_func = PIPE_FUNC_NEVER,
+ };
+}
+
static int
dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,
nir_shader *cs, const char *name, const char *how)
@@ -428,12 +1183,12 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,
* filled with a distinctive pattern -- had it been zeroed, a kernel named
* InitMinMax writing zeros would have been indistinguishable from this.
*/
- cs->info.workgroup_size[0] = GRID;
- cs->info.workgroup_size[1] = GRID;
- cs->info.workgroup_size[2] = 1;
+ cs->info.workgroup_size[0] = block[0];
+ cs->info.workgroup_size[1] = block[1];
+ cs->info.workgroup_size[2] = block[2];
cs->info.workgroup_size_variable = false;
- finalize_for_gallium(cs);
+ finalize_for_gallium(cs, screen);
struct pipe_context *pipe = screen->context_create(screen, NULL, 0);
if (!pipe) {
fprintf(stderr, "air_run: no context for a kernel\n");
@@ -444,6 +1199,14 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,
/* every buffer the kernel declares; the device ones are what it writes */
struct pipe_resource *bufs[32] = { 0 };
+ /* The bytes this harness wrote, kept until the readback. The dump used to
+ * recompute them from pattern_byte(), which --buffer N@OFF=HEX makes a lie:
+ * a patched word looked changed by the kernel. Keeping the fill is also the
+ * only way to say which words a kernel touched when it touches them past
+ * the sixteen the dump prints -- and that is exactly what a prediction out
+ * of tools/air/kinput.py has to be compared against.
+ */
+ uint8_t *before[32] = { 0 };
unsigned sizes[32] = { 0 };
bool writable[32] = { false };
bool img_read[MAX_IMG] = { false }, img_write[MAX_IMG] = { false };
@@ -457,12 +1220,28 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,
if (r->index >= ARRAY_SIZE(bufs))
continue;
uint8_t *bits = calloc(1, bytes);
- unsigned from_file = fill_buffer(bits, bytes);
+ bool kept = false;
+ unsigned from_file = fill_buffer(r->index, bits, bytes);
if (from_file)
printf("air_run: buffer at index %u, %u bytes, %u of them from "
"AIR_RUN_BUFFER\n", r->index, bytes, from_file);
if (r->kind == AIR_RESOURCE_CONSTANT_BUFFER) {
+ /* A transform matrix filled with the byte pattern sends every sample
+ * somewhere arbitrary, so what the shader computes says nothing about
+ * what it is for. The identity is the one distinguished value a matrix
+ * has, and the resource says it is one: air.arg_type_name is Apple's
+ * own ("float4x4"), carried through from the module's metadata. Any
+ * other constant buffer keeps the pattern -- this is the only place
+ * the harness pretends to know what a buffer means, and it says so.
+ */
+ if (r->type_name && !strncmp(r->type_name, "float4x4", 8) && bytes >= 64) {
+ float *m = (float *)bits;
+ memset(m, 0, 64);
+ m[0] = m[5] = m[10] = m[15] = 1.0f;
+ printf("air_run: constant buffer %u is a %s, so it holds the identity and "
+ "not the fill\n", r->index, r->type_name);
+ }
struct pipe_resource *b = pipe_buffer_create(screen, PIPE_BIND_CONSTANT_BUFFER,
PIPE_USAGE_DEFAULT, bytes);
pipe_buffer_write(pipe, b, 0, bytes, bits);
@@ -474,6 +1253,8 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,
PIPE_USAGE_STAGING, bytes);
pipe_buffer_write(pipe, bufs[r->index], 0, bytes, bits);
sizes[r->index] = bytes;
+ before[r->index] = bits;
+ kept = true;
writable[r->index] = r->write;
num_ssbo = MAX2(num_ssbo, r->index + 1);
num_written += r->write ? 1 : 0;
@@ -497,7 +1278,8 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,
tex_used[r->index] = true;
num_tex = MAX2(num_tex, r->index + 1);
}
- free(bits);
+ if (!kept)
+ free(bits);
}
unsigned num_img_written = 0;
for (unsigned i = 0; i < num_img; i++)
@@ -643,14 +1425,6 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,
views[i] = pipe->create_sampler_view(pipe, texes[i], &vtempl);
}
if (num_tex) {
- csampler = pipe->create_sampler_state(pipe, &(struct pipe_sampler_state){
- .wrap_s = PIPE_TEX_WRAP_CLAMP_TO_EDGE,
- .wrap_t = PIPE_TEX_WRAP_CLAMP_TO_EDGE,
- .wrap_r = PIPE_TEX_WRAP_CLAMP_TO_EDGE,
- .min_img_filter = PIPE_TEX_FILTER_NEAREST,
- .mag_img_filter = PIPE_TEX_FILTER_NEAREST,
- .min_mip_filter = PIPE_TEX_MIPFILTER_NONE,
- .unnormalized_coords = 1 });
/* one entry per texture, not the address of one pointer. bind_sampler_states
* reads num_tex pointers from what it is given, so passing &csampler made
* the driver take whatever followed that single variable on the stack and
@@ -658,10 +1432,24 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,
* ->seqno and died at a wild address. It only showed with two or more
* textures, which is why 22 kernels bound images here without trouble and
* four did not. The draw path below always did this correctly.
+ *
+ * One state per slot, and not one shared: a slot's sampler is whatever
+ * that slot declares, and a shader may mix its own with the API's.
*/
void *csamplers[MAX_IMG] = { 0 };
- for (unsigned i = 0; i < num_tex && i < MAX_IMG; i++)
- csamplers[i] = csampler;
+ bool compare_slot[MAX_IMG] = { 0 };
+ compare_sampled_slots(cs, compare_slot, MAX_IMG);
+ if (compare_without_a_function(info, compare_slot, MAX_IMG, name))
+ return 1;
+ for (unsigned i = 0; i < num_tex && i < MAX_IMG; i++) {
+ const struct air_sampler_state *own = NULL;
+ struct pipe_sampler_state st = sampler_for_slot(info, i, true,
+ compare_slot[i], &own);
+ csamplers[i] = pipe->create_sampler_state(pipe, &st);
+ csampler = csamplers[i];
+ if (own)
+ print_own_sampler(i, own, compare_slot[i]);
+ }
pipe->bind_sampler_states(pipe, PIPE_SHADER_COMPUTE, 0, num_tex, csamplers);
pipe->set_sampler_views(pipe, PIPE_SHADER_COMPUTE, 0, num_tex, 0, false, views);
}
@@ -678,19 +1466,23 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,
/* more than one thread, and the whole grid, so that a wrong index shows */
struct pipe_grid_info grid = {
.work_dim = 3,
- .block = { GRID, GRID, 1 },
+ .block = { block[0], block[1], block[2] },
.grid = { 1, 1, 1 },
};
- printf("air_run: dispatching %ux%ux1 threads in one workgroup; %u device buffer%s bound, "
- "%u of them written, filled with (i * 7 + 13) & 0xff\n",
- GRID, GRID, num_ssbo, num_ssbo == 1 ? "" : "s", num_written);
+ printf("air_run: dispatching %ux%ux%u threads in one workgroup; %u device buffer%s bound, "
+ "%u of them written, filled with %s\n",
+ block[0], block[1], block[2], num_ssbo, num_ssbo == 1 ? "" : "s", num_written,
+ zero_fill ? "ZEROS (--zero: a degenerate input, see docs/air.md section 5)"
+ : ones_fill ? "0xff BYTES (--ones: a degenerate input)"
+ : library_fill ? "the floats ((w * 13) % 29 - 14) / 4"
+ : "(i * 7 + 13) & 0xff");
if (num_img || num_tex)
printf("air_run: %u image%s bound (%u written) and %u texture%s, %ux%u, texel (x,y,z) of "
"slot n = (100+x+1000n, 200+y, 300+x+y+z, 400)\n", num_img,
num_img == 1 ? "" : "s", num_img_written, num_tex, num_tex == 1 ? "" : "s",
TEX_SIZE, TEX_SIZE);
pipe->launch_grid(pipe, &grid);
- pipe->flush(pipe, NULL, 0);
+ bool finished = finish_or_say_why(screen, pipe, "dispatch");
for (unsigned i = 0; i < num_ssbo; i++) {
if (!writable[i])
@@ -701,13 +1493,61 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,
fprintf(stderr, "air_run: device buffer %u could not be read back\n", i);
return 1;
}
- unsigned words = MIN2(sizes[i] / 4, (unsigned)(GRID * GRID));
+ unsigned words = MIN2(sizes[i] / 4, block[0] * block[1] * block[2]);
printf("air_run: device buffer %u after the dispatch, first %u words:\n", i, words);
for (unsigned w = 0; w < words; w++)
printf(" [%2u] %11u 0x%08x\n", w, p[w], p[w]);
+
+ /* A buffer the shader writes that comes back bit-identical to the fill
+ * is not a result, and the only thing worse than no number is a number
+ * that looks like one.
+ *
+ * This is not hypothetical and not rare. On the phone, a boot survives
+ * about two dozen dispatches that bind textures or images; after that
+ * every dispatch still "succeeds", every readback still prints, and
+ * nothing the GPU writes is ever visible again -- measured on
+ * 20 October (logs/182513-run.log: writes land for the 13th, 15th and
+ * 23rd kernel of the run and for nothing after it, while the same
+ * kernels and four probe kernels all write correctly in a short run,
+ * logs/182805-run.log). The same boot later refuses an allocation
+ * outright with "bo_allocate: alloc failed". So the words below can be
+ * a statement about how long the run had been going rather than about
+ * the shader, and whoever reads them has to be told which.
+ */
+ /* and then every word of the whole buffer that differs from what went in,
+ * capped so that a kernel which rewrites all 1024 of them does not push
+ * the rest of a run out of the boot log (the loader keeps the first
+ * 256 KB and the last 1 MB and drops the middle).
+ */
+ unsigned changed = 0, shown = 0;
+ const uint32_t *was = (const uint32_t *)before[i];
+ for (unsigned w = 0; was && w < sizes[i] / 4; w++) {
+ if (p[w] == was[w])
+ continue;
+ changed++;
+ if (shown < 24) {
+ printf(" changed [%4u] 0x%08x -> 0x%08x\n", w, was[w], p[w]);
+ shown++;
+ }
+ }
+ if (was && changed)
+ printf("air_run: device buffer %u: %u of its %u words differ from what this "
+ "harness wrote%s\n", i, changed, sizes[i] / 4,
+ shown < changed ? " (the first 24 are above)" : "");
+ else if (was)
+ printf("air_run: device buffer %u came back bit-identical to the fill, so this "
+ "dispatch wrote nothing visible. Either the shader writes elsewhere, or "
+ "the GPU has stopped making writes visible -- which it does after a few "
+ "dozen dispatches in one boot (docs/air.md section 4).\n", i);
pipe_buffer_unmap(pipe, xfer);
}
+ if (!finished) {
+ fprintf(stderr, "air_run: %s: the dispatch did not finish, so no number here is "
+ "evidence\n", name);
+ return 1;
+ }
+
/* and every image the kernel wrote: without this a dispatch that writes only
* an image -- which is most of the corpus's kernels -- would have nothing to
* read back, and air_run would have reported a successful run of something
@@ -745,6 +1585,60 @@ dispatch_kernel(struct pipe_screen *screen, const struct air_shader_info *info,
return 0;
}
+/* -- function constant values ---------------------------------------------
+ *
+ * --fc INDEX=VALUE supplies one function constant, and it exists because
+ * nothing in this tree ever supplied one. air_module_function_constants()
+ * registers every constant a module declares with its index and leaves
+ * air_constant_value.bits at zero, so air.is_function_constant_defined folds
+ * to true and every load yields zero -- for a constant whose global is an
+ * `externally_initialized ... undef` in section "air.fc_initializer" there is
+ * no default to fall back on, and zero is then not a default but a value this
+ * compiler invented. 34 of RenderBox's 47 modules declare such a constant and
+ * 25 of them derive their resource interface and their control flow from it in
+ * an "air.static_init" constructor, so zero picks one arbitrary specialization
+ * out of a state machine and the picture cannot say which.
+ *
+ * An index the module does not declare is refused rather than ignored: an
+ * option that silently sets nothing would make every run agree with every
+ * other one for the wrong reason.
+ */
+/* 64 and not 32, which is the number the corpus asks for rather than a round
+ * one: fixed_frag_lph_cph and fixed_frag_lph_cpf declare 58 function constants
+ * each, so the old array could not carry one pipeline's worth of them and the
+ * refusal was "more than 32 function constants" -- on the module this matters
+ * for most. The same bound is RW_MAX_CONSTANTS in air_metal.h, raised to 64
+ * for the same reason.
+ */
+static struct {
+ unsigned index;
+ uint64_t bits;
+} fc_set[64];
+static unsigned num_fc_set;
+
+static bool
+apply_fc_values(struct air_constant_value *fc, unsigned num_fc)
+{
+ for (unsigned i = 0; i < num_fc_set; i++) {
+ unsigned j;
+ for (j = 0; j < num_fc; j++) {
+ if (fc[j].index == fc_set[i].index) {
+ fc[j].bits = fc_set[i].bits;
+ fc[j].supplied = true;
+ printf("air_run: function constant %u = 0x%llx\n", fc_set[i].index,
+ (unsigned long long)fc_set[i].bits);
+ break;
+ }
+ }
+ if (j == num_fc) {
+ fprintf(stderr, "air_run: this module declares no function constant %u"
+ " (it declares %u)\n", fc_set[i].index, num_fc);
+ return false;
+ }
+ }
+ return true;
+}
+
int
main(int argc, char **argv)
{
@@ -762,7 +1656,7 @@ main(int argc, char **argv)
*/
bool want_library = false;
/* --buffer FILE is AIR_RUN_BUFFER as an argument, and it exists because the
- * device has no shell to set one with. Darwin 16.5's own root carries no sh at
+ * device has no shell to set one with. iOS 16.5's own root carries no sh at
* all -- /bin holds df and ps -- and the job list that runs this is read by
* our init, which splits a line on blanks and has no notion of a variable
* (userland/src/init.c, run_command). So a generated suite whose every
@@ -775,12 +1669,108 @@ main(int argc, char **argv)
setenv("IR3_SHADER_DEBUG", "disasm", 1);
argv++;
argc--;
+ } else if (!strcmp(argv[1], "--zero")) {
+ zero_fill = true;
+ argv++;
+ argc--;
+ } else if (!strcmp(argv[1], "--ones")) {
+ ones_fill = true;
+ argv++;
+ argc--;
+ } else if (!strcmp(argv[1], "--packed16") && argc > 3) {
+ packed16_fmt = packed16_by_name(argv[2], &packed16_name);
+ if (packed16_fmt == PIPE_FORMAT_NONE)
+ return 2;
+ packed16_word = (unsigned)strtoul(argv[3], NULL, 0) & 0xffff;
+ argv += 3;
+ argc -= 3;
+ } else if (!strcmp(argv[1], "--clear") && argc > 2) {
+ /* R,G,B,A for target 0, then ':' and four more for target 1, and so on.
+ * A colon and not a semicolon: these reach the phone as job lines, and
+ * the init that reads them splits on blanks and promises nothing about
+ * shell syntax -- a separator that a shell WOULD have meant something by
+ * is one nobody can be sure about by reading the line.
+ */
+ const char *spec = argv[2];
+ clear_count = 0;
+ while (spec && *spec && clear_count < MAX_CLEARS) {
+ float c[4] = { 0, 0, 0, 0 };
+ if (sscanf(spec, "%f,%f,%f,%f", &c[0], &c[1], &c[2], &c[3]) != 4) {
+ fprintf(stderr, "air_run: --clear wants R,G,B,A per target, ':' between "
+ "targets\n");
+ return 2;
+ }
+ for (unsigned i = 0; i < 4; i++)
+ clear_colour[clear_count].f[i] = c[i];
+ clear_count++;
+ const char *sep = strchr(spec, ':');
+ spec = sep ? sep + 1 : NULL;
+ }
+ clear_given = true;
+ argv += 2;
+ argc -= 2;
+ } else if (!strcmp(argv[1], "--halftex")) {
+ halftex = true;
+ argv++;
+ argc--;
+ } else if (!strcmp(argv[1], "--miplevels") && argc > 2) {
+ miplevel_fmt = packed16_by_name(argv[2], &packed16_name);
+ if (miplevel_fmt == PIPE_FORMAT_NONE)
+ return 2;
+ argv += 2;
+ argc -= 2;
+ } else if (!strcmp(argv[1], "--packed16-rt") && argc > 2) {
+ packed16_rt = packed16_by_name(argv[2], &packed16_name);
+ if (packed16_rt == PIPE_FORMAT_NONE)
+ return 2;
+ argv += 2;
+ argc -= 2;
} else if (!strcmp(argv[1], "--library")) {
want_library = library_fill = true;
argv++;
argc--;
+ } else if (!strcmp(argv[1], "--stitch") && argc > 2) {
+ /* A graph of air.visible functions, dispatched as one kernel. The
+ * argument buffer is filled the wrapper's way for the same reason the
+ * stitched entry point reads the wrapper's slots: a one-node graph
+ * then has to write exactly what --library writes for that function.
+ */
+ stitch_text = argv[2];
+ library_fill = true;
+ argv += 2;
+ argc -= 2;
+ } else if (!strcmp(argv[1], "--block") && argc > 2) {
+ unsigned x = 0, y = 0, z = 0;
+
+ if (sscanf(argv[2], "%u,%u,%u", &x, &y, &z) != 3 || !x || !y || !z) {
+ fprintf(stderr, "air_run: --block wants X,Y,Z and all three above zero\n");
+ return 2;
+ }
+ block[0] = x, block[1] = y, block[2] = z;
+ argv += 2;
+ argc -= 2;
+ } else if (!strcmp(argv[1], "--fc") && argc > 2) {
+ unsigned idx = 0;
+ uint64_t val = 0;
+
+ if (sscanf(argv[2], "%u=0x%llx", &idx, (unsigned long long *)&val) != 2 &&
+ sscanf(argv[2], "%u=%llu", &idx, (unsigned long long *)&val) != 2) {
+ fprintf(stderr, "air_run: --fc wants INDEX=VALUE, decimal or 0x\n");
+ return 2;
+ }
+ if (num_fc_set >= ARRAY_SIZE(fc_set)) {
+ fprintf(stderr, "air_run: more than %u function constants\n",
+ (unsigned)ARRAY_SIZE(fc_set));
+ return 2;
+ }
+ fc_set[num_fc_set].index = idx;
+ fc_set[num_fc_set].bits = val;
+ num_fc_set++;
+ argv += 2;
+ argc -= 2;
} else if (!strcmp(argv[1], "--buffer") && argc > 2) {
- setenv("AIR_RUN_BUFFER", argv[2], 1);
+ if (!add_buffer_patch(argv[2]))
+ setenv("AIR_RUN_BUFFER", argv[2], 1);
argv += 2;
argc -= 2;
} else {
@@ -788,8 +1778,10 @@ main(int argc, char **argv)
}
}
if (argc < 2) {
- fprintf(stderr, "usage: air_run [--disasm] [--library] [--buffer FILE] "
- "FILE[.metallib] [FUNCTION]\n");
+ fprintf(stderr, "usage: air_run [--disasm] [--library] [--block X,Y,Z] "
+ "[--buffer FILE | --buffer N@OFF=HEX] "
+ "[--fc INDEX=VALUE] "
+ "[--zero | --ones] FILE[.metallib] [FUNCTION]\n");
return 2;
}
@@ -817,6 +1809,27 @@ main(int argc, char **argv)
size_t bitcode_size = size;
const char *want = argc > 2 ? argv[2] : NULL;
const char *name = argv[1];
+
+ /* A stitched graph is still handed to air_to_nir as one module plus the
+ * graph, and that module is the OUTPUT node's: its own visible function is
+ * one node like any other.
+ */
+ const struct air_stitch_graph *graph = NULL;
+ struct air_stitch_function *stitch_fns = NULL;
+ unsigned num_stitch_fns = 0;
+ if (stitch_text) {
+ char why[256] = { 0 };
+ graph = air_stitch_parse(ctx, stitch_text, why, sizeof(why));
+ if (graph)
+ num_stitch_fns = air_stitch_functions_of(ctx, graph, data, size, &stitch_fns,
+ why, sizeof(why));
+ if (!graph || !num_stitch_fns) {
+ fprintf(stderr, "air_run: --stitch: %s\n", why[0] ? why : "no functions");
+ return 2;
+ }
+ want = graph->nodes[graph->output_node].function;
+ }
+
if (size >= 4 && !memcmp(data, "MTLB", 4)) {
struct air_metallib lib;
const char *err;
@@ -898,8 +1911,13 @@ main(int argc, char **argv)
/* the shader */
struct air_constant_value *fc;
unsigned num_fc = air_module_function_constants(ctx, bitcode, bitcode_size, &fc);
+ if (!apply_fc_values(fc, num_fc))
+ return 1;
struct air_to_nir_options air_options = { .constants = fc, .num_constants = num_fc,
- .library_wrapper = want_library };
+ .library_wrapper = want_library,
+ .stitch = graph,
+ .stitch_functions = stitch_fns,
+ .num_stitch_functions = num_stitch_fns };
struct air_shader_info info;
const char *err;
nir_shader *fs = air_to_nir(ctx, bitcode, bitcode_size, NULL, &air_options, options,
@@ -923,6 +1941,10 @@ main(int argc, char **argv)
enum pipe_format rt_format = PIPE_FORMAT_R32G32B32A32_FLOAT;
bool rt_is_float = true;
bool rt_is_half = false;
+ /* --packed16 renders INTO the same format it sampled, which is the written half of
+ * the probe: the pixel is packed back into 16 bits by the GPU and read out raw.
+ */
+ bool rt_is_packed16 = packed16_rt != PIPE_FORMAT_NONE;
/* Which attachment to read back: the lowest one the shader writes, not
* target 0. clearFragment04 writes FRAG_RESULT_DATA1 alone, and target 0
* then holds the clear colour -- a correct result that looks exactly like
@@ -977,6 +1999,11 @@ main(int argc, char **argv)
rt_is_float = true;
}
}
+ if (rt_is_packed16) {
+ rt_format = packed16_rt;
+ rt_is_float = false;
+ rt_is_half = false;
+ }
if (read_rt == ~0u) {
fprintf(stderr, "air_run: %s writes no colour attachment\n", name);
return 1;
@@ -1010,7 +2037,108 @@ main(int argc, char **argv)
return 1;
}
- finalize_for_gallium(fs);
+ /* A shader that discards cannot be run by this screen, and that is
+ * softpipe's limit rather than the frontend's. air.discard_fragment becomes
+ * NIR's `demote` (air_intrinsics.c says at length why it is demote and not
+ * terminate); nir_to_tgsi sends it to TGSI_OPCODE_DEMOTE, which the
+ * interpreter does not implement -- tgsi_exec.c has a case for KILL and
+ * none for DEMOTE, so exec_instruction falls through to its `assert(0)` and
+ * the whole run aborts at tgsi_exec.c:5943. An abort inside a check reads
+ * like a crash in the thing being checked, so it is refused here by name,
+ * the way the multisample textures below are.
+ *
+ * ir3 has both instructions and compiles these, so what is open is the
+ * VALUE a discarding shader leaves in the framebuffer, and that is a
+ * question for the phone.
+ */
+ /*
+ * On SOFTPIPE only, and that qualification is the whole point of the comment above:
+ * "ir3 has both instructions and compiles these, so what is open is the VALUE a
+ * discarding shader leaves in the framebuffer, and that is a question for the phone."
+ * The gate was written without it and asked only whether the module discards, so on
+ * the phone -- where the question was supposed to be answered -- fifteen of
+ * RenderBox's forty-seven modules were refused for a limitation of a screen that was
+ * not in use. `mask_color_fragment` is one of them, and it is what draws a widget.
+ *
+ * A refusal that names the wrong wall reads as principled when it is merely absent,
+ * which is the defect this file's own log line demonstrated: it printed "this screen's
+ * TGSI interpreter" over a run whose every line said "on Adreno 7c+ Gen 3".
+ */
+ if (fs->info.fs.uses_discard && screen->get_name &&
+ !strcmp(screen->get_name(screen), "softpipe")) {
+ fprintf(stderr, "air_run: %s discards fragments, and THIS screen's TGSI "
+ "interpreter has no DEMOTE case at all (tgsi_exec.c), so it is "
+ "not run rather than run against the wrong thing. On the Adreno "
+ "it runs: ir3 has the instruction\n", name);
+ return 1;
+ }
+
+ /* And the rest of the family this interpreter does not have.
+ *
+ * `grep -c TGSI_OPCODE_VOTE_ALL tgsi_exec.c` is 0, and so are BALLOT,
+ * READ_FIRST, READ_INVOC, READ_HELPER and DEMOTE: nir_to_tgsi emits all of
+ * them and softpipe's exec_instruction implements none, so it falls through
+ * to its own assert(0) -- an abort in the middle of a check, which reads
+ * like a crash in the thing being checked rather than like a screen that
+ * cannot do this. The quad swaps are worse again: nir_to_tgsi has no case
+ * for them either, prints "Unknown intrinsic" and then asserts inside
+ * ureg_swizzle on the register it never made.
+ *
+ * So a shader that asks for a cross-lane value is refused here by name. It
+ * is not a frontend limit and not a hardware one -- ir3 has real
+ * instructions for every one of these, and compiles them -- which is why
+ * the refusal names the screen.
+ */
+ const char *no_lanes = NULL;
+ nir_foreach_function_impl(impl, fs) {
+ nir_foreach_block(block, impl) {
+ nir_foreach_instr(instr, block) {
+ if (instr->type != nir_instr_type_intrinsic)
+ continue;
+ nir_intrinsic_op op = nir_instr_as_intrinsic(instr)->intrinsic;
+ switch (op) {
+ case nir_intrinsic_vote_all:
+ case nir_intrinsic_vote_any:
+ case nir_intrinsic_vote_feq:
+ case nir_intrinsic_vote_ieq:
+ case nir_intrinsic_ballot:
+ case nir_intrinsic_read_first_invocation:
+ case nir_intrinsic_read_invocation:
+ case nir_intrinsic_reduce:
+ case nir_intrinsic_inclusive_scan:
+ case nir_intrinsic_exclusive_scan:
+ case nir_intrinsic_shuffle:
+ case nir_intrinsic_shuffle_xor:
+ case nir_intrinsic_shuffle_up:
+ case nir_intrinsic_shuffle_down:
+ case nir_intrinsic_quad_broadcast:
+ case nir_intrinsic_quad_swap_horizontal:
+ case nir_intrinsic_quad_swap_vertical:
+ case nir_intrinsic_quad_swap_diagonal:
+ case nir_intrinsic_is_helper_invocation:
+ no_lanes = nir_intrinsic_infos[op].name;
+ break;
+ default:
+ break;
+ }
+ if (no_lanes)
+ break;
+ }
+ if (no_lanes)
+ break;
+ }
+ if (no_lanes)
+ break;
+ }
+ if (no_lanes) {
+ fprintf(stderr, "air_run: %s reads another lane (NIR's %s), and this screen's "
+ "TGSI interpreter implements none of the subgroup or quad "
+ "opcodes nir_to_tgsi emits for them, so it is not run rather "
+ "than aborting inside softpipe\n", name, no_lanes);
+ return 1;
+ }
+
+ finalize_for_gallium(fs, screen);
struct pipe_context *pipe = screen->context_create(screen, NULL, 0);
@@ -1046,7 +2174,19 @@ main(int argc, char **argv)
pipe->set_framebuffer_state(pipe, &fb);
pipe->set_viewport_states(pipe, 0, 1, &(struct pipe_viewport_state){
- .scale = { WIDTH / 2.0f, HEIGHT / 2.0f, 1.0f },
+ /* The y scale is NEGATIVE, and that is Metal's rule rather than a convention picked
+ * here. Metal's render-target origin is the top left and its NDC y points up, so clip
+ * y = +1 belongs in memory row 0; a positive y scale puts clip y = -1 there, which is
+ * OpenGL's drawable rule and what this code did until now (docs/metal.md 10.28). Mesa
+ * negates y in exactly this place, for exactly this reason, when it draws to a window.
+ *
+ * It is landed only now because until the varying was fixed (10.32) nothing could see
+ * it: every draw of the corpus either writes a constant or samples one fixed texel, and
+ * over a constant an orientation is invisible. The sixteen-pixel grid of 10.32 is what
+ * it is landed against -- it must turn from (100+x, 200+y, ...) into
+ * (100+x, 200+(3-y), ...), the texture mirrored vertically, on both back ends.
+ */
+ .scale = { WIDTH / 2.0f, -(HEIGHT / 2.0f), 1.0f },
.translate = { WIDTH / 2.0f, HEIGHT / 2.0f, 0.0f },
});
@@ -1063,21 +2203,51 @@ main(int argc, char **argv)
});
pipe->bind_rasterizer_state(pipe, rast);
- /* every constant buffer the shader reads, filled with the pattern */
+ /* Every buffer the shader reads, filled with the pattern.
+ *
+ * A `constant` buffer and a `device` buffer are TWO BINDINGS and not one.
+ * The frontend translates a load from the first into load_ubo and from the
+ * second into load_ssbo (docs/air.md section 3), and this path used to bind
+ * both with set_constant_buffer -- so a draw whose shader reads a device
+ * buffer read ZEROS, and read them silently: the pixel came back, four
+ * numbers wide, with nothing anywhere saying a binding had been missed. The
+ * compute path above has always bound the two separately; only the draw path
+ * had them confused. Found by tools/air/callptr.ll, whose last two lanes are
+ * a device buffer and whose first two are not, so the picture said which
+ * binding was wrong rather than that something was.
+ */
+ struct pipe_shader_buffer fsb[32] = { 0 };
+ unsigned num_fs_ssbo = 0;
for (unsigned i = 0; i < info.num_resources; i++) {
const struct air_resource *r = &info.resources[i];
if (r->kind != AIR_RESOURCE_CONSTANT_BUFFER && r->kind != AIR_RESOURCE_DEVICE_BUFFER)
continue;
unsigned bytes = r->size ? r->size : 256;
uint8_t *bits = malloc(bytes);
- unsigned from_file = fill_buffer(bits, bytes);
- struct pipe_resource *buf = pipe_buffer_create(screen, PIPE_BIND_CONSTANT_BUFFER,
- PIPE_USAGE_DEFAULT, bytes);
+ unsigned from_file = fill_buffer(r->index, bits, bytes);
+ bool device = r->kind == AIR_RESOURCE_DEVICE_BUFFER;
+ struct pipe_resource *buf =
+ pipe_buffer_create(screen,
+ device ? PIPE_BIND_SHADER_BUFFER : PIPE_BIND_CONSTANT_BUFFER,
+ PIPE_USAGE_DEFAULT, bytes);
pipe_buffer_write(pipe, buf, 0, bytes, bits);
- pipe->set_constant_buffer(pipe, PIPE_SHADER_FRAGMENT, r->index, false,
- &(struct pipe_constant_buffer){
- .buffer = buf, .buffer_size = bytes,
- });
+ if (device) {
+ if (r->index < ARRAY_SIZE(fsb)) {
+ fsb[r->index].buffer = buf;
+ fsb[r->index].buffer_size = bytes;
+ num_fs_ssbo = MAX2(num_fs_ssbo, r->index + 1);
+ } else {
+ fprintf(stderr, "air_run: %s binds a device buffer at %u, past the %zu this "
+ "path keeps room for\n", name, r->index, ARRAY_SIZE(fsb));
+ free(bits);
+ return 1;
+ }
+ } else {
+ pipe->set_constant_buffer(pipe, PIPE_SHADER_FRAGMENT, r->index, false,
+ &(struct pipe_constant_buffer){
+ .buffer = buf, .buffer_size = bytes,
+ });
+ }
if (from_file)
printf("air_run: %s buffer at index %u, %u bytes, %u of them from "
"AIR_RUN_BUFFER\n",
@@ -1089,6 +2259,8 @@ main(int argc, char **argv)
r->index, bytes);
free(bits);
}
+ if (num_fs_ssbo)
+ pipe->set_shader_buffers(pipe, PIPE_SHADER_FRAGMENT, 0, num_fs_ssbo, fsb, 0);
/* every sampled texture the shader declares, and a sampler for it */
unsigned num_tex = 0;
@@ -1107,6 +2279,32 @@ main(int argc, char **argv)
*/
enum pipe_texture_target target = PIPE_TEXTURE_2D;
enum glsl_sampler_dim dim = GLSL_SAMPLER_DIM_2D;
+ bool is_array = false;
+ /* A SHADOW SAMPLE WANTS A DEPTH TEXTURE, and binding a colour one is not a
+ * harmless approximation: GL and Vulkan both leave a shadow lookup on a
+ * non-depth texture undefined, and the Adreno says so by ignoring the
+ * comparison. The first device run of these cases came back with the TEXEL
+ * -- 101 where a comparison must answer 0 or 1, and for `always` and `never`
+ * too, which no compare function and no clamp can explain -- against
+ * r32g32b32a32_float bound for a Metal depth2d<float>, which is the one thing
+ * in that run that was this harness's own doing (docs/air.md 3.3).
+ *
+ * So a slot a shadow instruction reads gets a real depth texture, and its
+ * texels are the only pattern that fits: (x + TEX*y) / 16, which is sixteen
+ * distinct values, every one exactly representable in binary32 and inside the
+ * [0, 1] a depth format is normalised to. The colour pattern cannot be used --
+ * 100 does not fit in a depth texel at all.
+ */
+ bool shadow = false;
+ nir_foreach_function_impl(impl, fs) {
+ nir_foreach_block(block, impl) {
+ nir_foreach_instr(instr, block) {
+ if (instr->type == nir_instr_type_tex &&
+ nir_instr_as_tex(instr)->is_shadow)
+ shadow = true;
+ }
+ }
+ }
nir_foreach_function_impl(impl, fs) {
nir_foreach_block(block, impl) {
nir_foreach_instr(instr, block) {
@@ -1122,6 +2320,7 @@ main(int argc, char **argv)
tex_is_float = false;
}
dim = t->sampler_dim;
+ is_array = t->is_array;
if (dim == GLSL_SAMPLER_DIM_3D)
target = PIPE_TEXTURE_3D;
else if (dim == GLSL_SAMPLER_DIM_MS)
@@ -1131,77 +2330,245 @@ main(int argc, char **argv)
}
}
have_format:;
+ /* and a depth format where a comparison reads it, whatever the dest_type said:
+ * a shadow sample's result is a comparison and not a texel, so the element type
+ * the instruction carries says nothing about the texture's format.
+ */
+ if (halftex) {
+ if (dim != GLSL_SAMPLER_DIM_2D || is_array) {
+ fprintf(stderr, "air_run: --halftex binds a 2D table and %s samples "
+ "something else\n", name);
+ return 1;
+ }
+ tex_format = PIPE_FORMAT_R16G16B16A16_FLOAT;
+ tex_is_float = true;
+ }
+ if (packed16_fmt != PIPE_FORMAT_NONE || miplevel_fmt != PIPE_FORMAT_NONE) {
+ if (dim != GLSL_SAMPLER_DIM_2D || is_array) {
+ fprintf(stderr, "air_run: --packed16 and --miplevels sample a 2D texture "
+ "and %s is not one\n", name);
+ return 1;
+ }
+ tex_format = packed16_fmt != PIPE_FORMAT_NONE ? packed16_fmt : miplevel_fmt;
+ tex_is_float = true;
+ }
+ if (shadow) {
+ if (dim != GLSL_SAMPLER_DIM_2D || is_array) {
+ fprintf(stderr, "air_run: %s compares against a texture this harness makes no "
+ "depth version of (dim %u%s); it is not run rather than run "
+ "against a colour texture, which is what the Adreno answers "
+ "with the texel\n", name, dim, is_array ? ", array" : "");
+ return 1;
+ }
+ tex_format = PIPE_FORMAT_Z32_FLOAT;
+ tex_is_float = true;
+ }
if (dim == GLSL_SAMPLER_DIM_MS) {
fprintf(stderr, "air_run: %s samples a multisample texture; this harness binds "
"none, so it is not run rather than run against the wrong thing\n",
name);
return 1;
}
- if (dim != GLSL_SAMPLER_DIM_2D && dim != GLSL_SAMPLER_DIM_3D) {
+ if (dim != GLSL_SAMPLER_DIM_1D && dim != GLSL_SAMPLER_DIM_2D &&
+ dim != GLSL_SAMPLER_DIM_3D) {
fprintf(stderr, "air_run: %s samples a texture of a kind this harness does not "
"bind (dim %u)\n", name, dim);
return 1;
}
- unsigned depth = target == PIPE_TEXTURE_3D ? TEX_SIZE : 1;
- unsigned tex_z = target == PIPE_TEXTURE_3D ? TEX_Y : 0;
+
+ /* An ARRAY texture is a different resource and not a 2D one sampled with
+ * a third coordinate, so the harness has to make one or the check would
+ * be about nothing: a layer index handed to a 2D texture is ignored, and
+ * every layer would answer with layer 0's texels. TEX_SIZE layers, filled
+ * by the same rule as a 3D texture's slices -- texel (x, y, z) of slot n
+ * is (100+x+1000n, 200+y, 300+x+y+z, 400) -- so a lowering that dropped
+ * the layer, or put it in the wrong component, comes back with a number
+ * from the wrong slice rather than a plausible one.
+ *
+ * Gallium addresses an array's layers through pipe_box.z and .depth, the
+ * same fields a 3D texture's slices use, while the RESOURCE carries them
+ * in array_size rather than depth0 -- which is why depth0 and the box's
+ * depth part company here.
+ */
+ if (is_array)
+ target = dim == GLSL_SAMPLER_DIM_1D ? PIPE_TEXTURE_1D_ARRAY
+ : PIPE_TEXTURE_2D_ARRAY;
+ else if (dim == GLSL_SAMPLER_DIM_1D)
+ target = PIPE_TEXTURE_1D;
+ unsigned height = dim == GLSL_SAMPLER_DIM_1D ? 1 : TEX_SIZE;
+ unsigned slices = (target == PIPE_TEXTURE_3D || is_array) ? TEX_SIZE : 1;
+ unsigned tex_z = slices > 1 ? TEX_Y : 0;
struct pipe_resource ttempl = {
.target = target,
.format = tex_format,
- .width0 = TEX_SIZE, .height0 = TEX_SIZE, .depth0 = depth, .array_size = 1,
+ .width0 = TEX_SIZE, .height0 = height,
+ .depth0 = is_array ? 1 : slices,
+ .array_size = is_array ? slices : 1,
.bind = PIPE_BIND_SAMPLER_VIEW,
};
- struct pipe_box box = { .width = TEX_SIZE, .height = TEX_SIZE, .depth = depth };
+ if (halftex) {
+ ttempl.width0 = HALFTEX_W;
+ ttempl.height0 = 1;
+ }
+ if (miplevel_fmt != PIPE_FORMAT_NONE) {
+ /* the shape the census found, not this harness's 4x4: a two-byte format
+ * differs from a four-byte one most in how its levels are laid out, and
+ * 776x24 is where the window server asks for it
+ */
+ ttempl.width0 = MIPTEX_W;
+ ttempl.height0 = MIPTEX_H;
+ ttempl.last_level = MIPTEX_LEVELS - 1;
+ ttempl.bind |= PIPE_BIND_RENDER_TARGET;
+ }
+ struct pipe_box box = { .width = TEX_SIZE, .height = height, .depth = slices };
struct pipe_resource *texes[PIPE_MAX_SAMPLERS] = { 0 };
for (unsigned slot = 0; slot < num_tex && slot < PIPE_MAX_SAMPLERS; slot++) {
float texels[TEX_SIZE * TEX_SIZE * TEX_SIZE * 4];
int32_t itexels[TEX_SIZE * TEX_SIZE * TEX_SIZE * 4];
- for (unsigned z = 0; z < depth; z++) {
- for (unsigned y = 0; y < TEX_SIZE; y++) {
+ texes[slot] = screen->resource_create(screen, &ttempl);
+ if (halftex) {
+ uint16_t words[HALFTEX_W * 4];
+ for (unsigned i = 0; i < HALFTEX_W; i++)
+ halftex_texel(i, &words[i * 4]);
+ struct pipe_box hb = { .width = HALFTEX_W, .height = 1, .depth = 1 };
+ pipe->texture_subdata(pipe, texes[slot], 0, PIPE_MAP_WRITE, &hb, words,
+ HALFTEX_W * 4 * sizeof(uint16_t),
+ HALFTEX_W * 4 * sizeof(uint16_t));
+ printf("air_run: %ux1 r16g16b16a16_float table, texel i holding the half "
+ "BIT PATTERNS ((i<<7)&0x7bff, that|0x8000, i, 0x3c00) written as "
+ "integers -- no float-to-half conversion anywhere in this program\n",
+ HALFTEX_W);
+ continue;
+ }
+ if (miplevel_fmt != PIPE_FORMAT_NONE) {
+ /* every level filled with the binary code of its own number, packed
+ * through util_format so the harness never writes bits by hand
+ */
+ for (unsigned lv = 0; lv < MIPTEX_LEVELS; lv++) {
+ unsigned w = MAX2(1u, MIPTEX_W >> lv), h = MAX2(1u, MIPTEX_H >> lv);
+ unsigned texel = util_format_get_blocksize(tex_format);
+ uint8_t *rows = calloc((size_t)w * h, texel);
+ float v[4];
+ miplevel_texel(lv, v);
+ /* ONE texel packed and then repeated. util_format_pack_rgba reads w
+ * TEXELS from its source, not one -- handing it a single colour and a
+ * width of 776 reads 776 texels out of a four-float array, which is
+ * how the first version of this segfaulted.
+ */
+ uint8_t one[16] = { 0 };
+ util_format_pack_rgba(tex_format, one, v, 1);
+ for (size_t k = 0; k < (size_t)w * h; k++)
+ memcpy(rows + k * texel, one, texel);
+ struct pipe_box lb = { .width = w, .height = h, .depth = 1 };
+ pipe->texture_subdata(pipe, texes[slot], lv, PIPE_MAP_WRITE, &lb, rows,
+ w * texel, (size_t)w * h * texel);
+ free(rows);
+ }
+ printf("air_run: %ux%u %s pyramid, %u levels, level L filled with the "
+ "binary code of L (r=bit0 g=bit1 b=bit2 a=bit3)\n", MIPTEX_W,
+ MIPTEX_H, util_format_short_name(tex_format), MIPTEX_LEVELS);
+ continue;
+ }
+ if (packed16_fmt != PIPE_FORMAT_NONE) {
+ /* one 16-bit word in every texel, so no coordinate, filter or address
+ * mode can change which channel answers
+ */
+ uint16_t words[TEX_SIZE * TEX_SIZE];
+ for (unsigned k = 0; k < TEX_SIZE * TEX_SIZE; k++)
+ words[k] = (uint16_t)packed16_word;
+ pipe->texture_subdata(pipe, texes[slot], 0, PIPE_MAP_WRITE, &box, words,
+ TEX_SIZE * sizeof(uint16_t),
+ TEX_SIZE * TEX_SIZE * sizeof(uint16_t));
+ continue;
+ }
+ if (shadow) {
+ /* one float per texel, not four, and a depth row is a quarter as wide */
+ for (unsigned y = 0; y < TEX_SIZE; y++)
+ for (unsigned x = 0; x < TEX_SIZE; x++)
+ texels[y * TEX_SIZE + x] = depth_texel_value(x, y);
+ pipe->texture_subdata(pipe, texes[slot], 0, PIPE_MAP_WRITE, &box, texels,
+ TEX_SIZE * sizeof(float),
+ TEX_SIZE * TEX_SIZE * sizeof(float));
+ continue;
+ }
+ for (unsigned z = 0; z < slices; z++) {
+ for (unsigned y = 0; y < height; y++) {
for (unsigned x = 0; x < TEX_SIZE; x++) {
- unsigned i = ((z * TEX_SIZE + y) * TEX_SIZE + x) * 4;
+ unsigned i = ((z * height + y) * TEX_SIZE + x) * 4;
texel_value(slot, x, y, z, &texels[i]);
for (unsigned c = 0; c < 4; c++)
itexels[i + c] = (int32_t)texels[i + c];
}
}
}
- texes[slot] = screen->resource_create(screen, &ttempl);
pipe->texture_subdata(pipe, texes[slot], 0, PIPE_MAP_WRITE, &box,
tex_is_float ? (const void *)texels : (const void *)itexels,
TEX_SIZE * 4 * sizeof(float),
- TEX_SIZE * TEX_SIZE * 4 * sizeof(float));
+ height * TEX_SIZE * 4 * sizeof(float));
}
- void *sampler = pipe->create_sampler_state(pipe, &(struct pipe_sampler_state){
- .wrap_s = PIPE_TEX_WRAP_CLAMP_TO_EDGE,
- .wrap_t = PIPE_TEX_WRAP_CLAMP_TO_EDGE,
- .wrap_r = PIPE_TEX_WRAP_CLAMP_TO_EDGE,
- .min_img_filter = PIPE_TEX_FILTER_NEAREST,
- .mag_img_filter = PIPE_TEX_FILTER_NEAREST,
- .min_mip_filter = PIPE_TEX_MIPFILTER_NONE,
- });
+ /* the API's sampler where the shader declares none, and its own where it
+ * does -- per slot, since a shader may mix the two
+ */
struct pipe_sampler_view *views[PIPE_MAX_SHADER_SAMPLER_VIEWS] = { 0 };
void *samplers[PIPE_MAX_SAMPLERS] = { 0 };
+ bool compare_slot[PIPE_MAX_SAMPLERS] = { 0 };
+ compare_sampled_slots(fs, compare_slot, PIPE_MAX_SAMPLERS);
+ if (compare_without_a_function(&info, compare_slot, PIPE_MAX_SAMPLERS, name))
+ return 1;
for (unsigned i = 0; i < num_tex && i < PIPE_MAX_SAMPLERS; i++) {
struct pipe_sampler_view vtempl = { 0 };
u_sampler_view_default_template(&vtempl, texes[i], texes[i]->format);
views[i] = pipe->create_sampler_view(pipe, texes[i], &vtempl);
- samplers[i] = sampler;
+ const struct air_sampler_state *own = NULL;
+ struct pipe_sampler_state sst = sampler_for_slot(&info, i, false,
+ compare_slot[i], &own);
+ samplers[i] = pipe->create_sampler_state(pipe, &sst);
+ if (own)
+ print_own_sampler(i, own, compare_slot[i]);
}
pipe->set_sampler_views(pipe, PIPE_SHADER_FRAGMENT, 0, num_tex, 0, false, views);
pipe->bind_sampler_states(pipe, PIPE_SHADER_FRAGMENT, 0, num_tex, samplers);
float want[4];
texel_value(0, TEX_X, TEX_Y, tex_z, want);
- printf("air_run: %u texture%s bound: %ux%ux%u %s, texel (x,y,z) of slot n = "
+ if (miplevel_fmt != PIPE_FORMAT_NONE)
+ printf("air_run: %u texture%s bound: %ux%u %s, %u mip levels, level L holding "
+ "the binary code of L+1 -- so the four numbers below NAME the level "
+ "that answered, and 0 0 0 0 is no level at all\n", num_tex, num_tex == 1 ? "" : "s",
+ MIPTEX_W, MIPTEX_H, util_format_short_name(tex_format), MIPTEX_LEVELS);
+ else if (shadow)
+ printf("air_run: %u DEPTH texture%s bound: %ux%u %s, texel (x,y) = (x + 4y)/16, "
+ "because a shadow sample of a colour texture is undefined and the Adreno "
+ "answers it with the texel\n", num_tex, num_tex == 1 ? "" : "s",
+ TEX_SIZE, TEX_SIZE, util_format_short_name(tex_format));
+ else
+ printf("air_run: %u texture%s bound: %ux%ux%u %s%s, texel (x,y,z) of slot n = "
"(100+x+1000n, 200+y, 300+x+y+z, 400), nearest, clamped\n",
- num_tex, num_tex == 1 ? "" : "s", TEX_SIZE, TEX_SIZE, depth,
- util_format_short_name(tex_format));
- printf("air_run: expected %.9g %.9g %.9g %.9g (the centre of texel %u,%u,%u sampled "
- "once)\n", (double)want[0], (double)want[1], (double)want[2], (double)want[3],
- TEX_X, TEX_Y, tex_z);
+ num_tex, num_tex == 1 ? "" : "s", TEX_SIZE, height, slices,
+ util_format_short_name(tex_format),
+ is_array ? " array (z is the layer)" : "");
+ if (!shadow && miplevel_fmt == PIPE_FORMAT_NONE &&
+ packed16_fmt == PIPE_FORMAT_NONE)
+ printf("air_run: expected %.9g %.9g %.9g %.9g (the centre of texel %u,%u,%u "
+ "sampled once)\n", (double)want[0], (double)want[1], (double)want[2],
+ (double)want[3], TEX_X, TEX_Y, tex_z);
+ /* A comparison's result is not a texel, so no "expected" line is printed for one:
+ * the first device run of these cases printed the colour pattern's texel there and
+ * the GPU then answered WITH that texel, which read as agreement until the numbers
+ * were looked at. The line said what the comparison reads and the device said what
+ * it returned, and they were the same sentence.
+ */
+ for (unsigned i = 0; i < num_tex && i < PIPE_MAX_SAMPLERS; i++)
+ if (compare_slot[i]) {
+ printf("air_run: sampler %u COMPARES against depth texel (%u,%u) = %.9g, so "
+ "the result is a 0, a 1 or a filtered fraction of them and never a "
+ "texel\n", i, TEX_X, TEX_Y,
+ (double)depth_texel_value(TEX_X, TEX_Y));
+ break;
+ }
}
/* the quad */
@@ -1222,7 +2589,7 @@ main(int argc, char **argv)
pipe->bind_vertex_elements_state(pipe, velem);
nir_shader *vs = quad_vertex_shader(options, info.num_varyings);
- finalize_for_gallium(vs);
+ finalize_for_gallium(vs, screen);
void *vs_cso = pipe->create_vs_state(pipe, &(struct pipe_shader_state){
.type = PIPE_SHADER_IR_NIR, .ir.nir = vs,
});
@@ -1232,8 +2599,27 @@ main(int argc, char **argv)
});
pipe->bind_fs_state(pipe, fs_cso);
+ /* The clear colour is the PREVIOUS CONTENT of the attachment, and that is the
+ * only thing on this machine a framebuffer fetch can read: softpipe's TGSI
+ * interpreter has no FBFETCH at all, so the only driver here that can answer is
+ * the one on the phone, and there a fetch in a single draw reads what the clear
+ * put down. --clear turns the one number a fetch could return into a number
+ * CHOSEN, so a shader that returns the fetch unchanged is distinguishable from
+ * one that returns zero, which is what a fetch that read nothing returns.
+ */
union pipe_color_union clear = { .ui = { 0, 0, 0, 0 } };
- pipe->clear(pipe, PIPE_CLEAR_COLOR, NULL, &clear, 0.0, 0);
+ if (!clear_given) {
+ pipe->clear(pipe, PIPE_CLEAR_COLOR, NULL, &clear, 0.0, 0);
+ } else {
+ for (unsigned i = 0; i < num_rt; i++) {
+ const union pipe_color_union *c =
+ &clear_colour[MIN2(i, clear_count - 1)];
+ pipe->clear_render_target(pipe, fb.cbufs[i], c, 0, 0, WIDTH, HEIGHT, false);
+ printf("air_run: render target %u cleared to %.9g %.9g %.9g %.9g, which is "
+ "what a framebuffer fetch of it in this draw will read\n", i,
+ (double)c->f[0], (double)c->f[1], (double)c->f[2], (double)c->f[3]);
+ }
+ }
struct pipe_draw_info draw = {
.mode = MESA_PRIM_TRIANGLE_STRIP, .index_bounds_valid = true,
@@ -1241,7 +2627,11 @@ main(int argc, char **argv)
};
struct pipe_draw_start_count_bias sc = { .start = 0, .count = 4 };
pipe->draw_vbo(pipe, &draw, 0, NULL, &sc, 1);
- pipe->flush(pipe, NULL, 0);
+ if (!finish_or_say_why(screen, pipe, "draw")) {
+ fprintf(stderr, "air_run: %s: the draw did not finish, so no pixel here is "
+ "evidence\n", name);
+ return 1;
+ }
/* the pixel */
struct pipe_transfer *xfer;
@@ -1261,6 +2651,18 @@ main(int argc, char **argv)
want_disasm ? ", ir3's instructions are above" : "");
printf("air_run: the pixel at (0, 0) of render target %u came back as:\n",
read_rt);
+ if (rt_is_packed16) {
+ printf("air_run: packed16-rt %s: the shader's colour came back as the word "
+ "0x%04x\n", packed16_name, ((uint16_t *)pix)[0]);
+ pipe->texture_unmap(pipe, xfer);
+ printf("air_run: drew %ux%u into %s, %u render target%s\n", WIDTH, HEIGHT,
+ util_format_short_name(rt_format), num_rt, num_rt == 1 ? "" : "s");
+ pipe->destroy(pipe);
+ screen->destroy(screen);
+ ralloc_free(ctx);
+ glsl_type_singleton_decref();
+ return 0;
+ }
for (unsigned c = 0; c < 4; c++) {
if (rt_is_half)
printf(" %.9g\n", (double)_mesa_half_to_float(((uint16_t *)pix)[c]));
diff --git a/tools/mesa/air/air_stitch.c b/tools/mesa/air/air_stitch.c
new file mode 100644
index 0000000..e723ed7
--- /dev/null
+++ b/tools/mesa/air/air_stitch.c
@@ -0,0 +1,224 @@
+/*
+ * Copyright © 2026 Darwin on redwood contributors
+ * SPDX-License-Identifier: MIT
+ */
+
+/*
+ * A function-stitching graph, typed as text, and the functions it names.
+ *
+ * The graph itself is Metal's: `air_stitch_graph` in nir_air.h is
+ * MTLFunctionStitchingGraph field for field. What is here is the one thing
+ * Metal has no equivalent of -- a way to write a graph on a command line --
+ * and it is in the frontend's library rather than in one tool because every
+ * tool that touches stitching needs it: air2nir folds a graph, air_run
+ * dispatches one, and the Metal library's test will build the same graphs
+ * through the objective API, where they have to be the same graphs or the
+ * comparison is between two different things.
+ *
+ * "_pf_invertRed(in0); _pf_redContour(n0); _pf_textureDiff(n0, n1)"
+ *
+ * One node per `;', in dependency order, numbered by position. An argument is
+ * `inN' (the stitched function's own argument N), `nN' (node N's result),
+ * `texN' / `sampN' / `bufN' (a texture, a sampler or a constant buffer bound at
+ * index N), or `gridpos'. The output node is the last one unless the text ends
+ * with `=> nN'.
+ */
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "nir_air.h"
+#include "util/macros.h"
+#include "util/ralloc.h"
+
+/* Every refusal goes into the caller's buffer rather than to stderr: this is
+ * library code, and a tool that wants the sentence on one line with its own
+ * prefix cannot get it back out of stderr.
+ */
+static void PRINTFLIKE(3, 4)
+say(char *error, size_t size, const char *fmt, ...)
+{
+ if (!error || !size)
+ return;
+ va_list ap;
+ va_start(ap, fmt);
+ vsnprintf(error, size, fmt, ap);
+ va_end(ap);
+}
+
+static char *
+trim(char *s)
+{
+ while (*s == ' ' || *s == '\t')
+ s++;
+ char *e = s + strlen(s);
+ while (e > s && (e[-1] == ' ' || e[-1] == '\t'))
+ *--e = '\0';
+ return s;
+}
+
+static bool
+parse_arg(const char *tok, struct air_stitch_arg *out)
+{
+ static const struct { const char *prefix; enum air_stitch_arg_kind kind; } kinds[] = {
+ /* "in" before "n", or `in0' would read as node 0 of a prefix "n" */
+ { "in", AIR_STITCH_INPUT },
+ { "tex", AIR_STITCH_TEXTURE },
+ { "samp", AIR_STITCH_SAMPLER },
+ { "buf", AIR_STITCH_BUFFER },
+ { "n", AIR_STITCH_NODE },
+ };
+ if (!strcmp(tok, "gridpos")) {
+ *out = (struct air_stitch_arg){ .kind = AIR_STITCH_GRID_POSITION };
+ return true;
+ }
+ for (unsigned k = 0; k < ARRAY_SIZE(kinds); k++) {
+ size_t len = strlen(kinds[k].prefix);
+ if (strncmp(tok, kinds[k].prefix, len))
+ continue;
+ char *end = NULL;
+ unsigned long v = strtoul(tok + len, &end, 10);
+ if (!end || *end || end == tok + len)
+ continue;
+ *out = (struct air_stitch_arg){ .kind = kinds[k].kind, .index = (unsigned)v };
+ return true;
+ }
+ return false;
+}
+
+/* "f(a, b)" -> one node.
+ *
+ * strtok_r and not strtok, here and in the caller: strtok keeps one buffer for
+ * the whole program, so splitting a node's arguments inside the loop that
+ * splits the graph ENDED that loop -- a two-node graph parsed as one node and
+ * the shader stored node 0's result. That is exactly what a stitched call that
+ * had not been wired would look like, and it was the parser.
+ */
+static bool
+parse_node(void *ctx, char *text, struct air_stitch_node *node,
+ char *error, size_t size)
+{
+ text = trim(text);
+ char *open = strchr(text, '(');
+ char *close = strrchr(text, ')');
+ if (!open || !close || close < open) {
+ say(error, size, "`%s' is not `function(arguments)'", text);
+ return false;
+ }
+ *open = *close = '\0';
+ node->function = ralloc_strdup(ctx, trim(text));
+
+ char *args = open + 1;
+ unsigned n = 0;
+ bool empty = true;
+ for (char *p = args; *p; p++) {
+ n += *p == ',';
+ empty &= *p == ' ' || *p == '\t';
+ }
+ n = empty ? 0 : n + 1;
+
+ struct air_stitch_arg *out = rzalloc_array(ctx, struct air_stitch_arg, MAX2(n, 1));
+ unsigned idx = 0;
+ char *save = NULL;
+ for (char *tok = strtok_r(args, ",", &save); tok; tok = strtok_r(NULL, ",", &save)) {
+ tok = trim(tok);
+ if (idx >= n || !parse_arg(tok, &out[idx++])) {
+ say(error, size, "`%s' is not inN, nN, texN, sampN, bufN or gridpos", tok);
+ return false;
+ }
+ }
+ node->num_args = idx;
+ node->args = out;
+ return true;
+}
+
+const struct air_stitch_graph *
+air_stitch_parse(void *mem_ctx, const char *text, char *error, size_t size)
+{
+ char *copy = ralloc_strdup(mem_ctx, text);
+ struct air_stitch_graph *g = rzalloc(mem_ctx, struct air_stitch_graph);
+ g->name = "stitched";
+ g->output_node = ~0u;
+
+ char *arrow = strstr(copy, "=>");
+ if (arrow) {
+ *arrow = '\0';
+ char *tok = trim(arrow + 2);
+ char *e = tok + strlen(tok);
+ while (e > tok && e[-1] == ';')
+ *--e = '\0';
+ struct air_stitch_arg which;
+ if (!parse_arg(trim(tok), &which) || which.kind != AIR_STITCH_NODE) {
+ say(error, size, "`=> %s' is not `=> nN'", tok);
+ return NULL;
+ }
+ g->output_node = which.index;
+ }
+
+ unsigned count = 1;
+ for (const char *p = copy; *p; p++)
+ count += *p == ';';
+ struct air_stitch_node *nodes = rzalloc_array(mem_ctx, struct air_stitch_node, count);
+ unsigned n = 0;
+ char *save = NULL;
+ for (char *tok = strtok_r(copy, ";", &save); tok; tok = strtok_r(NULL, ";", &save)) {
+ if (!*trim(tok))
+ continue;
+ if (n >= count || !parse_node(mem_ctx, tok, &nodes[n++], error, size))
+ return NULL;
+ }
+ if (!n) {
+ say(error, size, "no nodes");
+ return NULL;
+ }
+ g->num_nodes = n;
+ g->nodes = nodes;
+ if (g->output_node == ~0u)
+ g->output_node = n - 1;
+ if (g->output_node >= n) {
+ say(error, size, "the output node is n%u of %u nodes", g->output_node, n);
+ return NULL;
+ }
+ return g;
+}
+
+unsigned
+air_stitch_functions_of(void *mem_ctx, const struct air_stitch_graph *g,
+ const void *data, size_t size,
+ struct air_stitch_function **out, char *error, size_t esize)
+{
+ struct air_metallib lib;
+ const char *why = NULL;
+ if (!air_metallib_parse(mem_ctx, data, size, &lib, &why)) {
+ say(error, esize, "the container: %s", why ? why : "?");
+ return 0;
+ }
+
+ struct air_stitch_function *fns =
+ rzalloc_array(mem_ctx, struct air_stitch_function, MAX2(g->num_nodes, 1));
+ unsigned num = 0;
+ for (unsigned i = 0; i < g->num_nodes; i++) {
+ const char *name = g->nodes[i].function;
+ bool have = false;
+ for (unsigned k = 0; k < num; k++)
+ have |= !strcmp(fns[k].name, name);
+ if (have)
+ continue; /* two nodes calling one function need it once */
+
+ const struct air_metallib_function *found = NULL;
+ for (unsigned k = 0; k < lib.num_functions; k++)
+ if (!strcmp(lib.functions[k].name, name))
+ found = &lib.functions[k];
+ if (!found) {
+ say(error, esize, "%s is not a function of this library", name);
+ return 0;
+ }
+ fns[num++] = (struct air_stitch_function){
+ found->name, found->bitcode, found->bitcode_size,
+ };
+ }
+ *out = fns;
+ return num;
+}
diff --git a/tools/mesa/air/air_to_nir.c b/tools/mesa/air/air_to_nir.c
index 26478c5..11a8db3 100644
--- a/tools/mesa/air/air_to_nir.c
+++ b/tools/mesa/air/air_to_nir.c
@@ -16,6 +16,7 @@
#include <stdio.h>
#include <string.h>
+#include "nir_constant_expressions.h"
#include "util/hash_table.h"
#include "util/u_math.h"
@@ -25,12 +26,348 @@ air_error(struct air_ctx *c, const char *fmt, ...)
if (!c->error[0]) {
va_list ap;
va_start(ap, fmt);
- vsnprintf(c->error, sizeof(c->error), fmt, ap);
+ int n = vsnprintf(c->error, sizeof(c->error), fmt, ap);
va_end(ap);
+
+ /* A REFUSAL THAT WAS CUT SAYS SO.
+ *
+ * This buffer is 256 bytes (air_private.h) and vsnprintf stops there in
+ * silence, so a longer sentence used to arrive mid-word with nothing to
+ * show for the rest -- which is this project's cardinal defect wearing a
+ * disguise: a message that reads as complete and is not. It is not
+ * hypothetical. The tile-shading refusal was written long enough to name
+ * both of its walls and every report printed it ending at
+ * "-featureProfile", so the half that said which wall decides was the
+ * half that vanished, and nothing anywhere said a word was missing.
+ *
+ * The marker carries the length vsnprintf WANTED, because that is the
+ * number a reader can act on: it says how much is gone and therefore
+ * whether the sentence should be shortened or the buffer grown. Widening
+ * the buffer is the other fix and is not this one -- air_ctx is in a
+ * header every object of the frontend includes, and a marker costs
+ * nothing to anyone who is not already over the limit.
+ */
+ if (n >= (int)sizeof(c->error)) {
+ char *tail = c->error + sizeof(c->error) - 20;
+ snprintf(tail, 20, " ...[cut, %d]", n);
+ }
}
return false;
}
+void
+air_note(struct air_ctx *c, const char *fmt, ...)
+{
+ struct air_shader_info *info = c->info;
+ va_list ap;
+ va_start(ap, fmt);
+ char *text = ralloc_vasprintf(c->mem_ctx, fmt, ap);
+ va_end(ap);
+ /* the same note twice says nothing twice: two resources in one group, or a
+ * callee translated for two call sites, would each add one */
+ for (unsigned i = 0; i < info->num_notes; i++)
+ if (!strcmp(info->notes[i], text))
+ return;
+ info->notes = reralloc(c->mem_ctx, info->notes, const char *, info->num_notes + 1);
+ info->notes[info->num_notes++] = text;
+}
+
+/* -- constants the builder did not fold ---------------------------------- */
+
+/* Is this def a constant expression, and what is its value?
+ *
+ * nir_builder does NOT fold: nir_build_alu inserts the instruction whatever
+ * its sources are (nir_builder.c), so `icmp eq i8 %c, 0` with %c an immediate
+ * is an ALU instruction here and nir_src_is_const says no. That matters
+ * because the two decisions this is for -- which edge of a branch exists, and
+ * which side of a `select` between two pointers is taken -- are made while
+ * translating, long before nir_opt_constant_folding runs, and a branch left in
+ * is a phi of pointers left unresolved.
+ *
+ * THE SEMANTICS ARE NIR'S OWN. Every operation is evaluated by
+ * nir_eval_const_opcode with this shader's float controls, which is the table
+ * and the mode nir_opt_constant_folding itself uses, so nothing here can
+ * disagree with the folder that runs after. What this adds to try_fold_alu is
+ * only recursion: that one requires every source to be a load_const, which a
+ * chain of an immediate and two operations never is.
+ */
+static bool
+def_const(nir_def *def, nir_const_value out[NIR_MAX_VEC_COMPONENTS],
+ unsigned depth, unsigned float_controls)
+{
+ if (!def || depth == 0 || def->num_components > NIR_MAX_VEC_COMPONENTS)
+ return false;
+
+ if (def->parent_instr->type == nir_instr_type_load_const) {
+ nir_load_const_instr *lc = nir_instr_as_load_const(def->parent_instr);
+ for (unsigned i = 0; i < def->num_components; i++)
+ out[i] = lc->value[i];
+ return true;
+ }
+ if (def->parent_instr->type != nir_instr_type_alu)
+ return false;
+
+ nir_alu_instr *alu = nir_instr_as_alu(def->parent_instr);
+ const nir_op_info *info = &nir_op_infos[alu->op];
+ nir_const_value src[NIR_ALU_MAX_INPUTS][NIR_MAX_VEC_COMPONENTS];
+ nir_const_value *srcs[NIR_ALU_MAX_INPUTS];
+
+ /* the unsized-type rule, copied from try_fold_alu rather than reasoned
+ * about: an op whose output type carries no size takes it from the first
+ * source that carries none either
+ */
+ unsigned bit_size = 0;
+ if (!nir_alu_type_get_type_size(info->output_type))
+ bit_size = alu->def.bit_size;
+
+ for (unsigned i = 0; i < info->num_inputs; i++) {
+ nir_const_value v[NIR_MAX_VEC_COMPONENTS];
+ if (!def_const(alu->src[i].src.ssa, v, depth - 1, float_controls))
+ return false;
+ if (bit_size == 0 &&
+ !nir_alu_type_get_type_size(info->input_types[i]))
+ bit_size = alu->src[i].src.ssa->bit_size;
+ for (unsigned j = 0; j < nir_ssa_alu_instr_src_components(alu, i); j++)
+ src[i][j] = v[alu->src[i].swizzle[j]];
+ srcs[i] = src[i];
+ }
+ if (bit_size == 0)
+ bit_size = 32;
+
+ memset(out, 0, sizeof(nir_const_value) * NIR_MAX_VEC_COMPONENTS);
+ nir_eval_const_opcode(alu->op, out, def->num_components, bit_size, srcs,
+ float_controls);
+ return true;
+}
+
+/* A one-component condition that is decided already. 16 is the recursion
+ * limit and not a judgement about shaders: the chains this is for are three
+ * instructions long (a load folded to an immediate, an icmp, the branch), and
+ * a limit is here so that a pathological module costs a refusal to fold and
+ * not a stack.
+ */
+/* The same question for an integer, which a switch asks of its selector and of
+ * every one of its case values. */
+static bool
+uint_const(struct air_ctx *c, nir_def *def, uint64_t *val)
+{
+ nir_const_value v[NIR_MAX_VEC_COMPONENTS];
+ if (!def || def->num_components != 1)
+ return false;
+ if (!def_const(def, v, 16, c->nir->info.float_controls_execution_mode))
+ return false;
+ *val = nir_const_value_as_uint(v[0], def->bit_size);
+ return true;
+}
+
+static bool
+cond_const(struct air_ctx *c, nir_def *def, bool *val)
+{
+ nir_const_value v[NIR_MAX_VEC_COMPONENTS];
+ if (!def || def->num_components != 1)
+ return false;
+ if (!def_const(def, v, 16, c->nir->info.float_controls_execution_mode))
+ return false;
+ *val = nir_const_value_as_bool(v[0], def->bit_size);
+ return true;
+}
+
+/* Two things the module-scope half of this file decides and the INSTRUCTION half
+ * needs: plan_global_folding's verdict for one global, and the name and
+ * required-ness of a function constant by index. Both are declared here because
+ * air_load_ptr and AIR_OP_{LOAD,STORE} come first in this file and the planner
+ * last, and a missing prototype in C is an implicit int that compiles with a
+ * warning and returns a pointer truncated to one.
+ */
+static struct air_global_plan *foldable_global_plan(struct air_ctx *c,
+ const struct air_value *v);
+static const char *function_constant_decl(struct air_ctx *c, unsigned index,
+ bool *required);
+
+/* -- translation-time slots ---------------------------------------------- */
+
+/* Does this type hold a pointer anywhere inside it? A slot exists for exactly
+ * the values that memory cannot hold, and that is what makes one necessary.
+ */
+static bool
+type_has_pointer(const struct air_type *t, unsigned depth)
+{
+ if (!t || depth == 0)
+ return false;
+ switch (t->kind) {
+ case AIR_TYPE_POINTER:
+ return true;
+ case AIR_TYPE_ARRAY:
+ case AIR_TYPE_VECTOR:
+ return type_has_pointer(t->elem, depth - 1);
+ case AIR_TYPE_STRUCT:
+ for (unsigned i = 0; i < t->num_members; i++)
+ if (type_has_pointer(t->members[i], depth - 1))
+ return true;
+ return false;
+ default:
+ return false;
+ }
+}
+
+static unsigned
+new_slot(struct air_ctx *c, const struct air_type *type, const struct air_value *init)
+{
+ c->slots = reralloc(c->mem_ctx, c->slots, struct air_slot, c->num_slots + 1);
+ c->slots[c->num_slots] = (struct air_slot) { .type = type, .init = init };
+ return c->num_slots++;
+}
+
+/* The value at byte offset 0 of a slot, for a load that wants `want`.
+ *
+ * Offset 0 of a struct or an array is offset 0 of its first element, and again
+ * of that element's first element, so descending is the whole of the
+ * arithmetic -- but only while the caller is asking for something that is not
+ * itself the aggregate. A load of the aggregate gets the aggregate.
+ */
+static struct air_val *
+slot_at_zero(struct air_val *v, const struct air_type *want)
+{
+ bool want_agg = want && (want->kind == AIR_TYPE_STRUCT ||
+ want->kind == AIR_TYPE_ARRAY);
+ while (!want_agg && v && v->kind == AIR_VAL_AGG && v->num_elems)
+ v = v->elems[0];
+ return v;
+}
+
+/* A pointer that IS a slot's base: class SLOT and offset zero. A slot holds a
+ * tree of values and not bytes, so an offset into one is not something to
+ * resolve -- `gep_into` may well have produced one (a getelementptr past the
+ * first member), and the honest answer to that is the refusal at the load.
+ */
+static bool
+slot_base(struct air_ctx *c, const struct air_val *v, unsigned *index)
+{
+ if (!v || v->kind != AIR_VAL_PTR || v->ptr.class != AIR_PTR_SLOT)
+ return false;
+ if (v->ptr.offset &&
+ (!nir_src_is_const(nir_src_for_ssa(v->ptr.offset)) ||
+ nir_src_as_uint(nir_src_for_ssa(v->ptr.offset)) != 0))
+ return false;
+ if (v->ptr.binding >= c->num_slots)
+ return false;
+ *index = v->ptr.binding;
+ return true;
+}
+
+static struct air_val *air_val_agg(struct air_ctx *c, unsigned n);
+
+/* A pointer into a slot at a byte offset the translator knows: class SLOT, a
+ * binding in range, and an offset that is constant or absent.
+ */
+static bool
+slot_at_const(struct air_ctx *c, const struct air_val *v, unsigned *index,
+ uint64_t *off)
+{
+ if (!v || v->kind != AIR_VAL_PTR || v->ptr.class != AIR_PTR_SLOT)
+ return false;
+ if (v->ptr.binding >= c->num_slots)
+ return false;
+ if (v->ptr.offset) {
+ if (!nir_src_is_const(nir_src_for_ssa(v->ptr.offset)))
+ return false;
+ *off = nir_src_as_uint(nir_src_for_ssa(v->ptr.offset));
+ } else {
+ *off = 0;
+ }
+ *index = v->ptr.binding;
+ return true;
+}
+
+/* THE CELL OF A SLOT'S TREE THAT A BYTE OFFSET AND A TYPE NAME TOGETHER.
+ *
+ * A slot holds a whole air_val because a pointer in this frontend is a
+ * resolution and not bytes (air_private.h, AIR_PTR_SLOT), and until now only
+ * offset zero could reach one: a `getelementptr` past the first member
+ * produced a byte offset, and a tree of values has no bytes for it to mean.
+ * That is still true of a byte offset ALONE, and it is why this takes a type
+ * as well. Offset 4 of `{ {ptr}, {ptr}, ptr, <2 x float> }` is the second
+ * member, the second member's first member, and nothing else -- but only
+ * because the store says it is storing a pointer. The same offset with
+ * `<2 x float>` would land nowhere and this returns NULL.
+ *
+ * So the rule is: descend while the type at hand is not the type asked for,
+ * taking at each level the one member or element whose byte range contains the
+ * remaining offset, and refuse unless the descent ends exactly on the asked
+ * type at offset zero. Types are interned per module, so `==` is the whole
+ * comparison. A partial overlap -- half of a pointer, two members at once --
+ * never matches and is refused by name at the caller, which is the behaviour
+ * this replaces and not a behaviour it adds.
+ *
+ * What the four `disparity_refinement_*` of CIPortraitBlurStitchableV3 look
+ * like, which is what this exists for: CoreImage hands a stitched function a
+ * `coreimage::Sampler` flattened into four arguments, and the function's first
+ * act is to `alloca` the struct back and store each argument into its field.
+ * Three of those fields are a texture, a sampler and a constant buffer -- the
+ * things a slot exists to hold -- and the fourth is a float2, which a slot can
+ * hold too and could not reach.
+ *
+ * Levels that nothing has written yet are made here rather than refused, for
+ * the reason agg_slot gives: an aggregate whose every field is NONE *is* the
+ * undef of that type, so building one changes no value.
+ */
+static struct air_val **
+slot_cell(struct air_ctx *c, unsigned s, uint64_t off, const struct air_type *want)
+{
+ struct air_val **cell = &c->slots[s].val;
+ const struct air_type *t = c->slots[s].type;
+
+ /* A struct nested 32 deep is not a thing Metal's compiler emits; the bound
+ * is here so that a cyclic or malformed type cannot spin.
+ */
+ for (unsigned depth = 0; depth < 32; depth++) {
+ if (t && t == want && off == 0)
+ return cell;
+ if (!t)
+ return NULL;
+
+ unsigned index, count;
+ const struct air_type *et;
+ if (t->kind == AIR_TYPE_STRUCT) {
+ count = t->num_members;
+ index = count;
+ uint64_t base = 0;
+ for (unsigned i = 0; i < count; i++) {
+ uint64_t mo = air_struct_member_offset(c->m, t, i);
+ uint64_t ms = air_type_alloc_size(c->m, t->members[i]);
+ if (off >= mo && off < mo + ms) {
+ index = i;
+ base = mo;
+ break;
+ }
+ }
+ if (index == count)
+ return NULL;
+ et = t->members[index];
+ off -= base;
+ } else if (t->kind == AIR_TYPE_ARRAY) {
+ uint64_t es = air_type_alloc_size(c->m, t->elem);
+ if (!es || off / es >= t->count)
+ return NULL;
+ count = t->count > UINT_MAX ? UINT_MAX : (unsigned)t->count;
+ index = (unsigned)(off / es);
+ et = t->elem;
+ off -= (uint64_t)index * es;
+ } else {
+ return NULL; /* a scalar, a vector or a pointer: no further in */
+ }
+
+ if (!*cell)
+ *cell = air_val_agg(c, count);
+ if (!*cell || (*cell)->kind != AIR_VAL_AGG || index >= (*cell)->num_elems)
+ return NULL;
+ cell = &(*cell)->elems[index];
+ t = et;
+ }
+ return NULL;
+}
+
+
/* -- values ------------------------------------------------------------- */
struct air_val *
@@ -51,6 +388,53 @@ air_val_ptr(struct air_ctx *c, struct air_ptr ptr)
return v;
}
+static struct air_val *
+air_val_ptr_select(struct air_ctx *c, nir_def *cond,
+ struct air_val *a, struct air_val *b)
+{
+ struct air_val *v = rzalloc(c->mem_ctx, struct air_val);
+ v->kind = AIR_VAL_PTR_SELECT;
+ v->def = cond;
+ v->num_elems = 2;
+ v->elems = rzalloc_array(c->mem_ctx, struct air_val *, 2);
+ v->elems[0] = a;
+ v->elems[1] = b;
+ return v;
+}
+
+/* Is this value a pointer in any of the three senses? */
+static bool
+val_is_pointerish(const struct air_val *v)
+{
+ return v && (v->kind == AIR_VAL_PTR || v->kind == AIR_VAL_PTR_SELECT);
+}
+
+/* Two resolutions that are the same resolution.
+ *
+ * This is what lets a `phi` or a `select` between pointers be answered without
+ * choosing anything at run time, and it is needed rather than merely tidy:
+ * LLVM duplicates a `getelementptr` onto two paths and then phis the copies
+ * back together, so 207 of the pointer phis in Resident Evil 4's library have
+ * two operands that name ONE member of ONE argument buffer. A pointer whose
+ * byte offset is not a constant is compared by the def itself -- the same def
+ * is the same offset, and two different defs cannot be proved equal here.
+ */
+static bool
+same_ptr(const struct air_ptr *a, const struct air_ptr *b)
+{
+ if (a->class != b->class || a->binding != b->binding || a->is_image != b->is_image)
+ return false;
+ if (a->offset == b->offset)
+ return true;
+ if (!a->offset || !b->offset)
+ return false;
+ if (!nir_src_is_const(nir_src_for_ssa(a->offset)) ||
+ !nir_src_is_const(nir_src_for_ssa(b->offset)))
+ return false;
+ return nir_src_as_uint(nir_src_for_ssa(a->offset)) ==
+ nir_src_as_uint(nir_src_for_ssa(b->offset));
+}
+
static struct air_val *
air_val_agg(struct air_ctx *c, unsigned n)
{
@@ -67,11 +451,129 @@ air_set(struct air_ctx *c, const struct air_value *v, struct air_val *val)
_mesa_hash_table_insert(c->values, v, val);
}
+/* An integer of a width NIR does not have, and how wide it is.
+ *
+ * Metal's own compiler makes them. A function constant holding a 3-bit field
+ * is narrowed with `trunc i32 %x to i3` and then switched on, which is how
+ * four of RenderBox's shaders pick a blend mode; `i2` occurs the same way.
+ * NIR has 1, 8, 16, 32 and 64 bits and nothing between, so such a value is
+ * carried here in the next width up with the low N bits significant and
+ * everything above them ZERO -- canonical form.
+ *
+ * Canonical form makes exactly five things exact, and that is the whole list:
+ * a truncation INTO the type (mask the low N bits), a truncation or a
+ * zero-extension OUT of it, an unsigned conversion to float, an unsigned or
+ * equality comparison, and a switch -- which is equalities. A signed
+ * comparison, a sign extension, an addition, a multiplication and a shift all
+ * read or write the bits above N and would be wrong. narrow_int_gate refuses
+ * everything that is not on the list, by width and by opcode, because a wrong
+ * answer that nothing says is wrong is the one failure this frontend must not
+ * have.
+ */
+static unsigned
+narrow_int_bits(const struct air_type *t)
+{
+ if (!t)
+ return 0;
+ const struct air_type *s = air_type_scalar(t);
+ if (!s || s->kind != AIR_TYPE_INT)
+ return 0;
+ unsigned b = s->bits;
+ if (!b || b > 64 || b == 1 || b == 8 || b == 16 || b == 32 || b == 64)
+ return 0;
+ return b;
+}
+
+/* The NIR width such a value is carried in: 32 bits, or 64 for the widths
+ * above it.
+ *
+ * NOT the next width up, which was the first choice and is worse for two
+ * reasons measured rather than guessed. An `i3` carried in eight bits makes
+ * `trunc i32 %x to i3` a `u2u8` followed by a mask, and nir_to_tgsi has no
+ * u2u8 at all -- softpipe aborts on it ("Unknown NIR opcode: u2u8"), which
+ * removes the one reference on this machine that RUNS the shader rather than
+ * folding it. At 32 bits the truncation of a 32-bit word is the mask alone,
+ * which is fewer instructions as well as portable. Canonical form is exactly
+ * as exact in 32 bits as in 8: the low N bits significant, the rest zero.
+ */
+static unsigned
+narrow_promoted_bits(unsigned n)
+{
+ return n <= 32 ? 32 : 64;
+}
+
+static uint64_t
+narrow_mask(unsigned n)
+{
+ return n >= 64 ? ~(uint64_t)0 : ((uint64_t)1 << n) - 1;
+}
+
+/* The width a value of this type is CARRIED in, promoting a width NIR has no
+ * register for instead of refusing it.
+ *
+ * Only for the places narrow_int_gate governs the uses of -- a constant, a
+ * truncation into the type, a phi's register. air_nir_bit_size stays the
+ * default everywhere else and stays a refusal, because a type that needs a
+ * register whose upper bits MEAN something has no NIR width and promoting it
+ * would be wrong rather than inconvenient.
+ */
+static unsigned
+air_narrow_bits(struct air_ctx *c, const struct air_type *t)
+{
+ unsigned n = narrow_int_bits(t);
+ return n ? narrow_promoted_bits(n) : air_nir_bit_size(c, t);
+}
+
unsigned
air_nir_bit_size(struct air_ctx *c, const struct air_type *t)
{
unsigned bits = air_type_bit_size(air_type_scalar(t));
- /* NIR has no i1 in SSA: booleans are 1-bit defs, which it does have */
+ /* NIR has no i1 in SSA: booleans are 1-bit defs, which it does have.
+ *
+ * It has no 2- or 4-bit integer at all, though, and that is not a
+ * hypothetical: an HLSL bit field reaches AIR as `i2` or `i4` through
+ * SPIRV-Cross, and every NIR builder handed one ASSERTS instead of
+ * refusing -- "Invalid Boolean size", "Invalid bit size" and "Invalid nir
+ * alu bit size" between them aborted 495 of Resident Evil 4's modules the
+ * first time the argument-buffer interface let them get as far as their
+ * bodies. An abort is not a refusal: it says nothing a report can count and
+ * it takes the run with it. So the width is named here, once, and 32 is
+ * returned so that the instruction under construction stays well formed
+ * while the error just set discards the shader.
+ */
+ if (bits != 1 && bits != 8 && bits != 16 && bits != 32 && bits != 64) {
+ /* Zero is a different fact wearing the same symptom, and saying so in one
+ * message made 79 modules read as carrying a 0-bit integer when what they
+ * carry is a clip-distance ARRAY as a shader output: air_type_bit_size has
+ * no answer for an aggregate, and this used to return it unexamined, so
+ * `[4 x float]` became a one-component float output and four clip planes
+ * became one with nothing in any log. A width NIR cannot hold and a type
+ * that has no single width are both refusals, and they are not the same
+ * refusal.
+ */
+ if (!bits)
+ air_error(c, "instruction: %s where a number belongs, which has no single "
+ "NIR width",
+ t->kind == AIR_TYPE_ARRAY ? "an array" :
+ t->kind == AIR_TYPE_STRUCT ? "a struct" :
+ t->kind == AIR_TYPE_POINTER ? "a pointer" : "a value of no size");
+ else
+ /* Reaching HERE, and not narrow_int_gate, means the width turned up
+ * somewhere that needs a register of its own -- a function parameter,
+ * a shader output, a phi, a value stored to memory -- rather than in
+ * an instruction whose low N bits are all it reads. Canonical form
+ * (narrow_int_bits) does not help there: the bits above N would have
+ * to mean something, and NIR has no width in which they do. So the
+ * sentence names both halves, because "a width NIR does not have" on
+ * its own sent the reader looking for a missing case in the builder
+ * when what is missing is a representation.
+ */
+ air_error(c, "instruction: a %u-bit integer where NIR needs a register of its "
+ "own, a width it has no register for; it is carried in the next "
+ "width up only where nothing reads above bit %u",
+ bits, bits - 1);
+ return 32;
+ }
return bits;
}
@@ -81,6 +583,35 @@ air_nir_components(const struct air_type *t)
return t->kind == AIR_TYPE_VECTOR ? t->count : 1;
}
+/* The glsl type a NIR function parameter needs to carry a value of this AIR
+ * type, matching what air_nir_bit_size and air_nir_components say the def will
+ * be. NIR tracks a value's real type by use, so the only thing that has to be
+ * right here is the shape.
+ */
+static const struct glsl_type *
+param_glsl_type(struct air_ctx *c, const struct air_type *t)
+{
+ unsigned bits = air_nir_bit_size(c, t);
+ return glsl_vector_type(bits == 16 ? GLSL_TYPE_FLOAT16 :
+ bits == 64 ? GLSL_TYPE_UINT64 : GLSL_TYPE_UINT,
+ air_nir_components(t));
+}
+
+/* Whether a struct member is a number this frontend can hand through a
+ * function parameter -- a scalar or a vector of one. Everything else takes no
+ * parameter: an array, a nested struct, a pointer. In practice the one that
+ * occurs is the tail padding Metal's compiler puts on a vertex-output struct
+ * (`[8 x i8]`), which the shader never writes and the caller never reads.
+ */
+static bool
+ret_member_is_number(const struct air_type *m)
+{
+ if (m->kind != AIR_TYPE_VECTOR && m->kind != AIR_TYPE_INT && !air_type_is_float(m))
+ return false;
+ const struct air_type *sc = air_type_scalar(m);
+ return sc->kind == AIR_TYPE_INT || air_type_is_float(sc);
+}
+
static struct air_val *air_constant_val(struct air_ctx *c, const struct air_constant *k);
struct air_val *
@@ -91,7 +622,32 @@ air_get(struct air_ctx *c, const struct air_value *v)
return e->data;
const struct air_constant *k = air_value_as_constant(v);
if (k) {
+ /* A constant is built once and then reused from wherever it is asked
+ * for, so it must be somewhere that dominates all of those places. A
+ * constant has no operands, so the top of the entry block always will.
+ *
+ * Built at the cursor of its first use instead, a constant that LLVM
+ * shares between a conditional block and a block after it is a use that
+ * its def does not dominate -- and nothing complains, because
+ * nir_lower_goto_ifs runs nir_repair_ssa, which repairs such a use into
+ * a phi whose other edge is undef. That is how `phi i16 [ %30, %27 ],
+ * [ -1, %22 ]` in MetalFaceMask_AddSeedPoint became `phi b3: %92,
+ * b4: %0 (undefined)`: the -1 was the same constant as the one the
+ * compare after the branch uses, so the store on the entry edge pointed
+ * at a def inside the branch (docs/air.md section 5).
+ */
+ bool outer = !c->in_const;
+ nir_cursor use = c->b.cursor;
+ if (outer) {
+ c->in_const = true;
+ c->b.cursor = c->const_cursor;
+ }
struct air_val *val = air_constant_val(c, k);
+ if (outer) {
+ c->const_cursor = c->b.cursor;
+ c->b.cursor = use;
+ c->in_const = false;
+ }
if (val)
air_set(c, v, val);
return val;
@@ -113,7 +669,8 @@ air_get_ssa(struct air_ctx *c, const struct air_value *v)
return NULL;
if (val->kind != AIR_VAL_SSA) {
air_error(c, "instruction: %s where a number belongs",
- val->kind == AIR_VAL_PTR ? "a pointer" : "an aggregate");
+ val->kind == AIR_VAL_PTR ? "a pointer" :
+ val->kind == AIR_VAL_PTR_SELECT ? "a chosen pointer" : "an aggregate");
return NULL;
}
return val->def;
@@ -127,6 +684,28 @@ scalar_constant(struct air_ctx *c, const struct air_type *t, uint64_t bits)
unsigned n = air_type_bit_size(t);
if (t->kind == AIR_TYPE_INT && t->bits == 1)
return nir_imm_bool(&c->b, bits & 1);
+ unsigned narrow = narrow_int_bits(t);
+ if (narrow) {
+ /* Canonical form (narrow_int_bits): the low N bits, zero above them.
+ *
+ * The mask here is the SECOND guard and not the first. The reader
+ * already truncates an integer constant to its type's width
+ * (truncate_to in air_module.c), so `i3 -4` arrives as 4 and not as a
+ * sign-extended 0xfffc; removing this mask and rebuilding changes no
+ * number in tools/air/callargs.ll, which was checked rather than
+ * assumed. What makes the switch's case live is the mask on the
+ * TRUNCATION into the type, in translate_cast, and that one is
+ * load-bearing: without it `trunc i32 %w to i3` keeps eight bits, no
+ * case matches and the default is taken.
+ *
+ * It stays because the invariant belongs to both halves: a constant of
+ * this type must be canonical however it was built, and AIR_CONST_DATA
+ * carries its words from the record without passing through
+ * truncate_to.
+ */
+ return nir_imm_intN_t(&c->b, bits & narrow_mask(narrow),
+ narrow_promoted_bits(narrow));
+ }
if (air_type_is_float(t)) {
switch (n) {
case 16: return nir_imm_intN_t(&c->b, bits, 16);
@@ -177,7 +756,7 @@ air_constant_val(struct air_ctx *c, const struct air_constant *k)
}
return v;
}
- return air_val_ssa(c, nir_imm_zero(&c->b, air_nir_components(t), air_nir_bit_size(c, t)));
+ return air_val_ssa(c, nir_imm_zero(&c->b, air_nir_components(t), air_narrow_bits(c, t)));
case AIR_CONST_UNDEF:
case AIR_CONST_POISON:
@@ -191,7 +770,7 @@ air_constant_val(struct air_ctx *c, const struct air_constant *k)
struct air_ptr p = { .class = AIR_PTR_NONE, .pointee = t->elem };
return air_val_ptr(c, p);
}
- return air_val_ssa(c, nir_undef(&c->b, air_nir_components(t), air_nir_bit_size(c, t)));
+ return air_val_ssa(c, nir_undef(&c->b, air_nir_components(t), air_narrow_bits(c, t)));
case AIR_CONST_AGGREGATE: {
if (t->kind == AIR_TYPE_VECTOR) {
@@ -235,6 +814,56 @@ air_constant_val(struct air_ctx *c, const struct air_constant *k)
if (k->expr_op == AIR_OP_CAST && k->expr_sub == AIR_CAST_BITCAST &&
k->base.type->kind == AIR_TYPE_POINTER)
return air_get(c, k->operands[0]);
+ /* ... and a constant getelementptr, which the GL-on-Metal vertex shaders
+ * use to name the first element of a module array
+ * (`getelementptr inbounds ([2 x <4 x float>], ptr @g, i64 0, i64 0)`).
+ * It is the instruction form with every index a constant, so the whole
+ * walk is done here in integers and the result is the base pointer with
+ * a constant byte offset and a narrowed pointee.
+ */
+ if (k->expr_op == AIR_OP_GEP) {
+ if (k->num_operands < 2 || !k->src_type)
+ return air_error(c, "instruction: a constant getelementptr with no base "
+ "or no source type"), NULL;
+ struct air_val *base = air_get(c, k->operands[0]);
+ if (!base)
+ return NULL;
+ if (base->kind != AIR_VAL_PTR)
+ return air_error(c, "instruction: a constant getelementptr on something "
+ "that is not a pointer"), NULL;
+ struct air_ptr p = base->ptr;
+ const struct air_type *t = k->src_type;
+ uint64_t off = 0, idx = 0;
+ if (!air_const_uint(k->operands[1], &idx))
+ return air_error(c, "instruction: a constant getelementptr whose first "
+ "index is not a constant"), NULL;
+ off += idx * air_type_alloc_size(c->m, t);
+ for (unsigned i = 2; i < k->num_operands; i++) {
+ if (!air_const_uint(k->operands[i], &idx))
+ return air_error(c, "instruction: a constant getelementptr with an index "
+ "that is not a constant"), NULL;
+ if (t->kind == AIR_TYPE_STRUCT) {
+ if (idx >= t->num_members)
+ return air_error(c, "instruction: a constant getelementptr to struct "
+ "field %u of %u", (unsigned)idx,
+ t->num_members), NULL;
+ off += air_struct_member_offset(c->m, t, idx);
+ t = t->members[idx];
+ } else if (t->kind == AIR_TYPE_ARRAY || t->kind == AIR_TYPE_VECTOR) {
+ off += idx * (t->kind == AIR_TYPE_ARRAY
+ ? air_type_alloc_size(c->m, t->elem)
+ : MAX2(air_type_bit_size(t->elem) / 8, 1));
+ t = t->elem;
+ } else {
+ return air_error(c, "instruction: a constant getelementptr into a "
+ "scalar"), NULL;
+ }
+ }
+ p.pointee = t;
+ if (off)
+ p = air_ptr_add(c, p, nir_imm_int(&c->b, off));
+ return air_val_ptr(c, p);
+ }
return air_error(c, "instruction: constant expression %u", k->expr_op), NULL;
}
return air_error(c, "instruction: constant kind %u", k->ckind), NULL;
@@ -247,7 +876,40 @@ air_ptr_add(struct air_ctx *c, struct air_ptr ptr, nir_def *bytes)
{
if (!bytes)
return ptr;
- ptr.offset = ptr.offset ? nir_iadd(&c->b, ptr.offset, bytes) : bytes;
+ if (!ptr.offset) {
+ ptr.offset = bytes;
+ return ptr;
+ }
+ /* Two constant offsets become one constant, and that is a correctness fix
+ * rather than a tidiness.
+ *
+ * A cached constant is built in the const region at the top of the entry
+ * block (air_get), while a module-scope variable's byte offset is built
+ * where materialise_globals runs, which is BELOW that region. A constant
+ * `getelementptr` into a threadgroup array -- `load i32, ptr addrspace(3)
+ * getelementptr([16 x i32], ptr @tbl, i64 0, i64 9)`, which is how
+ * SPIRV-Cross reads a fixed slot of one -- is a constant whose value was
+ * therefore `iadd %globaloffset, 36` sitting above the def of
+ * %globaloffset: a use its def does not dominate, which nir_validate
+ * rejects with "src->_parent & SRC_TAG_SEEN". 115 of Resident Evil 4's
+ * modules are that shape, and every one of them ABORTED the translator the
+ * first time the argument-buffer interface let their bodies be reached.
+ *
+ * Folding the pair removes the reference: a global's offset is always a
+ * constant here, so the whole chain is one immediate built in the const
+ * region with no operand at all. The latent ordering hazard it stands on is
+ * written down in docs/air.md rather than left to be rediscovered.
+ */
+ if (nir_src_is_const(nir_src_for_ssa(ptr.offset)) &&
+ nir_src_is_const(nir_src_for_ssa(bytes)) &&
+ ptr.offset->bit_size == 32 && bytes->bit_size == 32 &&
+ ptr.offset->num_components == 1 && bytes->num_components == 1) {
+ uint32_t sum = (uint32_t)nir_src_as_uint(nir_src_for_ssa(ptr.offset)) +
+ (uint32_t)nir_src_as_uint(nir_src_for_ssa(bytes));
+ ptr.offset = nir_imm_int(&c->b, sum);
+ return ptr;
+ }
+ ptr.offset = nir_iadd(&c->b, ptr.offset, bytes);
return ptr;
}
@@ -301,14 +963,122 @@ air_load_ptr(struct air_ctx *c, struct air_ptr ptr, const struct air_type *type,
v = &c->options->constants[i];
if (!v)
return air_error(c, "interface: function constant %u has no value", ptr.binding), NULL;
- if (comps != 1)
- return air_error(c, "interface: a function constant of %u components", comps), NULL;
- res = nir_imm_intN_t(&c->b, v->bits, as_bool ? 8 : bits);
+ /* READ, AND NOBODY SUPPLIED IT. air_module_function_constants registers
+ * every constant a module DECLARES, with bits = 0 and supplied = false
+ * (nir_air.h), so a load here answers zero whether or not a pipeline ever
+ * chose one -- and this frontend now SPECIALISES on that zero
+ * (plan_global_folding), which turns it from a number in an expression
+ * into which blocks exist.
+ *
+ * air_metal.c says this on a refusal, with a count and two names; this is
+ * the other half, and it is here rather than there because it lands in
+ * the corpus report and so becomes countable instead of anecdotal. One
+ * note per constant per module (air_note drops a repeat), so the report's
+ * row for a name is the number of modules that read it unsupplied.
+ *
+ * Required-ness is in the sentence because it is what decides whether the
+ * zero is a defect or a legal default: a declaration ending `i1 true` is
+ * one Metal obliges the pipeline to supply, and `i1 false` is one the
+ * module is supposed to ask about with air.is_function_constant_defined
+ * first. The second is ordinary; the first means a pipeline left out
+ * something it had to give.
+ */
+ if (!v->supplied) {
+ bool required = false;
+ const char *fcname = function_constant_decl(c, ptr.binding, &required);
+ /* SHORT ON PURPOSE. 70 of these appear over the corpus, one per
+ * constant, and the row that matters is the name beside the number of
+ * modules -- a census. Repeating what it means 70 times would bury
+ * exactly that, so the consequence is said once in docs/air.md 3 and
+ * what is here is the fact.
+ */
+ air_note(c, "the function constant %s (index %u, %s) is read and no pipeline "
+ "supplied a value, so it was taken as zero",
+ fcname ? fcname : "(unnamed)", ptr.binding,
+ required ? "i1 true, which Metal obliges a pipeline to supply"
+ : "i1 false, so absent is legal");
+ }
+ if (comps == 1) {
+ res = nir_imm_intN_t(&c->b, v->bits, as_bool ? 8 : bits);
+ break;
+ }
+ /* A VECTOR function constant. Metal allows one and the GL-on-Metal
+ * fragment shaders use two: `uchar3` opcode triples for the texture
+ * combiners (`fc_color_opcodes0/1`, declared `<3 x i8>`).
+ *
+ * air_constant_value carries one 64-bit word per constant, so the
+ * components have to be fields of it, and the packing is this frontend's
+ * own interface with the runtime rather than something read out of
+ * Apple's: component i sits at bit i*width, which is the word a memcpy
+ * of the value would produce on this little-endian target. NOTHING HERE
+ * MEASURES THAT -- every harness in tools/air supplies zero for every
+ * constant, so all components are zero whatever the packing, and the
+ * first thing that would tell a wrong packing from a right one is a
+ * pipeline that sets one of these two constants.
+ */
+ if (comps * bits > 64)
+ return air_error(c, "interface: a function constant of %u components of %u bits, "
+ "which does not fit the one word air_constant_value carries",
+ comps, bits), NULL;
+ nir_def *fields[NIR_MAX_VEC_COMPONENTS];
+ for (unsigned i = 0; i < comps; i++) {
+ uint64_t mask = bits >= 64 ? ~0ull : (1ull << bits) - 1;
+ fields[i] = nir_imm_intN_t(&c->b, (v->bits >> (i * bits)) & mask,
+ as_bool ? 8 : bits);
+ }
+ res = nir_vec(&c->b, fields, comps);
break;
}
+ case AIR_PTR_ARGBUF:
+ /* argbuf_load answers a load through an argument buffer before it gets
+ * here, so this is a load that resolution could not make: through a
+ * pointer CHOSEN at run time between two sets, which load_through_choice
+ * would read from both of. There is nothing to read either way -- the
+ * bytes in an argument buffer are the runtime's 8-byte handles.
+ */
+ return air_error(c, "instruction: a load of bytes out of an argument buffer"), NULL;
+ case AIR_PTR_SLOT:
+ /* AIR_OP_LOAD answers a whole load of a slot before it gets here, so
+ * this is one that could not be answered: through a CHOSEN pointer, one
+ * of whose sides is a slot (load_through_choice), or at an offset into
+ * one. There is nothing to read either way -- a slot is a tree of values.
+ */
+ return air_error(c, "instruction: a load of bytes out of a local that holds a "
+ "pointer"), NULL;
+ case AIR_PTR_NONE:
+ /* A LOAD THROUGH A LITERAL `null`, which is undefined behaviour in the
+ * source and therefore has exactly one honest translation: undef. Not a
+ * value invented for it -- a declaration that there is no value, which
+ * is what the source said.
+ *
+ * AIR_PTR_NONE arises from nothing else. air_constant_val builds it only
+ * for AIR_CONST_NULL of a pointer type, so the module literally wrote
+ * `null` here; a binding this frontend failed to resolve is a different
+ * class and still refuses.
+ *
+ * Where it comes from is Metal's own contract rather than a mistake.
+ * fixed_frag_lph_cph has `%124 = select i1 %123, ptr addrspace(1) null,
+ * ptr addrspace(1) %14` on `fc_has_asg_77_weights`, and passes %124 to
+ * its asg77 filter -- so when no weights are supplied the pointer IS
+ * null, and the path that dereferences it is reachable only through a
+ * switch on a FragmentShaderConfig field that comes from a bound buffer.
+ * The application is what must not select that filter without supplying
+ * weights; this frontend cannot know that it did not, and refusing the
+ * module for a path the application is forbidden to take would refuse
+ * the shader QuartzCore asks this driver for more than any other.
+ *
+ * It leaves a note, because a shader that reads undef is a fact a reader
+ * of the report should be able to count.
+ */
+ air_note(c, "a load through a literal `null` pointer is undef: the module has a "
+ "path that dereferences null, Metal's contract is that the pipeline "
+ "must not take it (in the GL-on-Metal uber shaders it is a filter "
+ "whose weights buffer was not supplied), and undef is what the source "
+ "says rather than a value chosen here");
+ return nir_undef(&c->b, comps, bits);
default:
return air_error(c, "instruction: a load through a %s pointer",
- ptr.class == AIR_PTR_NONE ? "null" : "texture or sampler"), NULL;
+ "texture or sampler"), NULL;
}
return as_bool ? nir_ine_imm(&c->b, res, 0) : res;
}
@@ -340,6 +1110,17 @@ air_store_ptr(struct air_ctx *c, struct air_ptr ptr, nir_def *value, unsigned al
case AIR_PTR_UBO:
case AIR_PTR_CONSTANT_DATA:
return air_error(c, "instruction: a store into constant memory");
+ case AIR_PTR_ARGBUF:
+ /* Writing a descriptor from inside the shader. The handle it would write
+ * is the runtime's, and this frontend has resolved the set's members to
+ * bindings rather than to addresses, so there is nothing to write.
+ */
+ return air_error(c, "instruction: a store into an argument buffer");
+ case AIR_PTR_SLOT:
+ /* as in air_load_ptr: a whole store into a slot is answered by
+ * AIR_OP_STORE, so this one is at an offset or through a chosen pointer */
+ return air_error(c, "instruction: a store of bytes into a local that holds a "
+ "pointer");
default:
return air_error(c, "instruction: a store through a %s pointer",
ptr.class == AIR_PTR_NONE ? "null" : "texture or sampler");
@@ -384,6 +1165,20 @@ md_key_int(const struct air_md *node, const char *key, uint64_t *out)
return i >= 0 && md_int(node, i + 1, out);
}
+/* the operand after the key, when it is a nested node: the shape Apple uses for
+ * a key whose value is itself a list, as `air.function_constant` is
+ * (`!{ptr @__metal_implicit_fc_pred_6, !"bool", !"vtx_attr_has_texcoords"}`)
+ */
+static const struct air_md *
+md_key_node(const struct air_md *node, const char *key)
+{
+ int i = md_key(node, key);
+ if (i < 0 || (unsigned)(i + 1) >= node->num_ops || !node->ops[i + 1] ||
+ node->ops[i + 1]->kind != AIR_MD_NODE)
+ return NULL;
+ return node->ops[i + 1];
+}
+
static const char *
md_key_str(const struct air_md *node, const char *key)
{
@@ -502,6 +1297,7 @@ enum air_sysval {
AIR_SV_THREAD_POS_GRID, AIR_SV_THREAD_POS_TG, AIR_SV_TG_POS_GRID,
AIR_SV_THREADS_PER_TG, AIR_SV_TGS_PER_GRID, AIR_SV_THREAD_INDEX_TG,
AIR_SV_THREADS_PER_GRID, AIR_SV_SIMD_LANE, AIR_SV_SIMD_INDEX, AIR_SV_SIMD_WIDTH,
+ AIR_SV_SIMDS_PER_TG,
};
static const struct { const char *key; enum air_sysval sv; } sysvals[] = {
@@ -523,6 +1319,11 @@ static const struct { const char *key; enum air_sysval sv; } sysvals[] = {
{ "air.thread_index_in_simdgroup", AIR_SV_SIMD_LANE },
{ "air.simdgroup_index_in_threadgroup", AIR_SV_SIMD_INDEX },
{ "air.threads_per_simdgroup", AIR_SV_SIMD_WIDTH },
+ /* [[thread_execution_width]]: Metal's older name for the same number, the
+ * width of the SIMD-group the kernel runs in (Resident Evil 4's mainCS and
+ * CS_MiniClusterCompaction take it as an input) */
+ { "air.thread_execution_width", AIR_SV_SIMD_WIDTH },
+ { "air.simdgroups_per_threadgroup", AIR_SV_SIMDS_PER_TG },
};
static nir_def *
@@ -550,6 +1351,12 @@ load_sysval(struct air_ctx *c, enum air_sysval sv)
case AIR_SV_SIMD_LANE: return nir_load_subgroup_invocation(&c->b);
case AIR_SV_SIMD_INDEX: return nir_load_subgroup_id(&c->b);
case AIR_SV_SIMD_WIDTH: return nir_load_subgroup_size(&c->b);
+ /* ir3 has this one and computes it rather than reading a register:
+ * ir3_nir_lower_subgroup_id_cs turns load_num_subgroups into
+ * 1 + ((threads_per_threadgroup - 1) >> the SIMD-width shift), which folds
+ * to a constant whenever the workgroup size is one (ir3_nir.c).
+ */
+ case AIR_SV_SIMDS_PER_TG: return nir_load_num_subgroups(&c->b);
}
return NULL;
}
@@ -571,19 +1378,90 @@ fit(struct air_ctx *c, nir_def *def, const struct air_type *t)
return def;
}
-static void
+/* Returns the resource it added, so that a caller with more to say about it --
+ * an argument-buffer member, which carries the Metal (set, id) it came from --
+ * can fill the rest in; NULL when the node was refused.
+ */
+static struct air_resource *
add_resource(struct air_ctx *c, enum air_resource_kind kind, unsigned index, unsigned size,
bool read, bool write, unsigned param, const struct air_md *node)
{
struct air_shader_info *info = c->info;
info->resources = reralloc(c->mem_ctx, info->resources, struct air_resource,
info->num_resources + 1);
- info->resources[info->num_resources++] = (struct air_resource) {
+ /* Metal's `[[raster_order_group(n)]]`. Until this was read it was DROPPED IN
+ * SILENCE, and that is worth naming rather than just fixing: the branches of
+ * translate_input match on the resource's kind and return, so the loop that
+ * refuses an `air.*` key nobody handled never runs over a node whose kind is
+ * recognised. Every key in such a node is therefore invisible unless
+ * something looks for it by name -- which is how a shader could ask for
+ * ordering between overlapping fragments and get none, with nothing in any
+ * log saying so. `inplace_copy_{lpf,lph}` of QuartzCore's library ask for it
+ * on the texture they write.
+ */
+ const char *name = md_key_str(node, "air.arg_name");
+ uint64_t group = 0;
+ bool has_group = md_key_int(node, "air.raster_order_group", &group);
+ if (has_group) {
+ if (group >= 32) {
+ /* the bitmask has 32 bits; a group number past them would be recorded
+ * as a different group, which is worse than not being recorded */
+ air_error(c, "interface: raster order group %u", (unsigned)group);
+ return NULL;
+ }
+ info->raster_order_groups |= 1u << group;
+ /* What the hardware can do is not what was asked. The ask is per
+ * resource; a6xx's GRAS_SC_CNTL.SINGLE_PRIM_MODE is per draw, and
+ * freedreno drives it from pipe_blend_state.blend_coherent
+ * (fd6_emit.cc, build_prim_mode). Setting that bit orders the whole draw,
+ * which is MORE than the shader asked for -- safe, and slower. There is
+ * no way to give exactly the ask and no way to give less, so the shader
+ * is translated and the difference is said out loud here instead of being
+ * left to a document.
+ */
+ air_note(c, "raster order group %u on %s %u (%s): a6xx orders a whole draw or "
+ "nothing (GRAS_SC_CNTL.SINGLE_PRIM_MODE), never one resource, so the "
+ "runtime must set pipe_blend_state.blend_coherent and the ordering "
+ "given is coarser than the shader asked for",
+ (unsigned)group,
+ kind == AIR_RESOURCE_IMAGE ? "image" :
+ kind == AIR_RESOURCE_TEXTURE ? "texture" :
+ kind == AIR_RESOURCE_SAMPLER ? "sampler" : "buffer",
+ index, name ? name : "unnamed");
+ }
+
+ info->resources[info->num_resources] = (struct air_resource) {
.kind = kind, .index = index, .size = size, .read = read, .write = write,
.param = param,
- .name = md_key_str(node, "air.arg_name"),
+ .name = name,
.type_name = md_key_str(node, "air.arg_type_name"),
+ .raster_order_group = has_group ? (int)group : -1,
+ /* a direct argument: Metal names it by .index and there is no set */
+ .argbuf_set = ~0u, .argbuf_id = ~0u,
};
+ return &info->resources[info->num_resources++];
+}
+
+/* A BOOLEAN varying is an integer varying.
+ *
+ * No hardware interpolates one bit, and make_io_var gives such an output
+ * GLSL_TYPE_INT, which is 32 bits wide. Metal spells a `bool2` vertex output
+ * as `<2 x i1>` in the returned struct -- RenderBox's stroke shaders pass the
+ * two line caps that way, and the fragment stage takes them back as
+ * `<2 x i1>` and zero-extends them -- so the store converts to the variable's
+ * width and the load converts back.
+ *
+ * Both halves or neither: storing a 1-bit def into a 32-bit variable is what
+ * nir_validate aborts on ("src->ssa->bit_size & bit_sizes"), and reading the
+ * integer back as though it were already a boolean worked only because the
+ * integer happens to hold 0 or 1 -- an agreement by accident, through a
+ * `u2u8` that would have been a `b2i8`.
+ */
+static bool
+io_is_boolean(const struct air_type *t)
+{
+ const struct air_type *s = air_type_scalar(t);
+ return s->kind == AIR_TYPE_INT && s->bits == 1;
}
static nir_variable *
@@ -622,23 +1500,689 @@ add_varying(struct air_ctx *c, const char *name, unsigned slot, unsigned comps,
};
}
-static bool
-translate_input(struct air_ctx *c, const struct air_md *node, unsigned *next_varying)
+/* Metal's `[[stage_in]]` / `[[attribute(n)]]`: a vertex attribute, read out of
+ * a vertex buffer by the fetch hardware rather than computed by the shader.
+ *
+ * AIR HAS NO STAGE-IN STRUCT. Apple's compiler flattens it: each member is an
+ * ordinary parameter of the entry point carrying its own node, and the struct
+ * type does not appear in the module at all. QuartzCore's std_vert0_lpf is the
+ * shape, read out of the real library and not inferred:
+ *
+ * define <{ <4 x float>, <4 x float>, <2 x float> }> @std_vert0_lpf(
+ * <4 x float> %0, <2 x float> %1, <4 x float> %2, ptr addrspace(2) %3, ...)
+ * !19 = !{!20, !21, !22, !23, !24}
+ * !20 = !{i32 0, !"air.vertex_input", !"air.location_index", i32 0, i32 1,
+ * !"air.arg_type_name", !"float4", !"air.arg_name", !"position"}
+ * !21 = !{i32 1, !"air.vertex_input", !"air.location_index", i32 1, i32 1,
+ * !"air.arg_type_name", !"float2", !"air.arg_name", !"texcoord0",
+ * !"air.arg_unused"}
+ * !22 = !{i32 2, !"air.vertex_input", !"air.location_index", i32 3, i32 1,
+ * !"air.arg_type_name", !"float4", !"air.arg_name", !"color"}
+ *
+ * Three things in that are worth saying out loud, because each one would be
+ * guessed wrong:
+ *
+ * - the index is Metal's `[[attribute(n)]]` and NOT a position in the list.
+ * These three are 0, 1 and 3, so a translator that numbered them in order
+ * would feed `color` from attribute 2 -- a wrong vertex, drawn, with
+ * nothing to say so.
+ * - `air.arg_unused` occurs ON an attribute, and eight of QuartzCore's
+ * fourteen such shaders have one. Nothing reads it, so no NIR input is made:
+ * an input costs a VFD fetch slot and makes the pipeline demand a binding
+ * that the application's MTLVertexDescriptor may legally not have for an
+ * attribute the shader does not read.
+ * - the format, offset, stride and buffer of the attribute are NOWHERE in
+ * the module. They are the MTLVertexDescriptor's, which is API state. What
+ * this builds is the shader's half of the match, recorded in
+ * air_shader_info.vertex_attributes for the runtime to check against it.
+ *
+ * The NIR input is always 32 bits wide and the value narrowed afterwards, which
+ * is a decision with a measurement behind it rather than a convention copied
+ * from GL: ir3 is perfectly willing to put a 16-bit vertex input in the half
+ * register file -- a `half4` attribute compiles to `@in(hr0.x)` and a
+ * `cov.f16f32` -- but `ir3_shader_variant.inputs[]` has no half flag (only
+ * `outputs[]` does), prints that input as `r0.x`, and
+ * `fd6_program.cc:emit_vfd_dest` programs VFD_DEST_CNTL_INSTR_REGID straight
+ * out of that field. The fetch would write the full register while the shader
+ * read the half one. 32 bits in and a narrowing instruction keeps this in the
+ * regime both freedreno drivers use, and the VFD converts the format either way.
+ */
+static void
+add_vertex_attribute(struct air_ctx *c, const struct air_md *node, unsigned index,
+ unsigned comps, unsigned bits, bool is_float, bool used,
+ unsigned driver_location)
{
- uint64_t param;
- if (!md_int(node, 0, &param))
- return air_error(c, "interface: an input without a parameter number");
- if (param >= c->fn->num_args)
- return air_error(c, "interface: input for parameter %u of %u", (unsigned)param,
- c->fn->num_args);
- const struct air_type *t = c->fn->args[param].type;
+ struct air_shader_info *info = c->info;
+ const char *tn = md_key_str(node, "air.arg_type_name");
- /* resources */
- if (md_key(node, "air.buffer") >= 0) {
- uint64_t as = 0, index = 0, size = 0;
- md_key_int(node, "air.address_space", &as);
- md_key_int(node, "air.location_index", &index);
- md_key_int(node, "air.buffer_size", &size);
+ /* `int4` and `uint4` are the same `<4 x i32>`, so the sign is in Apple's
+ * reflection string and in nothing the translator can see. `packed_uchar4`
+ * is the same question with a prefix.
+ */
+ const char *base = tn;
+ if (base && !strncmp(base, "packed_", 7))
+ base += 7;
+ enum air_attribute_base_type bt =
+ is_float ? AIR_ATTRIB_FLOAT :
+ (base && base[0] == 'u') ? AIR_ATTRIB_UINT : AIR_ATTRIB_INT;
+
+ const struct air_md *fc = md_key_node(node, "air.function_constant");
+
+ info->vertex_attributes = reralloc(c->mem_ctx, info->vertex_attributes,
+ struct air_vertex_attribute,
+ info->num_vertex_attributes + 1);
+ info->vertex_attributes[info->num_vertex_attributes++] =
+ (struct air_vertex_attribute) {
+ .index = index,
+ .location = VERT_ATTRIB_GENERIC0 + index,
+ .driver_location = driver_location,
+ .components = comps,
+ .bit_size = bits,
+ .base_type = bt,
+ .name = md_key_str(node, "air.arg_name"),
+ .type_name = tn,
+ .used = used,
+ /* operand 2 of the predicate node is its name; 0 is the global and 1
+ * its type (`!"bool"`) */
+ .function_constant = md_str(fc, 2),
+ };
+ if (used)
+ info->vertex_attributes_read |= 1u << index;
+}
+
+static bool
+translate_vertex_attribute(struct air_ctx *c, const struct air_md *node, unsigned param,
+ const struct air_type *t)
+{
+ const char *tn = md_key_str(node, "air.arg_type_name");
+ const char *name = md_key_str(node, "air.arg_name");
+
+ int li = md_key(node, "air.location_index");
+ uint64_t index = 0, span = 1;
+ if (li < 0 || !md_int(node, li + 1, &index))
+ return air_error(c, "interface: a vertex input (%s) with no air.location_index",
+ name ? name : "unnamed");
+
+ /* The second integer after the key. What it MEANS is inferred -- Apple's own
+ * form for an array of resources, where it is the number of consecutive
+ * indices the argument takes -- and nothing in reach can exercise another
+ * value: all 974 air.location_index nodes of the five system libraries and
+ * all 4 764 on Resident Evil 4's vertex inputs carry 1. An attribute
+ * spanning several indices (a matrix member of a stage-in struct is the case
+ * that would) is therefore refused by name rather than silently read from
+ * the first of them.
+ */
+ if (md_int(node, li + 2, &span) && span != 1)
+ return air_error(c, "interface: a vertex input (%s) spanning %u attribute indices",
+ name ? name : "unnamed", (unsigned)span);
+
+ /* Metal has 31 attribute slots; NIR's gl_vert_attrib has 16 generic ones,
+ * and VERT_ATTRIB_GENERIC0 + 16 is VERT_ATTRIB_EDGEFLAG. Unchecked, index
+ * 16 translated onto the edge flag and index 30 past the end of the enum --
+ * both compiled through ir3 without a word, which is the failure this
+ * refusal exists to replace.
+ */
+ if (index >= MAX_VERTEX_GENERIC_ATTRIBS)
+ return air_error(c, "interface: vertex attribute %u (%s), and NIR has %u generic "
+ "vertex attributes",
+ (unsigned)index, name ? name : "unnamed",
+ MAX_VERTEX_GENERIC_ATTRIBS);
+
+ /* Two members at one index would make two NIR inputs at one location, two
+ * VFD fetches into the same register, and a pipeline whose element array is
+ * longer than the attribute set it describes. Metal forbids it; refusing is
+ * what keeps a malformed module from being drawn.
+ */
+ for (unsigned i = 0; i < c->info->num_vertex_attributes; i++)
+ if (c->info->vertex_attributes[i].index == index)
+ return air_error(c, "interface: two vertex inputs at attribute %u (%s and %s)",
+ (unsigned)index,
+ c->info->vertex_attributes[i].name
+ ? c->info->vertex_attributes[i].name : "unnamed",
+ name ? name : "unnamed");
+
+ if (!ret_member_is_number(t))
+ return air_error(c, "interface: a vertex input (%s) of type %s, which is neither a "
+ "scalar nor a vector",
+ name ? name : "unnamed", tn ? tn : "?");
+
+ unsigned comps = air_nir_components(t);
+ unsigned bits = air_nir_bit_size(c, t);
+ bool is_float = air_type_is_float(air_type_scalar(t));
+ /* A vertex attribute is one fetch of at most four components; `double` has
+ * no place in MSL at all, and an `i1` or an `i64` attribute has no
+ * MTLVertexFormat to come from.
+ */
+ if (comps > 4 || (is_float ? (bits != 16 && bits != 32)
+ : (bits != 8 && bits != 16 && bits != 32)))
+ return air_error(c, "interface: a vertex input (%s) of type %s",
+ name ? name : "unnamed", tn ? tn : "?");
+
+ if (md_key(node, "air.arg_unused") >= 0) {
+ add_vertex_attribute(c, node, index, comps, bits, is_float, false, ~0u);
+ c->params[param] = air_val_ssa(c, nir_undef(&c->b, comps, bits));
+ return true;
+ }
+
+ unsigned location = VERT_ATTRIB_GENERIC0 + index;
+ const struct glsl_type *gt = is_float ? glsl_vec_type(comps)
+ : glsl_vector_type(GLSL_TYPE_INT, comps);
+ nir_variable *var = nir_variable_create(c->nir, nir_var_shader_in, gt,
+ name ? name : "attr");
+ var->data.location = location;
+ unsigned driver_location = c->nir->num_inputs++;
+ var->data.driver_location = driver_location;
+ /* An interpolation mode is a fragment-stage word and INTERP_MODE_SMOOTH on a
+ * vertex attribute would be a claim about something that does not happen.
+ */
+ var->data.interpolation = INTERP_MODE_NONE;
+
+ /* nir_shader_gather_info does not run on this path (ir3 calls it only for a
+ * binning variant), so what it would have set is set here, as for the
+ * fragment position above. A runtime matching an MTLVertexDescriptor can
+ * read either this or air_shader_info.vertex_attributes_read.
+ */
+ c->nir->info.inputs_read |= BITFIELD64_BIT(location);
+
+ nir_def *v = nir_load_var(&c->b, var);
+ if (bits != 32) {
+ /* The fetch delivered a full register; the shader asked for a narrower
+ * type. For an integer this is a truncation and the same instruction for
+ * either sign -- which of the two the HARDWARE must do when it widens an
+ * 8- or 16-bit component to 32 bits is the format's business, and the
+ * format is the MTLVertexDescriptor's.
+ */
+ v = is_float ? nir_f2f16(&c->b, v) : nir_u2uN(&c->b, v, bits);
+ }
+ add_vertex_attribute(c, node, index, comps, bits, is_float, true, driver_location);
+
+ /* An attribute the module reads only when a function-constant predicate is
+ * true is still a fetch here, and that is a caveat rather than an error: the
+ * parameter exists whatever the predicate says, so the input is built and
+ * vertex_attributes_read gets its bit. A runtime that required a binding for
+ * every bit of that mask would refuse a perfectly legal pipeline -- one
+ * whose function constants say the attribute is absent and whose
+ * MTLVertexDescriptor therefore configures nothing at that index. Which it
+ * is cannot be decided here, because the value of the predicate is the
+ * pipeline's: these globals are written by the module's air.static_init
+ * constructor from the constants the API supplies. So the condition goes in
+ * the log at translation time rather than only in a document.
+ */
+ const struct air_md *fc = md_key_node(node, "air.function_constant");
+ if (fc) {
+ const char *pred = md_str(fc, 2);
+ air_note(c, "vertex attribute %u (%s) is gated by the function-constant predicate "
+ "%s: the parameter exists either way, so the attribute is fetched and "
+ "its bit is in vertex_attributes_read, and a pipeline whose constants "
+ "make the predicate false may legally configure nothing at that index "
+ "-- the runtime must read air_vertex_attribute.function_constant before "
+ "demanding a binding",
+ (unsigned)index, name ? name : "unnamed", pred ? pred : "(unnamed)");
+ }
+
+ c->params[param] = air_val_ssa(c, v);
+ return true;
+}
+
+/* -- argument buffers ---------------------------------------------------- */
+
+/* The module's own declaration of a named struct type, by that name.
+ *
+ * This is the SECOND statement of what an argument buffer holds. Apple's
+ * metadata is the first; it names every member's kind, index and access but
+ * not its address space -- not one of the 82 048 members of Resident Evil 4's
+ * 7 857 argument buffers carries `air.address_space` -- and a constant buffer
+ * bound as a device buffer is a shader reading the wrong descriptor with
+ * nothing to say so. The struct type says it, member by member, and the two
+ * descriptions are checked against each other before anything is bound: the
+ * member count, every member's byte offset and every member's size. They
+ * agree on all 7 857 sets of that library, which is what makes using it a
+ * reading rather than a guess.
+ */
+static const struct air_type *
+named_struct(const struct air_module *m, const char *name)
+{
+ for (unsigned i = 0; i < m->num_types; i++)
+ if (m->types[i].kind == AIR_TYPE_STRUCT && !m->types[i].opaque &&
+ m->types[i].name && !strcmp(m->types[i].name, name))
+ return &m->types[i];
+ return NULL;
+}
+
+/* The `air.*` key of a descriptor this frontend does not bind, for the refusal
+ * to name -- `air.command_buffer` and `air.render_pipeline_state` are the two
+ * in reach, both in the one kernel of Resident Evil 4 that encodes draws on
+ * the GPU. The housekeeping keys every descriptor carries are skipped so that
+ * the message names the kind and not the first word in the node.
+ */
+static const char *
+descriptor_kind_key(const struct air_md *desc)
+{
+ static const char *const plumbing[] = {
+ "air.location_index", "air.arg_name", "air.arg_type_name", "air.arg_type_size",
+ "air.arg_type_align_size", "air.struct_type_info", "air.address_space",
+ "air.buffer_size", "air.read", "air.write", "air.read_write", "air.sample",
+ "air.arg_unused", "air.raster_order_group", "air.indirect_argument",
+ };
+ for (unsigned i = 0; i < (desc ? desc->num_ops : 0); i++) {
+ const char *s = md_str(desc, i);
+ if (!s || strncmp(s, "air.", 4))
+ continue;
+ bool known = false;
+ for (unsigned k = 0; k < ARRAY_SIZE(plumbing); k++)
+ known |= !strcmp(s, plumbing[k]);
+ if (!known)
+ return s;
+ }
+ return NULL;
+}
+
+/* `air.struct_type_info` of an argument buffer, into a member table.
+ *
+ * The shape, read off Apple's own output: five fixed fields -- byte offset,
+ * size, a flag, the type name, the member name -- and then the key
+ * `air.indirect_argument` and a node, which is an ORDINARY resource
+ * descriptor. It differs from a direct `air.buffer` / `air.texture` /
+ * `air.sampler` argument in one operand, the leading integer, which is the
+ * member's number in the set where a direct argument's is its parameter
+ * number. So everything needed to bind it is in the vocabulary
+ * translate_input has read since the beginning.
+ *
+ * A member whose key comes BEFORE the five fields is how Apple writes one
+ * whose own type is a struct (`!"air.struct_type_info", !N, i32 off, ...`),
+ * which in an argument buffer is inline constant data rather than a resource;
+ * that falls into the first refusal below, as it must -- the runtime's half of
+ * this feature records descriptors against (buffer, offset) and has no bytes
+ * to record for a member that is not one.
+ */
+static bool
+argbuf_members(struct air_ctx *c, const struct air_md *sti, const struct air_type *st,
+ struct air_argbuf *ab)
+{
+ unsigned n = 0;
+ for (unsigned i = 0; i < sti->num_ops; n++) {
+ uint64_t off = 0, msize = 0, flag = 0;
+ const char *mtype = md_str(sti, i + 3), *mname = md_str(sti, i + 4);
+ if (!md_int(sti, i, &off) || !md_int(sti, i + 1, &msize) ||
+ !md_int(sti, i + 2, &flag) || !mtype || !mname) {
+ const char *key = md_str(sti, i);
+ return air_error(c, "interface: member %u of the argument buffer %s is "
+ "described by %s and not by five fields and a descriptor",
+ n, ab->name ? ab->name : "unnamed",
+ key ? key : "something this translator cannot read");
+ }
+
+ const char *key = md_str(sti, i + 5);
+ const struct air_md *desc = NULL;
+ if (key && !strcmp(key, "air.indirect_argument") && i + 6 < sti->num_ops &&
+ sti->ops[i + 6] && sti->ops[i + 6]->kind == AIR_MD_NODE)
+ desc = sti->ops[i + 6];
+ if (!desc)
+ return air_error(c, "interface: member %s of the argument buffer %s is followed "
+ "by %s and not by an air.indirect_argument descriptor",
+ mname, ab->name ? ab->name : "unnamed", key ? key : "nothing");
+ i += 7;
+
+ /* A set inside a set. The member's descriptor would have a member table
+ * of its own, reached through a pointer LOADED out of this one and then
+ * walked -- which is the general pointer model and not this one.
+ */
+ if (md_key(desc, "air.indirect_buffer") >= 0)
+ return air_error(c, "interface: member %s of the argument buffer %s is itself an "
+ "argument buffer", mname, ab->name ? ab->name : "unnamed");
+
+ bool is_buffer = md_key(desc, "air.buffer") >= 0;
+ bool is_texture = md_key(desc, "air.texture") >= 0;
+ bool is_sampler = md_key(desc, "air.sampler") >= 0;
+ if (is_buffer + is_texture + is_sampler != 1) {
+ const char *what = descriptor_kind_key(desc);
+ return air_error(c, "interface: member %s of the argument buffer %s is %s, and "
+ "this frontend binds a buffer, a texture or a sampler",
+ mname, ab->name ? ab->name : "unnamed",
+ is_buffer + is_texture + is_sampler > 1
+ ? "more than one kind at once"
+ : what ? what : "a kind this translator does not know");
+ }
+
+ int mli = md_key(desc, "air.location_index");
+ uint64_t mid = 0, mspan = 1;
+ if (mli < 0 || !md_int(desc, mli + 1, &mid))
+ return air_error(c, "interface: member %s of the argument buffer %s has no "
+ "air.location_index", mname,
+ ab->name ? ab->name : "unnamed");
+ /* An ARRAY of descriptors in one member: the shader could then pick one
+ * at run time, which no resolution can do. Every one of Resident Evil
+ * 4's 82 048 members spans exactly one index, so this is a guard on a
+ * shape not yet seen rather than a gap with modules behind it.
+ */
+ if (md_int(desc, mli + 2, &mspan) && mspan != 1)
+ return air_error(c, "interface: member %s of the argument buffer %s spans %u "
+ "descriptor indices", mname,
+ ab->name ? ab->name : "unnamed", (unsigned)mspan);
+
+ /* against the module's own struct type */
+ if (n >= st->num_members)
+ return air_error(c, "interface: the argument buffer %s declares more members "
+ "than the %u its struct type has",
+ ab->name ? ab->name : "unnamed", st->num_members);
+ const struct air_type *mt = st->members[n];
+ uint64_t tyoff = air_struct_member_offset(c->m, st, n);
+ uint64_t tysize = air_type_alloc_size(c->m, mt);
+ if (tyoff != off || tysize != msize)
+ return air_error(c, "interface: member %s of the argument buffer %s is %u bytes "
+ "at byte %u, and the module's struct type puts %u bytes "
+ "there at %u", mname, ab->name ? ab->name : "unnamed",
+ (unsigned)msize, (unsigned)off, (unsigned)tysize,
+ (unsigned)tyoff);
+
+ enum air_resource_kind rk;
+ unsigned addrspace;
+ bool read = md_key(desc, "air.read") >= 0 || md_key(desc, "air.read_write") >= 0 ||
+ md_key(desc, "air.sample") >= 0;
+ bool write = md_key(desc, "air.write") >= 0 || md_key(desc, "air.read_write") >= 0;
+
+ if (is_buffer) {
+ /* The member IS the pointer, and its address space is the only thing
+ * that says whether the descriptor is a constant buffer or a device
+ * one. A frontend that reused the direct-argument path unchanged
+ * would read air.address_space, find none, and refuse at "a buffer in
+ * address space 0" -- which is the better of the two ways to be
+ * wrong.
+ */
+ if (mt->kind != AIR_TYPE_POINTER || (mt->addrspace != 1 && mt->addrspace != 2))
+ return air_error(c, "interface: the buffer member %s of the argument buffer "
+ "%s is not a device or constant pointer in the module's "
+ "struct type", mname, ab->name ? ab->name : "unnamed");
+ addrspace = mt->addrspace;
+ rk = mt->addrspace == 1 ? AIR_RESOURCE_DEVICE_BUFFER : AIR_RESOURCE_CONSTANT_BUFFER;
+ } else {
+ /* A texture or a sampler arrives as the one-field `metal::texture2d`
+ * / `metal::sampler` wrapper the CoreImage stitched functions take
+ * (section 7.8), here as a struct member instead of a parameter and
+ * with the handle LOADED rather than passed. Keyed on Apple's own
+ * name, as the library wrapper is: "a struct with one pointer in it"
+ * would quietly accept anything shaped alike.
+ */
+ if (mt->kind != AIR_TYPE_STRUCT || !mt->name ||
+ strncmp(mt->name, "struct.metal::", 14) || mt->num_members != 1 ||
+ mt->members[0]->kind != AIR_TYPE_POINTER)
+ return air_error(c, "interface: the %s member %s of the argument buffer %s is "
+ "not one of Apple's one-field metal:: wrappers in the "
+ "module's struct type", is_sampler ? "sampler" : "texture",
+ mname, ab->name ? ab->name : "unnamed");
+ addrspace = mt->members[0]->addrspace;
+ rk = is_sampler ? AIR_RESOURCE_SAMPLER
+ : write ? AIR_RESOURCE_IMAGE : AIR_RESOURCE_TEXTURE;
+ }
+
+ ab->members = reralloc(c->mem_ctx, ab->members, struct air_argbuf_member, n + 1);
+ ab->members[n] = (struct air_argbuf_member) {
+ .offset = off, .kind = rk, .metal_id = (unsigned)mid, .index = ~0u,
+ .addrspace = addrspace, .read = read, .write = write,
+ .name = mname, .type_name = mtype, .node = desc,
+ };
+ ab->num_members = n + 1;
+ }
+
+ if (n != st->num_members)
+ return air_error(c, "interface: the argument buffer %s declares %u members and its "
+ "struct type has %u", ab->name ? ab->name : "unnamed", n,
+ st->num_members);
+ return true;
+}
+
+/* Metal's argument buffer: `air.indirect_buffer`, one struct of descriptors
+ * bound at one buffer index, which is how a Vulkan descriptor set reaches
+ * Metal through SPIRV-Cross and how an MSL `[[id(n)]]` struct reaches it from
+ * Metal itself. 7 620 of Resident Evil 4's 7 879 modules have one and 0 of
+ * iOS 16.5's 671 do, so this is the one interface word that is invisible from
+ * Apple's own libraries and is 97% of a game (docs/air.md section 8.2).
+ *
+ * The parameter does not become a buffer to read: it becomes a pointer of its
+ * own class, and the member table built here is what a load through it
+ * resolves against (argbuf_load). No address, no descriptor indirection, and
+ * nothing of the `32bit_index_offset` model the rest of the frontend is built
+ * on changes.
+ */
+static bool
+translate_argument_buffer(struct air_ctx *c, const struct air_md *node, unsigned param)
+{
+ const char *name = md_key_str(node, "air.arg_name");
+ const char *tn = md_key_str(node, "air.arg_type_name");
+
+ int li = md_key(node, "air.location_index");
+ uint64_t set_index = 0, span = 1, size = 0;
+ if (li < 0 || !md_int(node, li + 1, &set_index))
+ return air_error(c, "interface: an argument buffer (%s) with no air.location_index",
+ name ? name : "unnamed");
+ if (md_int(node, li + 2, &span) && span != 1)
+ return air_error(c, "interface: an argument buffer (%s) spanning %u buffer indices",
+ name ? name : "unnamed", (unsigned)span);
+ md_key_int(node, "air.buffer_size", &size);
+
+ const struct air_md *sti = md_key_node(node, "air.struct_type_info");
+ if (!sti)
+ return air_error(c, "interface: an argument buffer (%s) with no "
+ "air.struct_type_info, so nothing says what is in it",
+ name ? name : "unnamed");
+
+ const struct air_type *st = NULL;
+ if (tn) {
+ char *sname = ralloc_asprintf(c->mem_ctx, "struct.%s", tn);
+ st = named_struct(c->m, sname);
+ }
+ if (!st)
+ return air_error(c, "interface: an argument buffer (%s) whose struct type %s the "
+ "module does not declare, so its members' address spaces "
+ "cannot be read", name ? name : "unnamed", tn ? tn : "(unnamed)");
+
+ /* Two sets at one Metal buffer index would have one record at draw time and
+ * two sets of members wanting it. */
+ for (unsigned i = 0; i < c->num_argbufs; i++)
+ if (c->argbufs[i].metal_index == set_index)
+ return air_error(c, "interface: two argument buffers at buffer index %u (%s and "
+ "%s)", (unsigned)set_index,
+ c->argbufs[i].name ? c->argbufs[i].name : "unnamed",
+ name ? name : "unnamed");
+
+ c->argbufs = reralloc(c->mem_ctx, c->argbufs, struct air_argbuf, c->num_argbufs + 1);
+ struct air_argbuf *ab = &c->argbufs[c->num_argbufs];
+ *ab = (struct air_argbuf) {
+ .metal_index = (unsigned)set_index, .size = (unsigned)size, .param = param,
+ .name = name, .type_name = tn,
+ };
+ if (!argbuf_members(c, sti, st, ab))
+ return false;
+ /* eight bytes per member is what MTLArgumentEncoder writes and what the
+ * runtime's encodedLength has to answer; a set whose declared size is not
+ * that is a layout this frontend has not seen */
+ if (size != 8ull * ab->num_members)
+ return air_error(c, "interface: the argument buffer %s is %u bytes with %u members, "
+ "and this frontend knows the eight-bytes-per-member layout "
+ "only", name ? name : "unnamed", (unsigned)size,
+ ab->num_members);
+
+ struct air_ptr p = { .class = AIR_PTR_ARGBUF, .binding = c->num_argbufs++ };
+ c->params[param] = air_val_ptr(c, p);
+ return true;
+}
+
+/* The flat per-kind index of every member of every argument buffer, and one
+ * air_resource each.
+ *
+ * Why this is a decision and not a transcription: a member's own
+ * `air.location_index` is Metal's `[[id(n)]]`, which is per set, so set 0's
+ * member 0 and set 1's member 0 both say 0 and 237 of Resident Evil 4's
+ * modules have two sets. One counter per kind in (set, member) order gives
+ * each member a binding of its own, and `argbuf_set` / `argbuf_id` on the
+ * resource keep the Metal name the application uses -- the runtime binds by
+ * the first and the application names by the second.
+ *
+ * The counters start above the highest index a DIRECT argument of that kind
+ * already took, so a member and an argument cannot land in one Gallium slot.
+ * None of Resident Evil 4's 7 620 modules mixes the two -- those shaders have
+ * no direct resources at all -- so this is the part of the allocation nothing
+ * in reach exercises, and it is here because the collision it prevents would
+ * be a draw over the wrong resource rather than an error.
+ *
+ * Called after every input is translated, so that the direct arguments are all
+ * known, and before plan_globals, so that a shader's own `constexpr sampler`
+ * still lands after the samplers the API binds.
+ */
+static void
+allocate_argbuf_indices(struct air_ctx *c)
+{
+ unsigned next_buffer = 0, next_texture = 0, next_sampler = 0;
+ for (unsigned i = 0; i < c->info->num_resources; i++) {
+ const struct air_resource *r = &c->info->resources[i];
+ switch (r->kind) {
+ case AIR_RESOURCE_CONSTANT_BUFFER:
+ case AIR_RESOURCE_DEVICE_BUFFER:
+ next_buffer = MAX2(next_buffer, r->index + 1);
+ break;
+ case AIR_RESOURCE_TEXTURE:
+ case AIR_RESOURCE_IMAGE:
+ next_texture = MAX2(next_texture, r->index + 1);
+ break;
+ case AIR_RESOURCE_SAMPLER:
+ next_sampler = MAX2(next_sampler, r->index + 1);
+ break;
+ default:
+ break;
+ }
+ }
+
+ for (unsigned s = 0; s < c->num_argbufs; s++) {
+ struct air_argbuf *ab = &c->argbufs[s];
+ for (unsigned k = 0; k < ab->num_members; k++) {
+ struct air_argbuf_member *m = &ab->members[k];
+ switch (m->kind) {
+ case AIR_RESOURCE_CONSTANT_BUFFER:
+ case AIR_RESOURCE_DEVICE_BUFFER:
+ m->index = next_buffer++;
+ break;
+ case AIR_RESOURCE_TEXTURE:
+ case AIR_RESOURCE_IMAGE:
+ m->index = next_texture++;
+ break;
+ default:
+ m->index = next_sampler++;
+ break;
+ }
+ struct air_resource *r = add_resource(c, m->kind, m->index, 0, m->read, m->write,
+ ab->param, m->node);
+ if (!r)
+ return; /* add_resource has said why */
+ r->argbuf_set = ab->metal_index;
+ r->argbuf_id = m->metal_id;
+ r->argbuf_offset = (unsigned)m->offset;
+ }
+ }
+}
+
+/* A load through an argument-buffer pointer RESOLVES instead of loading: the
+ * byte offset names a member, and the member's descriptor names a binding.
+ *
+ * Keying on the byte offset rather than on the member number is what makes the
+ * three shapes SPIRV-Cross emits one case. A buffer member is the pointer
+ * itself; a texture or sampler member is a one-field wrapper, and the
+ * `getelementptr` into that wrapper adds zero bytes -- so the texture shape
+ * resolves by the same lookup as the buffer shape, with nothing special about
+ * it and no second path to keep right.
+ */
+static struct air_val *
+argbuf_load(struct air_ctx *c, struct air_ptr ptr, const struct air_type *type)
+{
+ const struct air_argbuf *ab = &c->argbufs[ptr.binding];
+ const char *abname = ab->name ? ab->name : "unnamed";
+
+ uint64_t off = 0;
+ if (ptr.offset) {
+ if (!nir_src_is_const(nir_src_for_ssa(ptr.offset))) {
+ air_error(c, "instruction: a load through the argument buffer %s at an offset "
+ "that is not constant, which is a descriptor chosen at run time",
+ abname);
+ return NULL;
+ }
+ off = nir_src_as_uint(nir_src_for_ssa(ptr.offset));
+ }
+
+ const struct air_argbuf_member *m = NULL;
+ for (unsigned i = 0; i < ab->num_members; i++)
+ if (ab->members[i].offset == off)
+ m = &ab->members[i];
+ if (!m) {
+ air_error(c, "instruction: a load from byte %u of the argument buffer %s, which "
+ "declares no member there", (unsigned)off, abname);
+ return NULL;
+ }
+
+ /* A number read out of the set rather than a descriptor: Metal's
+ * `constantDataAtIndex:`. Every member of this set is a resource -- that is
+ * what argbuf_members checked -- so this is a load of a descriptor's bytes,
+ * which are the runtime's 8-byte handle and mean nothing to a driver that
+ * binds by index.
+ */
+ if (type->kind != AIR_TYPE_POINTER) {
+ air_error(c, "instruction: a load of a value out of the argument buffer %s, whose "
+ "member %s at byte %u is a resource and has no bytes to read",
+ abname, m->name ? m->name : "unnamed", (unsigned)off);
+ return NULL;
+ }
+
+ /* The address space of the pointer the shader LOADS, against the one the
+ * module's struct type declares for that member. The two come from
+ * different records -- the load instruction's own result type and the
+ * module's type table -- and either one alone would bind a constant buffer
+ * as a device buffer without a word if it were wrong.
+ */
+ if (type->addrspace != m->addrspace) {
+ air_error(c, "instruction: a pointer in address space %u loaded from member %s of "
+ "the argument buffer %s, which the module's struct type puts in "
+ "address space %u", type->addrspace, m->name ? m->name : "unnamed",
+ abname, m->addrspace);
+ return NULL;
+ }
+
+ struct air_ptr p = { .binding = m->index, .pointee = type->elem };
+ switch (m->kind) {
+ case AIR_RESOURCE_CONSTANT_BUFFER: p.class = AIR_PTR_UBO; break;
+ case AIR_RESOURCE_DEVICE_BUFFER: p.class = AIR_PTR_SSBO; break;
+ case AIR_RESOURCE_SAMPLER: p.class = AIR_PTR_SAMPLER; break;
+ case AIR_RESOURCE_IMAGE:
+ p.class = AIR_PTR_TEXTURE;
+ p.is_image = true;
+ break;
+ default:
+ p.class = AIR_PTR_TEXTURE;
+ break;
+ }
+ return air_val_ptr(c, p);
+}
+
+static bool
+translate_input(struct air_ctx *c, const struct air_md *node, unsigned *next_varying)
+{
+ uint64_t param;
+ if (!md_int(node, 0, &param))
+ return air_error(c, "interface: an input without a parameter number");
+ if (param >= c->fn->num_args)
+ return air_error(c, "interface: input for parameter %u of %u", (unsigned)param,
+ c->fn->num_args);
+ const struct air_type *t = c->fn->args[param].type;
+
+ /* resources */
+ if (md_key(node, "air.buffer") >= 0) {
+ uint64_t as = 0, index = 0, size = 0;
+ /* AIR before 2.4 (MapKit's Clear::clear_vertex, Metal 2.3 for iOS 14)
+ * writes no air.address_space on a buffer: the parameter's own pointer
+ * type is then the only statement of it, and it is the same fact -- the
+ * key, where present, restates the address space of that pointer.
+ */
+ if (!md_key_int(node, "air.address_space", &as) && t->kind == AIR_TYPE_POINTER)
+ as = t->addrspace;
+ md_key_int(node, "air.location_index", &index);
+ md_key_int(node, "air.buffer_size", &size);
bool read = md_key(node, "air.read") >= 0 || md_key(node, "air.read_write") >= 0;
bool write = md_key(node, "air.write") >= 0 || md_key(node, "air.read_write") >= 0;
@@ -650,13 +2194,18 @@ translate_input(struct air_ctx *c, const struct air_md *node, unsigned *next_var
case 3:
p.class = AIR_PTR_SHARED;
rk = AIR_RESOURCE_THREADGROUP_BUFFER;
- /* a threadgroup argument is sized at dispatch; its base is where the
- * previous ones end, which only the runtime knows. One is all the
- * corpus uses, so its base is zero and anything else is refused.
+ /* a threadgroup argument is sized at dispatch, so it can only go
+ * after everything whose size is known here: the module's static
+ * threadgroup arrays, which plan_globals lays out from offset 0. Its
+ * base is set once they are (below plan_globals). At base 0 it would
+ * share its bytes with the first of them. One is all the corpus uses;
+ * a second would start where the first's runtime length ends, which
+ * only the runtime knows, and is refused.
*/
if (index != 0)
return air_error(c, "interface: a second threadgroup buffer (index %u)",
(unsigned)index);
+ c->tg_param_plus1 = param + 1;
break;
default:
return air_error(c, "interface: a buffer in address space %u", (unsigned)as);
@@ -666,6 +2215,9 @@ translate_input(struct air_ctx *c, const struct air_md *node, unsigned *next_var
return true;
}
+ if (md_key(node, "air.indirect_buffer") >= 0)
+ return translate_argument_buffer(c, node, param);
+
if (md_key(node, "air.texture") >= 0) {
uint64_t index = 0;
md_key_int(node, "air.location_index", &index);
@@ -695,6 +2247,65 @@ translate_input(struct air_ctx *c, const struct air_md *node, unsigned *next_var
return true;
}
+ /* A TILE SHADER's imageblock. Named here rather than left to the catch-all
+ * at the end of this function, because that one printed "interface: input
+ * air.imageblock" -- the key it met, which reads like a decision and was
+ * only a gap. There are two walls behind it and the near one is NOT the
+ * Adreno's, so the sentence has to say which is which or the next reader
+ * spends a day on the wrong one.
+ *
+ * THE NEAR WALL: nothing on this device can ask for these modules.
+ * QuartzCore reaches its four tile_downsample_* pipelines from exactly two
+ * places, and both ask one question first -- the same bit either way, bit 23
+ * of the capability word at ctx+0xa3c:
+ *
+ * CA::OGL::BlurState::tile_downsample 0x1883d3578 get(property 37), and
+ * returns 0 on a zero
+ * CA::OGL::MetalContext::warmup_shaders 0x188476350 ldrb [ctx, #0xa3e]
+ * 0x188476354 tbz w8, #7 -> skip
+ *
+ * `get` (0x1882e8a78) is that bit and a CA default; the bit is set by
+ * CA::OGL::new_metal_context (0x18847fbfc) from -[MTLDevice
+ * supportsTileShaders]; and Metal answers that as
+ * -[_MTLDevice supportsTileShaders] (0x181aa897c) ->
+ * -[_MTLDeviceFeatureQueries familySupportsTileShaders] (0x181b216b0), whose
+ * whole body is `supportsFamily:` with w2 = 0x3ec -- MTLGPUFamilyApple4.
+ * -[_MTLDevice initGPUFamilySupport] (0x181a1be1c) builds that list from
+ * -featureProfile through a jump table at 0x181b5bc7c: profile 0 claims
+ * Apple1, profile 1 adds Apple2, and Apple4 first appears at profile 5
+ * (0x181a1c424). This port answers 0 or 1 (userland/src/redwoodmetal.m,
+ * boot-arg metal_profile=), and the phone itself has printed
+ * supportsFamily:Apple4 = 0.
+ *
+ * Apple4 is not a later generation's feature, which is the part worth
+ * saying out loud: it is the iPhone X's OWN family, and Apple's A11 driver
+ * answers profile 5. These four shaders run on this very phone under iOS.
+ * The only reason nothing here can ask for them is that this port claims
+ * less than its hardware is.
+ *
+ * THE FAR WALL is the Adreno's, and it is why raising the profile would buy
+ * a refusal rather than a shader: Mesa 25.0.7 has no tile memory anywhere in
+ * src/freedreno, and the one instruction that reads the tile buffer is
+ * fragment-only by assertion (ir3_compiler_nir.c, nir_texop_txf_ms_fb).
+ * docs/air.md section 3 has what the DRIVER would have to do instead -- break
+ * the pass, resolve the attachment, dispatch an ordinary kernel -- which is
+ * the round trip tile shading exists to avoid, and which belongs to whoever
+ * writes the pass splitting and not to this function.
+ *
+ * Only the near wall is in the message, because air_error's buffer is 256
+ * bytes (air_private.h) and vsnprintf cuts the rest off mid-word without
+ * saying so: a longer sentence would have ended at "-featureProfile" in
+ * every report, and the second wall is exactly the half that would have
+ * gone. The message names the one that decides and points here for the one
+ * that would decide next.
+ */
+ if (md_key(node, "air.imageblock") >= 0)
+ return air_error(c, "interface: an imageblock input -- a tile shader nothing on "
+ "this device can ask for: CoreAnimation builds these only "
+ "behind -[MTLDevice supportsTileShaders] = "
+ "supportsFamily:Apple4, and this port claims at most Apple2 "
+ "(docs/air.md 3 has the other wall)");
+
/* system values */
for (unsigned i = 0; i < ARRAY_SIZE(sysvals); i++) {
if (md_key(node, sysvals[i].key) < 0)
@@ -709,25 +2320,198 @@ translate_input(struct air_ctx *c, const struct air_md *node, unsigned *next_var
/* interpolated inputs of a fragment shader, and vertex attributes */
if (md_key(node, "air.position") >= 0) {
if (c->nir->info.stage == MESA_SHADER_FRAGMENT) {
- nir_def *pos = nir_load_frag_coord(&c->b);
+ /* A position the shader does not read gets no input at all, and the order of
+ * these two tests is the whole reason the test is here: every one of Apple's
+ * seventeen texturedFragment* declares `float4 pos [[position]]` and marks it
+ * air.arg_unused, so putting the position machinery before this check would
+ * give all seventeen a dead varying slot and a dead FS_COORD property -- a
+ * change to 17 shaders already compared pixel by pixel, bought for nothing.
+ */
+ if (md_key(node, "air.arg_unused") >= 0) {
+ c->params[param] = air_val_ssa(c, nir_undef(&c->b, air_nir_components(t),
+ air_nir_bit_size(c, t)));
+ return true;
+ }
+ /*
+ * Metal's [[position]] in a fragment function is the window coordinate with
+ * the origin at the TOP left, y down, and the pixel centre at a half-integer
+ * -- which is what origin_upper_left plus !pixel_center_integer says in NIR.
+ * NIR's default is OpenGL's bottom-left origin, so leaving these unset would
+ * give every position-dependent Metal shader a y counted from the wrong edge.
+ *
+ * And the value is a SYSTEM-VALUE VARIABLE, not a bare nir_load_frag_coord,
+ * because the two Gallium drivers this frontend feeds want opposite forms and
+ * only one direction has a pass:
+ *
+ * - freedreno sets pipe_caps.fs_position_is_sysval, and ir3 compile_asserts
+ * `slot != VARYING_SLOT_POS` in its fragment-input setup
+ * (ir3_compiler_nir.c): it will only take the intrinsic, which is what
+ * nir_lower_system_values turns this variable into;
+ * - softpipe does NOT set that cap, and nir_to_tgsi turns the intrinsic into
+ * ureg_DECL_system_value(TGSI_SEMANTIC_POSITION), which softpipe fills only
+ * for compute (sp_compute.c is the one writer of machine->SystemValue). A
+ * fragment shader therefore read (0,0,0,0) -- measured, not reasoned: a
+ * hand-written module returning its own position printed four zeros while
+ * the same module returning a constant printed the constant. Softpipe wants
+ * the position as an input varying, which only
+ * nir_lower_sysvals_to_varyings produces, and only from a variable.
+ *
+ * So the frontend emits the form spirv_to_nir emits, for the same reason, and
+ * the consumer lowers it per its screen's cap (finalize_for_gallium).
+ */
+ nir_variable *var = nir_variable_create(c->nir, nir_var_system_value,
+ glsl_vec4_type(), "position");
+ var->data.location = SYSTEM_VALUE_FRAG_COORD;
+ var->data.interpolation = INTERP_MODE_NOPERSPECTIVE;
+ nir_def *pos = nir_load_var(&c->b, var);
c->params[param] = air_val_ssa(c, fit(c, pos, t));
c->nir->info.fs.origin_upper_left = true;
c->nir->info.fs.pixel_center_integer = false;
+ /* nothing in this path calls nir_shader_gather_info, and nir_to_tgsi emits the
+ * FS_COORD_ORIGIN property only for a shader whose info says it reads the
+ * position -- without this the property is absent and softpipe falls back on
+ * TGSI_FS_COORD_ORIGIN_UPPER_LEFT (0), which is the right answer by accident.
+ */
+ BITSET_SET(c->nir->info.system_values_read, SYSTEM_VALUE_FRAG_COORD);
+ c->nir->info.inputs_read |= VARYING_BIT_POS;
return true;
}
return air_error(c, "interface: air.position as an input of a %s shader",
gl_shader_stage_name(c->nir->info.stage));
}
+
+ /* Metal's `[[color(n)]]` argument to a fragment function: the value render target n
+ * holds NOW, read inside the shader. That is framebuffer fetch, and Metal's name for
+ * it is programmable blending.
+ *
+ * The modules say which feature it is rather than this being inferred from the name:
+ * every one of the twenty in QuartzCore's library carries
+ * `!"air.compile.framebuffer_fetch_enable"` among its `air.compile_options`, the
+ * option Apple's compiler sets exactly when a fragment argument is `[[color(n)]]`.
+ *
+ * The shape is ONE integer after the key -- the target -- where the output form has
+ * two, the second being the dual-source index:
+ *
+ * output !{!"air.render_target", i32 0, i32 0, !"air.arg_type_name", !"float4", ...}
+ * input !{i32 1, !"air.render_target", i32 2, !"air.arg_type_name", !"float", ...}
+ *
+ * so the two are read here and in translate_output at their own offsets, not through
+ * one shared helper that would read a dual-source index out of an argument name.
+ *
+ * NIR's form of a framebuffer fetch is a READ OF THE OUTPUT: load_output carrying
+ * io_semantics.fb_fetch_output, which is what nir_lower_io makes of a GLSL `inout`
+ * colour and what nir_lower_blend emits for a blend it has to lower in software. The
+ * intrinsic is built here directly instead of as a second nir_var_shader_out variable
+ * at the same location, because both consumers of this frontend lower IO before the
+ * backend sees it and two output variables at one location would take two driver
+ * locations and two entries in ir3's output table for one attachment.
+ *
+ * It is read as FOUR 32-BIT FLOATS whatever the argument's type, and narrowed after:
+ * nir_lower_fb_read -- the pass ir3 runs in ir3_nir_post_finalize -- rewrites
+ * load_output into a txf_ms_fb whose destination is fixed at 4 x f32 and rewrites the
+ * intrinsic's uses with it. So `float` (path_blit_frag's coverage mask) loses three
+ * channels and `half4` (every _lph build) loses precision here, where the conversion
+ * is visible, rather than silently inside the lowering.
+ */
+ int rt_in = md_key(node, "air.render_target");
+ if (rt_in >= 0) {
+ if (c->nir->info.stage != MESA_SHADER_FRAGMENT)
+ return air_error(c, "interface: air.render_target as an input of a %s shader",
+ gl_shader_stage_name(c->nir->info.stage));
+ /* An argument nothing reads must not turn the shader into a framebuffer-fetch
+ * shader: info.fs.uses_fbfetch_output alone makes a6xx flush the pipeline per
+ * overlapping primitive (fd6_emit.cc, build_prim_mode), and an unused argument
+ * needs no value at all. None of the corpus's twenty is marked this way; the
+ * order is here so that one would cost nothing rather than a flush.
+ */
+ if (md_key(node, "air.arg_unused") >= 0) {
+ c->params[param] = air_val_ssa(c, nir_undef(&c->b, air_nir_components(t),
+ air_nir_bit_size(c, t)));
+ return true;
+ }
+ uint64_t n = 0;
+ if (!md_int(node, rt_in + 1, &n))
+ return air_error(c, "interface: an air.render_target input without a target");
+ if (n >= MAX_DRAW_BUFFERS)
+ return air_error(c, "interface: air.render_target input %u", (unsigned)n);
+ /* Only a float target. The lowering hands back floats converted from the
+ * attachment's format, so an int4 or uint4 `[[color(n)]]` would arrive as the
+ * wrong value rather than the wrong type -- and nothing would say so. The corpus
+ * declares float4, half4 and float and nothing else, so the refusal is a guard on
+ * a case not yet seen rather than a gap with modules behind it.
+ */
+ unsigned bits = air_nir_bit_size(c, t);
+ if (!air_type_is_float(air_type_scalar(t)) || (bits != 16 && bits != 32)) {
+ const char *tn = md_key_str(node, "air.arg_type_name");
+ return air_error(c, "interface: an air.render_target input of type %s",
+ tn ? tn : "?");
+ }
+
+ nir_io_semantics sem = {
+ .location = FRAG_RESULT_DATA0 + n,
+ .num_slots = 1,
+ .fb_fetch_output = true,
+ /* Coherent, which is Metal's guarantee and not a safe default picked here: a
+ * `[[color(n)]]` read sees what the primitives already drawn left behind, which
+ * is the whole of how CoreAnimation blends a layer in place. On a6xx the flag
+ * is what makes the GMEM path flush per overlap instead of not at all
+ * (fd6_emit.cc); the sysmem path flushes either way.
+ */
+ .fb_fetch_output_coherent = true,
+ };
+ nir_def *fetched = nir_load_output(&c->b, 4, 32, nir_imm_int(&c->b, 0),
+ .dest_type = nir_type_float32,
+ .io_semantics = sem);
+ unsigned want = air_nir_components(t);
+ if (want < 4)
+ fetched = nir_trim_vector(&c->b, fetched, want);
+ if (bits == 16)
+ fetched = nir_f2f16(&c->b, fetched);
+
+ /* nir_shader_gather_info does not run on this path -- ir3 calls it only for a
+ * binning variant -- so what it would have set from the intrinsic
+ * (nir_gather_info.c) is set here. Without uses_fbfetch_output the draw gets no
+ * flush at all and the read returns whatever the tile buffer happened to hold.
+ */
+ c->nir->info.outputs_read |= BITFIELD64_BIT(sem.location);
+ c->nir->info.fs.uses_fbfetch_output = true;
+ c->nir->info.fs.fbfetch_coherent = true;
+ c->info->fbfetch_targets |= 1u << n;
+
+ /* The layer the fetch reads is not this frontend's to choose, and it is
+ * not safe by default, so it goes in the log rather than only in a
+ * document. nir_lower_fb_read builds the coordinate as (x, y, layer)
+ * with the layer from nir_load_layer_id, which ir3_nir_lower_layer_id
+ * turns into a flat varying read of VARYING_SLOT_LAYER. If the pipeline's
+ * vertex stage writes air.render_target_array_index, ir3_link_shaders
+ * finds it by slot and the fetch reads the right layer; if nothing wrote
+ * it, ir3_link_add skips the entry (regid(63, 0)) and the varying holds
+ * whatever the VPC location held. Which of the two it is cannot be known
+ * from this module -- the fragment shader is identical either way -- so
+ * the condition is stated for whoever builds the pipeline.
+ */
+ air_note(c, "the framebuffer fetch reads its layer from VARYING_SLOT_LAYER, which "
+ "nir_lower_fb_read asks for and this module cannot write: for a LAYERED "
+ "render target the pipeline's vertex stage must write "
+ "air.render_target_array_index, or the fetch reads an undefined layer. "
+ "A single-layer target is unaffected -- there is no second layer in the "
+ "descriptor to reach");
+
+ c->params[param] = air_val_ssa(c, fetched);
+ return true;
+ }
+
if (md_key(node, "air.fragment_input") >= 0 || md_key(node, "air.vertex_input") >= 0 ||
md_key(node, "air.stage_in") >= 0) {
- unsigned slot;
- if (c->nir->info.stage == MESA_SHADER_VERTEX) {
- uint64_t index = 0;
- md_key_int(node, "air.location_index", &index);
- slot = VERT_ATTRIB_GENERIC0 + index;
- } else {
- slot = VARYING_SLOT_VAR0 + (*next_varying)++;
- }
+ /* A vertex attribute and an interpolated varying are two different things
+ * that AIR spells with keys in the same family, and only the one below is
+ * a varying: it is linked to the stage before it, interpolated, and takes
+ * its slot from the order it appears in. An attribute is fetched from a
+ * vertex buffer at Metal's own index and has nothing to interpolate.
+ */
+ if (c->nir->info.stage == MESA_SHADER_VERTEX)
+ return translate_vertex_attribute(c, node, param, t);
+ unsigned slot = VARYING_SLOT_VAR0 + (*next_varying)++;
const char *name = md_key_str(node, "air.arg_name");
nir_variable *var = make_io_var(c, nir_var_shader_in, t, slot, name);
var->data.driver_location = c->nir->num_inputs++;
@@ -741,7 +2525,10 @@ translate_input(struct air_ctx *c, const struct air_md *node, unsigned *next_var
var->data.sample = md_key(node, "air.sample") >= 0;
if (c->nir->info.stage == MESA_SHADER_FRAGMENT)
add_varying(c, name, slot, air_nir_components(t), node);
- c->params[param] = air_val_ssa(c, nir_load_var(&c->b, var));
+ nir_def *loaded = nir_load_var(&c->b, var);
+ if (io_is_boolean(t))
+ loaded = nir_ine_imm(&c->b, loaded, 0); /* see io_is_boolean */
+ c->params[param] = air_val_ssa(c, loaded);
return true;
}
@@ -811,6 +2598,41 @@ translate_output(struct air_ctx *c, const struct air_md *node, unsigned field,
return air_error(c, "interface: an output with no key this translator knows");
}
+ /* An output whose type is not a number has no varying here, and until this
+ * was checked it got one anyway: an `air.clip_distance` output arrives as
+ * an ARRAY of float, and air_type_scalar has no answer for an array, so the
+ * varying was declared with whatever air_nir_bit_size made of that. NIR's
+ * own form is an output of several slots, which this frontend does not
+ * build, so it is refused by name instead.
+ *
+ * The length is in the message because it decides how much work closing
+ * this is, and the measured length is not the one the general case
+ * suggests: over all 86 of Resident Evil 4's modules that stop here --
+ * every one a vertex shader -- the member is `[1 x float]`, an array of
+ * exactly ONE, and Metal's own `air.arg_type_name` on the node says
+ * `float`. So there is no module in reach that declares two clip planes,
+ * and the shape that is actually in the way is an array of one, which is a
+ * scalar wearing a bracket. Closing that is two things and not one: this
+ * declaration would take the element type, and emit_return would have to
+ * store elems[0] of a one-element aggregate instead of skipping a field
+ * that is not an AIR_VAL_SSA -- which is what silently drops the plane
+ * today. It is left refused rather than half built, because a clip distance
+ * that is written to nothing is a shader that draws without the clipping it
+ * asked for, and nothing in this tree can check a vertex clip distance
+ * numerically.
+ */
+ if (t->kind == AIR_TYPE_ARRAY || t->kind == AIR_TYPE_STRUCT) {
+ const char *tn = md_key_str(node, "air.arg_type_name");
+ if (t->kind == AIR_TYPE_ARRAY)
+ return air_error(c, "interface: an output (%s), declared %s, that arrives as an "
+ "array of %u, which this frontend has no varying for",
+ name ? name : "unnamed", tn ? tn : "an aggregate",
+ (unsigned)t->count);
+ return air_error(c, "interface: an output (%s), declared %s, that arrives as a "
+ "struct of %u members, which this frontend has no varying for",
+ name ? name : "unnamed", tn ? tn : "an aggregate", t->num_members);
+ }
+
nir_variable *var = make_io_var(c, nir_var_shader_out, t, location, name);
var->data.index = index;
var->data.driver_location = c->nir->num_outputs++;
@@ -822,22 +2644,72 @@ translate_output(struct air_ctx *c, const struct air_md *node, unsigned field,
/* -- instructions ------------------------------------------------------- */
+/* A load through a pointer this frontend could not resolve: load through BOTH
+ * sides of the choice and select the values. That is the only form NIR has --
+ * there is no select of pointers in the 32bit_index_offset model -- and it is
+ * not free of consequence, so it is said out loud: both sides are read, so a
+ * side that is not a valid place to read from is read anyway. In the shaders
+ * that need this one side is a module constant array and the other a buffer
+ * the pipeline binds, so both are readable; a pipeline that bound neither
+ * would read a descriptor it did not set up.
+ */
+static nir_def *
+load_through_choice(struct air_ctx *c, struct air_val *v, const struct air_type *t,
+ unsigned align)
+{
+ if (v->kind == AIR_VAL_PTR)
+ return air_load_ptr(c, v->ptr, t, align);
+ if (v->kind != AIR_VAL_PTR_SELECT) {
+ air_error(c, "instruction: a load through something that is not a pointer");
+ return NULL;
+ }
+ nir_def *a = load_through_choice(c, v->elems[0], t, align);
+ nir_def *b = load_through_choice(c, v->elems[1], t, align);
+ if (!a || !b)
+ return NULL;
+ air_note(c, "a pointer chosen at run time (`select` between two buffers): both sides "
+ "are LOADED and the values selected, because NIR has no select of "
+ "pointers -- so a side the pipeline left unbound is read anyway, and its "
+ "value discarded");
+ return nir_bcsel(&c->b, v->def, a, b);
+}
+
static bool
-translate_gep(struct air_ctx *c, const struct air_inst *in)
+gep_into(struct air_ctx *c, struct air_val *base, const struct air_inst *in,
+ struct air_val **out)
{
- struct air_val *base = air_get(c, in->operands[0]);
- if (!base)
- return false;
+ /* A getelementptr on a pointer CHOSEN at run time: the index goes into both
+ * sides and the choice stays. Both sides have the same pointee type -- the
+ * select that made the choice required it -- so this is one walk per side
+ * and not a different offset on each.
+ */
+ if (base->kind == AIR_VAL_PTR_SELECT) {
+ struct air_val *a = NULL, *b = NULL;
+ if (!gep_into(c, base->elems[0], in, &a) ||
+ !gep_into(c, base->elems[1], in, &b))
+ return false;
+ *out = air_val_ptr_select(c, base->def, a, b);
+ return true;
+ }
if (base->kind != AIR_VAL_PTR)
- return air_error(c, "instruction: getelementptr on something that is not a pointer");
+ return air_error(c, "instruction: getelementptr on something that is not a "
+ "pointer");
struct air_ptr p = base->ptr;
const struct air_type *t = in->aux_type;
- /* index 0 steps over objects of the source type */
+ /* index 0 steps over objects of the source type. A constant one folds, for
+ * the reason the element index below gives: `getelementptr inbounds i8, ptr
+ * %local, i64 4` is the shape a byte offset into a local arrives in, and a
+ * slot can only resolve an offset it can read (slot_cell).
+ */
uint64_t stride = air_type_alloc_size(c->m, t);
uint64_t first_const;
- if (!air_const_uint(in->operands[1], &first_const) || first_const != 0) {
+ if (air_const_uint(in->operands[1], &first_const)) {
+ uint32_t off = (uint32_t)first_const * (uint32_t)stride;
+ if (off)
+ p = air_ptr_add(c, p, nir_imm_int(&c->b, off));
+ } else {
nir_def *first = air_get_ssa(c, in->operands[1]);
if (!first)
return false;
@@ -862,14 +2734,44 @@ translate_gep(struct air_ctx *c, const struct air_inst *in)
} else if (t->kind == AIR_TYPE_ARRAY || t->kind == AIR_TYPE_VECTOR) {
uint64_t es = t->kind == AIR_TYPE_ARRAY ? air_type_alloc_size(c->m, t->elem)
: MAX2(air_type_bit_size(t->elem) / 8, 1);
- p = air_ptr_add(c, p, nir_imul_imm(&c->b, nir_u2u32(&c->b, idx), es));
+ /* A CONSTANT INDEX IS A CONSTANT OFFSET, and saying so here is not an
+ * optimisation: a slot resolves a byte offset into one of its members
+ * only when that offset is decidable while translating (slot_cell), and
+ * `getelementptr inbounds i8, ptr %local, i64 4` -- which is how a byte
+ * offset into a struct reaches one -- used to arrive as an SSA value
+ * because nir_builder does not fold. The arithmetic is the same one
+ * the SSA path performs: nir_u2u32 takes the low 32 bits and
+ * nir_imul_imm multiplies in 32-bit wrapping, so this folds to the
+ * identical number rather than to a more correct one.
+ */
+ uint64_t ci;
+ if (air_const_uint(in->operands[i], &ci)) {
+ uint32_t off = (uint32_t)ci * (uint32_t)es;
+ if (off)
+ p = air_ptr_add(c, p, nir_imm_int(&c->b, off));
+ } else {
+ p = air_ptr_add(c, p, nir_imul_imm(&c->b, nir_u2u32(&c->b, idx), es));
+ }
t = t->elem;
} else {
return air_error(c, "instruction: getelementptr into a scalar");
}
}
p.pointee = t;
- air_set(c, &in->base, air_val_ptr(c, p));
+ *out = air_val_ptr(c, p);
+ return true;
+}
+
+static bool
+translate_gep(struct air_ctx *c, const struct air_inst *in)
+{
+ struct air_val *base = air_get(c, in->operands[0]);
+ if (!base)
+ return false;
+ struct air_val *res = NULL;
+ if (!gep_into(c, base, in, &res))
+ return false;
+ air_set(c, &in->base, res);
return true;
}
@@ -933,10 +2835,21 @@ translate_cast(struct air_ctx *c, const struct air_inst *in)
nir_def *x = air_get_ssa(c, in->operands[0]);
if (!x)
return false;
- unsigned bits = air_nir_bit_size(c, dt);
+ /* A truncation into a width NIR has no register for is the one way such a
+ * value is ever made (narrow_int_bits), so the width is taken from there
+ * rather than from air_nir_bit_size, which would refuse it. Anything else
+ * that touches the result or the source is refused by narrow_int_gate.
+ */
+ unsigned narrow = narrow_int_bits(dt);
+ unsigned bits = narrow ? narrow_promoted_bits(narrow) : air_nir_bit_size(c, dt);
nir_def *res;
switch (in->sub) {
case AIR_CAST_TRUNC:
+ if (narrow) {
+ /* into canonical form: the low N bits, zero above them */
+ res = nir_iand_imm(&c->b, nir_u2uN(&c->b, x, bits), narrow_mask(narrow));
+ break;
+ }
res = bits == 1 ? nir_i2b(&c->b, nir_iand_imm(&c->b, x, 1)) : nir_u2uN(&c->b, x, bits);
break;
case AIR_CAST_ZEXT:
@@ -957,12 +2870,88 @@ translate_cast(struct air_ctx *c, const struct air_inst *in)
break;
case AIR_CAST_FPTRUNC: res = nir_f2fN(&c->b, x, bits); break;
case AIR_CAST_FPEXT: res = nir_f2fN(&c->b, x, bits); break;
- case AIR_CAST_BITCAST:
- if (air_nir_components(dt) != air_nir_components(st) ||
- air_nir_bit_size(c, dt) != air_nir_bit_size(c, st))
- return air_error(c, "instruction: a bitcast that changes the shape of its value");
- res = nir_mov(&c->b, x);
+ /* A bitcast moves no bits: it renames them. When the two types have the
+ * same shape -- the same component count and the same component width --
+ * the rename is a NIR `mov`, because a NIR def carries a width and a
+ * component count and no notion of float or integer, so `i32 -> float` and
+ * `<4 x i32> -> <4 x float>` are already the same def.
+ *
+ * When the shape differs the bits have to be redistributed between
+ * components, and NIR has an operation for exactly one redistribution of
+ * exactly one width: pack_32_2x16 and unpack_32_2x16. That is the whole of
+ * what this corpus asks for, and the number is the reason this is two cases
+ * and not a general repacker -- `tools/air/typecensus.py` over all 7 879 of
+ * Resident Evil 4's modules, reading the types out of llvm-dis rather than
+ * out of our own reader, finds 111 657 bitcasts of 14 distinct type pairs,
+ * and the ONLY two that change shape are:
+ *
+ * i32 -> <2 x half> 44 136 of them, in 2 927 modules
+ * <2 x half> -> i32 4 333 of them, in 688 modules
+ *
+ * with 0 shape-changing bitcasts in the whole 456-module system corpus.
+ * There is no third shape to be found behind these, so the pair below is
+ * not a sample of a family: it is the family. Anything else keeps a refusal
+ * -- and one that names the two shapes, so that a third, if some other
+ * library has one, arrives as an arithmetic fact instead of a mystery.
+ *
+ * The half/half-pair of an `i32` is how an HLSL `min16float2` reaches here
+ * through SPIRV-Cross: the pair travels through the shader packed in one
+ * 32-bit register and is unpacked where it is used.
+ *
+ * Keyed on the SHAPE and not on the AIR type names, deliberately. NIR's
+ * SSA has no float at 32 bits that is distinct from an integer at 32 bits,
+ * so `float -> <2 x half>` -- which this corpus does not contain -- is the
+ * same bit movement as `i32 -> <2 x half>` and is right for the same
+ * reason. What a case keyed on the names would buy is a refusal for a
+ * module whose bits are being moved correctly.
+ *
+ * Which half goes where is little-endian, which is not a choice: LLVM lays
+ * element 0 of a vector at the lowest address, so element 0 of the
+ * `<2 x half>` is bits 0..15 of the `i32`, and `unpack_32_2x16` is defined
+ * `dst.x = src0.x; dst.y = src0.x >> 16` -- the same halves in the same
+ * order. Getting that backwards would not crash and would not look wrong:
+ * both halves are still finite halves, so the shader would draw with every
+ * pair of 16-bit values transposed. tools/air/loweringcheck.py gives the
+ * two halves values that cannot be confused for each other (13 and 113, 1
+ * and -2) and checks them twice -- against LLVM running the same module on
+ * this processor, and against the four numbers written out in the case
+ * list, which come from Python's own binary16 codec and so are not LLVM's
+ * endianness restated. It also says what the transposed answer would have
+ * been. The drawn reference is NOT available for these: softpipe's
+ * nir_to_tgsi has no 16-bit arithmetic at all, with or without a bitcast.
+ */
+ case AIR_CAST_BITCAST: {
+ unsigned dn = air_nir_components(dt), sn = air_nir_components(st);
+ unsigned db = bits, sb = air_nir_bit_size(c, st);
+ if (dn == sn && db == sb)
+ res = nir_mov(&c->b, x);
+ else if (sn == 1 && sb == 32 && dn == 2 && db == 16)
+ res = nir_unpack_32_2x16(&c->b, x);
+ else if (sn == 2 && sb == 16 && dn == 1 && db == 32)
+ res = nir_pack_32_2x16(&c->b, x);
+ /* i32 <-> <4 x i8>: MPS's and CoreMedia's image kernels (mediaserverd's
+ * hbf_compute_residual_error_kernel) take a word apart into its bytes; the
+ * same little-endian order, element 0 the low byte */
+ else if (sn == 1 && sb == 32 && dn == 4 && db == 8)
+ res = nir_unpack_32_4x8(&c->b, x);
+ else if (sn == 4 && sb == 8 && dn == 1 && db == 32)
+ res = nir_pack_32_4x8(&c->b, x);
+ /* i64 <-> <2 x i32>, the low word first */
+ else if (sn == 1 && sb == 64 && dn == 2 && db == 32)
+ res = nir_unpack_64_2x32(&c->b, x);
+ else if (sn == 2 && sb == 32 && dn == 1 && db == 64)
+ res = nir_pack_64_2x32(&c->b, x);
+ /* any other repacking of as many bits (<2 x i32> -> <4 x half> in
+ * mediaserverd's CNN kernels): nir_extract_bits moves them in the same
+ * little-endian order the cases above spell out */
+ else if (sn * sb == dn * db && sb >= 8 && db >= 8 && dn <= NIR_MAX_VEC_COMPONENTS)
+ res = nir_extract_bits(&c->b, &x, 1, 0, dn, db);
+ else
+ return air_error(c, "instruction: a bitcast from %u x %u bits to %u x %u bits, "
+ "which is a repacking this frontend does not do",
+ sn, sb, dn, db);
break;
+ }
default:
return air_error(c, "instruction: cast %u", in->sub);
}
@@ -1012,15 +3001,64 @@ translate_cmp(struct air_ctx *c, const struct air_inst *in)
/* an aggregate value's field, by the index path of extract/insertvalue */
static struct air_val **
-agg_slot(struct air_ctx *c, struct air_val *agg, const uint32_t *idx, unsigned n)
+agg_slot(struct air_ctx *c, struct air_val *agg, const struct air_type *type,
+ const uint32_t *idx, unsigned n)
{
+ /* extractvalue and insertvalue always carry at least one index. Without this
+ * the loop below never runs and the function returns the address of its own
+ * parameter -- which gcc points out (-Wreturn-local-addr) and which would be
+ * a write through a dead stack slot if anything ever took the path.
+ */
+ if (!n) {
+ air_error(c, "instruction: an aggregate access with no index");
+ return NULL;
+ }
struct air_val **slot = &agg;
+ const struct air_type *t = type;
for (unsigned i = 0; i < n; i++) {
- if (!*slot || (*slot)->kind != AIR_VAL_AGG || idx[i] >= (*slot)->num_elems) {
+ /* The TYPE is walked beside the value because the value alone cannot say
+ * how many elements a level has that nothing has written yet.
+ */
+ unsigned count;
+ const struct air_type *et;
+ if (t && t->kind == AIR_TYPE_STRUCT) {
+ count = t->num_members;
+ et = idx[i] < count ? t->members[idx[i]] : NULL;
+ } else if (t && t->kind == AIR_TYPE_ARRAY) {
+ count = t->count > UINT_MAX ? UINT_MAX : (unsigned)t->count;
+ et = idx[i] < count ? t->elem : NULL;
+ } else {
+ count = 0;
+ et = NULL;
+ }
+ if (!et) {
+ air_error(c, "instruction: an aggregate index out of its type");
+ return NULL;
+ }
+ /* A level nothing has written yet, made here rather than refused.
+ *
+ * `insertvalue [3 x [2 x float]] undef, float %x, 0, 0` reaches the row
+ * before the row exists: an undef aggregate is built with its fields
+ * NONE on purpose, so a NESTED one has no row for the column to go in.
+ * Making it changes no value -- an aggregate whose every field is NONE
+ * *is* the undef of that type, which is exactly what air_constant_val
+ * builds for the top level -- so this is the same value said in the form
+ * the walk can continue through.
+ *
+ * It is 16 of RenderBox's 47 modules, every one of them building a 3x2
+ * transform this way before passing it to a shape function, and the
+ * refusal it used to raise ("an aggregate index out of its value") named
+ * the symptom at the second index rather than the missing level at the
+ * first.
+ */
+ if (!*slot)
+ *slot = air_val_agg(c, count);
+ if ((*slot)->kind != AIR_VAL_AGG || idx[i] >= (*slot)->num_elems) {
air_error(c, "instruction: an aggregate index out of its value");
return NULL;
}
slot = &(*slot)->elems[idx[i]];
+ t = et;
}
return slot;
}
@@ -1074,12 +3112,65 @@ air_value_kind_name(const struct air_value *v)
return "a constant of an unknown kind";
}
+/* The one place a value of a width NIR has no register for is let through.
+ *
+ * Such a value is carried in the next width up with the bits above it zero
+ * (narrow_int_bits), which is exact for a short list of operations and wrong
+ * for the rest -- and wrong in the quiet way: `add i3` in a 8-bit register
+ * carries into bit 3 and the shader computes something, so nothing fails. The
+ * list is therefore checked here, over every instruction of every module,
+ * rather than relied on from the four modules that happen to need it: the gate
+ * is what makes the representation safe, and a representation whose safety
+ * rests on nobody writing the other case is not safe.
+ *
+ * The ones that pass are the ones canonical form makes exact: a truncation
+ * into the type, a truncation or a zero-extension out of it, an unsigned
+ * conversion to float, an equality or an unsigned comparison, and a switch,
+ * which the CFG walk emits as equalities against constants of the same width.
+ * Everything else is refused by width, opcode and operand number.
+ */
static bool
-translate_instruction(struct air_ctx *c, const struct air_inst *in)
+narrow_int_gate(struct air_ctx *c, const struct air_inst *in)
{
- switch (in->op) {
- case AIR_OP_BINOP:
- return translate_binop(c, in);
+ unsigned n = narrow_int_bits(in->base.type);
+ if (n && !(in->op == AIR_OP_CAST && in->sub == AIR_CAST_TRUNC))
+ return air_error(c, "instruction: a %u-bit integer made by opcode %u, a width NIR "
+ "has no register for; only a truncation into one is exact in "
+ "the next width up", n, in->op);
+
+ for (unsigned i = 0; i < in->num_operands; i++) {
+ const struct air_value *v = in->operands[i];
+ unsigned m = v ? narrow_int_bits(v->type) : 0;
+ if (!m)
+ continue;
+ if (in->op == AIR_OP_SWITCH)
+ continue;
+ if (in->op == AIR_OP_CMP &&
+ (in->sub == AIR_ICMP_EQ || in->sub == AIR_ICMP_NE ||
+ in->sub == AIR_ICMP_ULT || in->sub == AIR_ICMP_ULE ||
+ in->sub == AIR_ICMP_UGT || in->sub == AIR_ICMP_UGE))
+ continue;
+ if (in->op == AIR_OP_CAST &&
+ (in->sub == AIR_CAST_TRUNC || in->sub == AIR_CAST_ZEXT ||
+ in->sub == AIR_CAST_UITOFP))
+ continue;
+ return air_error(c, "instruction: a %u-bit integer reaching opcode %u as operand "
+ "%u, a width NIR has no register for; only a switch, an "
+ "equality, an unsigned comparison and a widening that ignores "
+ "the sign are exact in the next width up", m, in->op, i);
+ }
+ return true;
+}
+
+static bool
+translate_instruction(struct air_ctx *c, const struct air_inst *in)
+{
+ if (!narrow_int_gate(c, in))
+ return false;
+
+ switch (in->op) {
+ case AIR_OP_BINOP:
+ return translate_binop(c, in);
case AIR_OP_UNOP: {
nir_def *x = air_get_ssa(c, in->operands[0]);
if (!x)
@@ -1094,6 +3185,61 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)
case AIR_OP_GEP:
return translate_gep(c, in);
case AIR_OP_SELECT: {
+ /* A select between two POINTERS, which the GL-on-Metal uber shaders use
+ * to fall back on a module constant array when the pipeline bound no
+ * buffer. It is not a number, so it cannot be a bcsel here; the choice
+ * is kept and made where the pointer is loaded through.
+ */
+ struct air_val *pa = air_get(c, in->operands[1]);
+ struct air_val *pb = air_get(c, in->operands[2]);
+ if (!pa || !pb)
+ return false;
+ if (val_is_pointerish(pa) || val_is_pointerish(pb)) {
+ if (!val_is_pointerish(pa) || !val_is_pointerish(pb))
+ return air_error(c, "instruction: a select between a pointer and "
+ "something that is not one");
+ /* A choice that is not a choice. The GL-on-Metal uber shaders pick
+ * between the pipeline's sampler and their own inline one on a
+ * function-constant predicate, and once that predicate is an
+ * immediate there is one pointer here and not two -- which matters
+ * because the alternative below keeps the choice and a TEXTURE or a
+ * SAMPLER is not a value it can load. Folding first is also what
+ * answers the argument-buffer case under it for free when the
+ * condition is known, instead of refusing a select that has already
+ * been decided.
+ */
+ nir_def *pcond = air_get_ssa(c, in->operands[0]);
+ bool ptaken;
+ if (pcond && pcond->num_components == 1 &&
+ cond_const(c, pcond, &ptaken)) {
+ air_set(c, &in->base, ptaken ? pa : pb);
+ return true;
+ }
+ /* A choice between two members of an argument buffer cannot be kept
+ * the way a choice between two buffers can: AIR_VAL_PTR_SELECT loads
+ * both sides and selects the values, and a texture or a sampler is
+ * not a value to load. Two names for ONE member is no choice at all
+ * and is answered; anything else is refused rather than drawn over
+ * whichever of the two the code happened to take.
+ */
+ if ((pa->kind == AIR_VAL_PTR && pa->ptr.class == AIR_PTR_ARGBUF) ||
+ (pb->kind == AIR_VAL_PTR && pb->ptr.class == AIR_PTR_ARGBUF)) {
+ if (pa->kind == AIR_VAL_PTR && pb->kind == AIR_VAL_PTR &&
+ same_ptr(&pa->ptr, &pb->ptr)) {
+ air_set(c, &in->base, pa);
+ return true;
+ }
+ return air_error(c, "instruction: a select between two different places in "
+ "an argument buffer");
+ }
+ nir_def *pc = air_get_ssa(c, in->operands[0]);
+ if (!pc)
+ return false;
+ if (pc->num_components != 1)
+ return air_error(c, "instruction: a vector select between pointers");
+ air_set(c, &in->base, air_val_ptr_select(c, pc, pa, pb));
+ return true;
+ }
nir_def *cond = air_get_ssa(c, in->operands[0]);
nir_def *x = air_get_ssa(c, in->operands[1]);
nir_def *y = air_get_ssa(c, in->operands[2]);
@@ -1109,7 +3255,17 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)
nir_def *i = air_get_ssa(c, in->operands[1]);
if (!v || !i)
return false;
- air_set(c, &in->base, air_val_ssa(c, nir_vector_extract(&c->b, v, nir_u2u32(&c->b, i))));
+ /* A BOOLEAN index. `extractelement <2 x i1> %54, i1 %55` is in
+ * RenderBox's stroke shaders -- the line's two caps picked by which end
+ * of the segment the pixel is nearer -- and `u2u32` is the wrong
+ * conversion for it: a NIR boolean is a 1-bit def, not a 1-bit integer,
+ * and widening one with an unsigned conversion is undefined where
+ * nir_b2i32 is exactly 0 or 1. ir3 says so outright ("invalid src bit
+ * size: 1") and refuses the shader; a backend that did not say so would
+ * have read lane ~0u instead of lane 1.
+ */
+ nir_def *idx = i->bit_size == 1 ? nir_b2i32(&c->b, i) : nir_u2u32(&c->b, i);
+ air_set(c, &in->base, air_val_ssa(c, nir_vector_extract(&c->b, v, idx)));
return true;
}
case AIR_OP_INSERTELT: {
@@ -1118,11 +3274,27 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)
nir_def *i = air_get_ssa(c, in->operands[2]);
if (!v || !x || !i)
return false;
- if (!nir_src_is_const(nir_src_for_ssa(i)))
- return air_error(c, "instruction: insertelement at a run-time index");
- air_set(c, &in->base,
- air_val_ssa(c, nir_vector_insert_imm(&c->b, v, x,
- nir_src_as_uint(nir_src_for_ssa(i)))));
+ /* AN INDEX DECIDED WHILE THE SHADER RUNS, which NIR has its own lowering
+ * for: nir_vector_insert builds `bcsel(i == {0,1,2,3}, scalar, vec)`,
+ * one comparison per component, and takes the constant path when the
+ * index is one. So this is Mesa's rule rather than one written here, and
+ * the constant case goes through exactly the code it went through before
+ * (nir_vector_insert calls nir_vector_insert_imm itself).
+ *
+ * The one place the two languages differ is an index PAST the vector.
+ * LLVM says an out-of-range insertelement is poison; the bcsel chain
+ * misses every component and yields the vector unchanged. Poison permits
+ * any value, so an unchanged vector is a refinement of it and not a
+ * disagreement -- which is the only reason this can be done without
+ * inventing a semantics.
+ *
+ * It is the wall fixed_frag_lph_cph reaches once its texture and sampler
+ * choices are resolved, under BOTH the all-ones configuration and the
+ * compositor's own (docs/air.md section 3), which is what made it worth
+ * taking rather than refusing: two configurations that disagree about
+ * everything else converge here.
+ */
+ air_set(c, &in->base, air_val_ssa(c, nir_vector_insert(&c->b, v, x, i)));
return true;
}
case AIR_OP_SHUFFLEVEC: {
@@ -1186,14 +3358,23 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)
struct air_val *agg = air_get(c, in->operands[0]);
if (!agg)
return false;
- struct air_val **slot = agg_slot(c, agg, in->indices, in->num_indices);
+ struct air_val **slot = agg_slot(c, agg, in->operands[0]->type,
+ in->indices, in->num_indices);
if (!slot)
return false;
if (!*slot) {
- /* a field of an undef aggregate */
- air_set(c, &in->base,
- air_val_ssa(c, nir_undef(&c->b, air_nir_components(in->base.type),
- air_nir_bit_size(c, in->base.type))));
+ /* A field of an undef aggregate: the undef of that field's OWN type,
+ * built the one way this translator knows how to build an undef of
+ * anything -- which matters because the field may itself be an
+ * aggregate, and a nir_undef cannot be one.
+ */
+ struct air_constant u = { .base = { .kind = AIR_VALUE_CONSTANT,
+ .type = in->base.type },
+ .ckind = AIR_CONST_UNDEF };
+ struct air_val *v = air_constant_val(c, &u);
+ if (!v)
+ return false;
+ air_set(c, &in->base, v);
return true;
}
air_set(c, &in->base, *slot);
@@ -1205,7 +3386,8 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)
if (!agg || !val)
return false;
struct air_val *copy = clone_val(c, agg);
- struct air_val **slot = agg_slot(c, copy, in->indices, in->num_indices);
+ struct air_val **slot = agg_slot(c, copy, in->operands[0]->type,
+ in->indices, in->num_indices);
if (!slot)
return false;
*slot = val;
@@ -1219,6 +3401,48 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)
uint64_t count;
if (!air_const_uint(in->operands[0], &count))
return air_error(c, "instruction: an alloca of a run-time size");
+ /* An alloca is bytes in scratch -- unless it holds a POINTER, which
+ * scratch cannot: a pointer in this frontend is a resolution, a binding
+ * and a byte offset decided while translating, and has no bytes to put
+ * anywhere. Then it is a translation-time SLOT instead (AIR_PTR_SLOT in
+ * air_private.h, and the whole-store and whole-load cases above).
+ *
+ * Spelled out because what stood here twice described code that was not
+ * here. It first promised a slot "see AIR_VAL_SLOT" when no such value
+ * kind existed -- that slot had been written and removed -- and the
+ * correction then said no slot would help, which was true of the module
+ * count and not of the wall: the slot closes the FIRST of four walls in
+ * the two GL-on-Metal fragment shaders, and the three behind it are
+ * either closed by specialising on the function constants
+ * (plan_global_folding) or, in the case of air.get_null_texture_2d, a
+ * refusal that now says whose wall it is. tools/air/chosencheck.sh is
+ * the record of all four.
+ *
+ * The same comment also said those two shaders "store their last two
+ * buffer arguments into such a local and never read them back".
+ * Parameters 34 and 35 of fixed_frag_lph_cph are `air.sampler`
+ * (img_samp_0, img_samp_1), not buffers, and the module does read them
+ * back: `select` against its own inline sampler, then `load`, into the
+ * sampler argument of 27 air.sample_texture_2d calls and 12 calls to its
+ * own box-sample functions. That is why the slot has to hold a whole
+ * air_val and not a number.
+ */
+ /* ONE object that holds a POINTER: a translation-time slot, because
+ * scratch cannot hold one. `count == 1` and nothing else, so an array of
+ * pointers still goes to memory and still refuses at the store -- an
+ * array would need an index, and an index into a tree of values is the
+ * thing this deliberately does not do.
+ */
+ if (count == 1 && type_has_pointer(in->aux_type, 8)) {
+ struct air_ptr sp = {
+ .class = AIR_PTR_SLOT,
+ .binding = new_slot(c, in->aux_type, NULL),
+ .pointee = in->aux_type,
+ };
+ air_set(c, &in->base, air_val_ptr(c, sp));
+ return true;
+ }
+
uint64_t size = air_type_alloc_size(c->m, in->aux_type) * count;
unsigned align = MAX2(in->align, air_type_abi_align(c->m, in->aux_type));
c->scratch_size = ALIGN_POT(c->scratch_size, MAX2(align, 1));
@@ -1232,11 +3456,71 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)
return true;
}
case AIR_OP_LOAD: {
+ /* A program-scope variable whose single static-initialiser store turned
+ * out to be a constant: the load IS that constant, rebuilt in whatever
+ * builder is asking. Rebuilt and not shared, because a nir_def belongs
+ * to one impl and a callee may load the same variable -- which is the
+ * same rule materialise_globals follows for the offset.
+ */
+ struct air_global_plan *gpl = foldable_global_plan(c, in->operands[0]);
+ if (gpl && gpl->known) {
+ air_set(c, &in->base,
+ air_val_ssa(c, nir_imm_intN_t(&c->b, gpl->bits, gpl->bit_size)));
+ return true;
+ }
struct air_val *p = air_get(c, in->operands[0]);
if (!p)
return false;
+ unsigned slot;
+ if (slot_base(c, p, &slot)) {
+ struct air_val *v = slot_at_zero(c->slots[slot].val, in->base.type);
+ if (!v)
+ return air_error(c, "instruction: a load from a local that holds a "
+ "pointer and has not been written");
+ air_set(c, &in->base, v);
+ return true;
+ }
+ /* Not the slot's base: a member of it, if the offset and the loaded type
+ * name one between them (slot_cell). Tried AFTER slot_base and not
+ * instead of it, so that nothing that translated before this existed can
+ * take a different path: offset zero with a type slot_at_zero's looser
+ * descent accepts still goes the way it went.
+ */
+ uint64_t soff;
+ if (slot_at_const(c, p, &slot, &soff)) {
+ struct air_val **cell = slot_cell(c, slot, soff, in->base.type);
+ if (cell && *cell) {
+ air_set(c, &in->base, *cell);
+ return true;
+ }
+ if (cell)
+ return air_error(c, "instruction: a load from a member of a local that "
+ "holds a pointer and has not been written");
+ }
+ if (p->kind == AIR_VAL_PTR && p->ptr.class == AIR_PTR_SLOT)
+ return air_error(c, "instruction: a load at an offset into a local that holds "
+ "a pointer which does not name one of its members, so "
+ "there is no value there to give");
+ if (p->kind == AIR_VAL_PTR_SELECT) {
+ nir_def *res = load_through_choice(c, p, in->base.type, in->align);
+ if (!res)
+ return false;
+ air_set(c, &in->base, air_val_ssa(c, res));
+ return true;
+ }
if (p->kind != AIR_VAL_PTR)
return air_error(c, "instruction: a load from something that is not a pointer");
+ /* An argument buffer resolves before it loads: the byte offset names a
+ * member and the member's descriptor names a binding, so this load
+ * becomes a pointer and not an instruction.
+ */
+ if (p->ptr.class == AIR_PTR_ARGBUF) {
+ struct air_val *res = argbuf_load(c, p->ptr, in->base.type);
+ if (!res)
+ return false;
+ air_set(c, &in->base, res);
+ return true;
+ }
nir_def *res = air_load_ptr(c, p->ptr, in->base.type, in->align);
if (!res)
return false;
@@ -1245,11 +3529,81 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)
}
case AIR_OP_STORE: {
struct air_val *p = air_get(c, in->operands[1]);
+ if (!p)
+ return false;
+ /* A POINTER stored into memory. There is nothing to write: a pointer in
+ * this frontend is a resolution -- a binding and a byte offset decided
+ * while translating -- and not an address, so it has no bytes. Naming
+ * that here rather than letting air_get_ssa say "a pointer where a
+ * number belongs" is the difference between a wall with a reason and a
+ * wall with a symptom; the reason is in docs/air.md section 3.
+ */
+ struct air_val *sv = air_get(c, in->operands[0]);
+ if (!sv)
+ return false;
+ unsigned sslot;
+ uint64_t soff;
+ if (slot_base(c, p, &sslot) &&
+ in->operands[0]->type == c->slots[sslot].type) {
+ if (c->slots[sslot].init)
+ return air_error(c, "instruction: a store into a read-only global, which is "
+ "where its own initialiser lives");
+ c->slots[sslot].val = sv;
+ return true;
+ }
+ /* A store into one MEMBER of a slot. The type stored and the byte offset
+ * name the member between them, and neither does on its own
+ * (slot_cell). This is what the four `disparity_refinement_*` stop at:
+ * CoreImage's stitched functions rebuild a `coreimage::Sampler` in a
+ * local out of the four arguments it was flattened into.
+ */
+ if (slot_at_const(c, p, &sslot, &soff)) {
+ if (c->slots[sslot].init)
+ return air_error(c, "instruction: a store into a read-only global, which is "
+ "where its own initialiser lives");
+ struct air_val **cell = slot_cell(c, sslot, soff,
+ in->operands[0]->type);
+ if (cell) {
+ *cell = sv;
+ return true;
+ }
+ return air_error(c, "instruction: a store into a local that holds a pointer at "
+ "byte %u, which names no member of it: a tree of values has "
+ "no bytes for an offset that straddles two of them",
+ (unsigned)soff);
+ }
+ if (p->kind == AIR_VAL_PTR && p->ptr.class == AIR_PTR_SLOT)
+ return air_error(c, "instruction: a store at an offset computed at run time "
+ "into a local that holds a pointer, so which member it "
+ "names is not decidable here");
+ if (val_is_pointerish(sv))
+ return air_error(c, "instruction: a store of a pointer into memory, which this "
+ "frontend has no address to write");
+ if (p->kind == AIR_VAL_PTR_SELECT)
+ return air_error(c, "instruction: a store through a chosen pointer, which would "
+ "have to write both of them");
nir_def *v = air_get_ssa(c, in->operands[0]);
- if (!p || !v)
+ if (!v)
return false;
if (p->kind != AIR_VAL_PTR)
return air_error(c, "instruction: a store to something that is not a pointer");
+ /* The one store into a foldable program-scope variable. Its value is
+ * recorded when it is a constant, and the memory is written either way:
+ * plan_global_folding has proved that nothing else reaches these bytes,
+ * so the store is dead when the fold answers -- and leaving it in costs
+ * one DCE'd instruction, where leaving it OUT would be a promise about
+ * every access shape that scan might have missed.
+ */
+ struct air_global_plan *gpl = foldable_global_plan(c, in->operands[1]);
+ if (gpl) {
+ nir_const_value cv[NIR_MAX_VEC_COMPONENTS];
+ if (v->num_components == 1 &&
+ def_const(v, cv, 16, c->nir->info.float_controls_execution_mode)) {
+ gpl->known = true;
+ gpl->bits = nir_const_value_as_uint(cv[0], v->bit_size);
+ gpl->bit_size = v->bit_size;
+ }
+ }
return air_store_ptr(c, p->ptr, v, in->align);
}
case AIR_OP_CALL: {
@@ -1260,6 +3614,7 @@ translate_instruction(struct air_ctx *c, const struct air_inst *in)
}
case AIR_OP_RET:
case AIR_OP_BR:
+ case AIR_OP_SWITCH:
case AIR_OP_UNREACHABLE:
case AIR_OP_PHI:
return true; /* handled by the CFG walk */
@@ -1296,6 +3651,30 @@ emit_return(struct air_ctx *c, const struct air_inst *in)
* caller passed, because a NIR call has no destination. The entry point
* has no such parameter and its return is its render targets, below.
*/
+ if (c->num_ret_fields) {
+ /* A callee returning a struct: one out-parameter per member, because a
+ * NIR parameter carries a number and a struct here is a tree of values.
+ * The members with no parameter are the ones nothing can read (see
+ * ret_fields in air_private.h), and a member the callee left out of its
+ * insertvalue chain is undef -- Metal's own vertex-output structs leave
+ * their padding field exactly that way, so this is the normal case and
+ * not a defect.
+ */
+ if (v->kind != AIR_VAL_AGG || v->num_elems != c->num_ret_fields)
+ return air_error(c, "instruction: a returned struct that was never built, "
+ "from a function called in the module");
+ for (unsigned i = 0; i < v->num_elems; i++) {
+ if (!c->ret_fields[i] || !v->elems[i])
+ continue;
+ if (v->elems[i]->kind != AIR_VAL_SSA)
+ return air_error(c, "instruction: field %u of a returned struct is not a "
+ "number, from a function called in the module", i);
+ nir_def *d = v->elems[i]->def;
+ nir_store_deref(&c->b, c->ret_fields[i], d,
+ nir_component_mask(d->num_components));
+ }
+ return true;
+ }
if (c->ret_deref) {
nir_def *d = air_get_ssa(c, in->operands[0]);
if (!d)
@@ -1314,8 +3693,11 @@ emit_return(struct air_ctx *c, const struct air_inst *in)
return air_error(c, "interface: field %u of the result has no output", i);
if (!v->elems[i] || v->elems[i]->kind != AIR_VAL_SSA)
continue; /* a field left undef is a target nothing writes */
- nir_store_var(&c->b, c->outputs[i].var, v->elems[i]->def,
- BITFIELD_MASK(v->elems[i]->def->num_components));
+ nir_def *d = v->elems[i]->def;
+ if (io_is_boolean(rt->members[i]) && d->bit_size == 1)
+ d = nir_b2iN(&c->b, d, 32); /* see io_is_boolean */
+ nir_store_var(&c->b, c->outputs[i].var, d,
+ BITFIELD_MASK(d->num_components));
}
return true;
}
@@ -1373,6 +3755,372 @@ reverse_post_order(struct air_ctx *c, const struct air_function *f, unsigned *or
static bool translate_body(struct air_ctx *c);
static void materialise_globals(struct air_ctx *c);
+/* A module's globals and its static initialisers: per MODULE, and a stitched
+ * shader is built from several, so these run once for each of them rather than
+ * once for the shader (stitch_graph below).
+ */
+static void plan_globals(struct air_ctx *c);
+static void run_static_initializers(struct air_ctx *c);
+
+/* Does this value hold a pointer anywhere inside it? */
+static bool
+val_holds_ptr(const struct air_val *v)
+{
+ if (!v)
+ return false;
+ if (val_is_pointerish(v))
+ return true;
+ if (v->kind != AIR_VAL_AGG)
+ return false;
+ for (unsigned i = 0; i < v->num_elems; i++)
+ if (val_holds_ptr(v->elems[i]))
+ return true;
+ return false;
+}
+
+/* -- an aggregate handed to a call of a module function -------------------
+ *
+ * A NIR function parameter carries a NUMBER -- a scalar, or a vector of them.
+ * A struct or an array is a tree of values in this translator and never a NIR
+ * value at all (air_private.h), so a function called with one BY VALUE has to
+ * take the numbers inside it, one parameter each, and put the tree back
+ * together on the other side.
+ *
+ * This is what Metal's own compiler asks for here and not an exotic case.
+ * RenderBox splits its shader state into one-field `RenderState` structs and
+ * hands every shape, blend, filter and resolve function its globals as a
+ * struct of matrices -- `{ [3 x [2 x float]], float }`, and for the blur
+ * `{ float, [2 x float], [2 x float], [2 x float], [4 x [2 x float]],
+ * [4 x half] }`, nineteen numbers in one argument. 26 of that library's 47
+ * modules stop at such a call.
+ *
+ * The order is extractvalue's: member 0's numbers, then member 1's, depth
+ * first. The caller and the callee walk the same type with the same function,
+ * which is what keeps the sources and the parameters in step -- and every call
+ * site checks the count it built against nir_call_instr.num_params as well,
+ * because "in step" is a claim and a call whose sources are one short is a
+ * segfault inside nir_instr_insert rather than a message.
+ */
+#define AIR_MAX_ARG_LEAVES 64
+
+/* the numbers inside a type, or why it has one that no parameter can carry */
+static bool
+type_leaves(struct air_ctx *c, const struct air_type *t,
+ const struct air_type **out, unsigned *n)
+{
+ if (!t)
+ return air_error(c, "intrinsic: a call argument of no type");
+ if (t->kind == AIR_TYPE_STRUCT || t->kind == AIR_TYPE_ARRAY) {
+ unsigned count = t->kind == AIR_TYPE_STRUCT ? t->num_members
+ : (unsigned)MIN2(t->count, UINT_MAX);
+ for (unsigned i = 0; i < count; i++) {
+ const struct air_type *et = t->kind == AIR_TYPE_STRUCT ? t->members[i] : t->elem;
+ if (!type_leaves(c, et, out, n))
+ return false;
+ }
+ return true;
+ }
+ if (t->kind != AIR_TYPE_VECTOR && t->kind != AIR_TYPE_INT && !air_type_is_float(t))
+ return air_error(c, "intrinsic: a call passing %s inside a value, which is not a "
+ "number a parameter can carry",
+ t->kind == AIR_TYPE_POINTER ? "a pointer" : "a type of no size");
+ if (*n >= AIR_MAX_ARG_LEAVES)
+ return air_error(c, "intrinsic: a call passing a value of more than %u numbers",
+ AIR_MAX_ARG_LEAVES);
+ if (out)
+ out[*n] = t;
+ (*n)++;
+ return true;
+}
+
+/* the caller's side: the defs to pass, in that same order */
+static bool
+val_leaves(struct air_ctx *c, const struct air_type *t, struct air_val *v,
+ nir_def **out, unsigned *n)
+{
+ if (!t)
+ return air_error(c, "intrinsic: a call argument of no type");
+ if (t->kind == AIR_TYPE_STRUCT || t->kind == AIR_TYPE_ARRAY) {
+ if (v && v->kind != AIR_VAL_AGG)
+ return air_error(c, "intrinsic: a call passing %s where its type says an "
+ "aggregate",
+ v->kind == AIR_VAL_SSA ? "a number" : "a pointer");
+ unsigned count = t->kind == AIR_TYPE_STRUCT ? t->num_members
+ : (unsigned)MIN2(t->count, UINT_MAX);
+ for (unsigned i = 0; i < count; i++) {
+ const struct air_type *et = t->kind == AIR_TYPE_STRUCT ? t->members[i] : t->elem;
+ if (!val_leaves(c, et, v && i < v->num_elems ? v->elems[i] : NULL, out, n))
+ return false;
+ }
+ return true;
+ }
+ if (*n >= AIR_MAX_ARG_LEAVES)
+ return air_error(c, "intrinsic: a call passing a value of more than %u numbers",
+ AIR_MAX_ARG_LEAVES);
+ if (!v) {
+ /* A number nothing wrote is undef, which is what the aggregate already
+ * says: an undef struct is built with its fields NONE (air_constant_val),
+ * and Metal's compiler does pass whole `undef` globals -- filter_blur is
+ * handed `PlaneGlobals undef` because the blur reads none of it.
+ */
+ out[(*n)++] = nir_undef(&c->b, air_nir_components(t), air_narrow_bits(c, t));
+ return true;
+ }
+ if (v->kind != AIR_VAL_SSA)
+ return air_error(c, "intrinsic: a call passing %s where a number belongs",
+ v->kind == AIR_VAL_AGG ? "an aggregate" : "a pointer");
+ out[(*n)++] = v->def;
+ return true;
+}
+
+/* the callee's side: the tree, back out of its parameters */
+static struct air_val *
+val_from_leaves(struct air_ctx *c, const struct air_type *t, nir_def **in, unsigned *n)
+{
+ if (t->kind == AIR_TYPE_STRUCT || t->kind == AIR_TYPE_ARRAY) {
+ unsigned count = t->kind == AIR_TYPE_STRUCT ? t->num_members
+ : (unsigned)MIN2(t->count, UINT_MAX);
+ struct air_val *agg = air_val_agg(c, count);
+ for (unsigned i = 0; i < count; i++) {
+ const struct air_type *et = t->kind == AIR_TYPE_STRUCT ? t->members[i] : t->elem;
+ if (!(agg->elems[i] = val_from_leaves(c, et, in, n)))
+ return NULL;
+ }
+ return agg;
+ }
+ return air_val_ssa(c, in[(*n)++]);
+}
+
+/* Whether a resolution inside this value carries an offset computed at run
+ * time. Checked before anything is built, so the refusal names the argument
+ * rather than arriving from the middle of the callee's body.
+ */
+static bool
+val_offset_is_computed(const struct air_val *v)
+{
+ if (!v)
+ return false;
+ if (v->kind == AIR_VAL_AGG) {
+ for (unsigned i = 0; i < v->num_elems; i++)
+ if (val_offset_is_computed(v->elems[i]))
+ return true;
+ return false;
+ }
+ return v->kind == AIR_VAL_PTR && v->ptr.offset &&
+ !nir_src_is_const(nir_src_for_ssa(v->ptr.offset));
+}
+
+/* The caller's resolution, with every byte offset in it rebuilt in whichever
+ * builder is current now.
+ *
+ * A pointer here is *(where, byte offset)* and the offset is an nir_def, which
+ * belongs to the impl that made it; handing the callee the caller's def is the
+ * cross-function def/use that the globals used to cause, and that is why an
+ * argument carrying one was refused outright.
+ *
+ * A CONSTANT offset need not be refused, and every one that occurs is
+ * constant: what a RenderBox shader hands its shape function is an ALLOCA,
+ * whose offset is the scratch cursor at translation time (AIR_OP_ALLOCA builds
+ * nir_imm_int of it), and the callee reads the struct back out of scratch with
+ * geps of its own. Nothing is shared -- the NUMBER is copied and a new def
+ * made here -- so the callee's offset is a def of the callee's own impl while
+ * the resolution is the same resolution. The scratch it names is the same
+ * scratch because air_ctx.scratch_size is the SHADER's, not the function's,
+ * and nir_inline_functions folds the callee into the caller afterwards anyway.
+ *
+ * FOURTEEN of RenderBox's 47 modules stop here once the aggregate-index barrier
+ * above is lifted, chronod's first fault among them, and thirteen of the
+ * fourteen need only this: the one that does not is uber_fragment, whose
+ * offset is read out of its globals and goes through ptr_offset_to_pass. The
+ * number is the frontend's own count of the modules that reported this
+ * refusal, not a reading of the IR -- a scan of the bitcode for "an alloca
+ * passed to a call" gave 12 and then 14 depending on how the call line was
+ * matched, which is a reason to take the count from the thing that counts.
+ */
+/* The byte offset of a pointer argument that has to be PASSED as a number
+ * rather than copied as one.
+ *
+ * Where a pointer points is a translation-time fact -- which buffer, at which
+ * Metal index -- and that half is bound into the callee whatever the offset is.
+ * The offset is the half that can be a VALUE: uber_fragment reads an index out
+ * of its globals and hands `device void *` at that byte to sample_image
+ * (`getelementptr inbounds i8, ptr addrspace(1) %7, i64 %416`), so there is
+ * nothing constant to copy and nothing resolved to bind.
+ *
+ * So the binding is bound and the offset becomes one more NIR parameter, which
+ * puts the def on the right side of the function boundary: the caller passes
+ * its own def as a call source, and the callee's offset is its own
+ * load_param. That is the same thing nir_call does for every other number and
+ * it is why this is exact rather than a workaround.
+ *
+ * Only a pointer that IS the argument, not one inside a struct: a resolution
+ * nested in an aggregate with a run-time offset has no case in either corpus,
+ * and rebind_here still refuses it by name rather than guessing a layout for
+ * the parameters it would take.
+ */
+static nir_def *
+ptr_offset_to_pass(struct air_val *v)
+{
+ if (!v || v->kind != AIR_VAL_PTR || !v->ptr.offset)
+ return NULL;
+ return nir_src_is_const(nir_src_for_ssa(v->ptr.offset)) ? NULL : v->ptr.offset;
+}
+
+/* A SLOT PASSED TO A CALL, and the parameters it costs.
+ *
+ * A slot NUMBER means the same thing in a callee as in a caller -- air_ctx.slots
+ * is the SHADER's table, not the function's, the same way scratch_size is -- so
+ * what cannot cross a function boundary is not the slot but the DEFS inside it.
+ * A slot holds a tree of resolutions, and two kinds of thing in that tree belong
+ * to the caller's impl: a plain value (`coreimage::Sampler`'s `float2 coord`),
+ * and a pointer's byte offset when it is computed rather than constant. Both go
+ * the way the run-time offset of a plain pointer argument goes
+ * (ptr_offset_to_pass): one more nir_parameter the caller supplies as a call
+ * source and the callee reads with load_param, which puts the def on the side of
+ * the boundary it belongs to instead of hiding the crossing.
+ *
+ * It used to be refused outright, as a latent hole nothing asked for. Something
+ * asks for it now: the four `disparity_refinement_*` of
+ * CIPortraitBlurStitchableV3 are a stitched function whose first act is to
+ * rebuild the `coreimage::Sampler` CoreImage flattened into four arguments, and
+ * 126 modules of the image do the same thing (docs/air.md 10).
+ *
+ * THE TWO FUNCTIONS BELOW ARE A PAIR AND THE ORDER IS THE CONTRACT. slot_leaves
+ * collects the caller's defs; rebind_slot_contents rebuilds the tree with the
+ * callee's load_params in their place. They walk identically -- aggregate
+ * elements in index order, then a pointer's offset, then a value -- and the
+ * caller checks the two counts against each other rather than trusting them,
+ * because a call one source short is a null nir_src that nir_instr_insert
+ * dereferences while adding uses: a segfault, not a message. That is the same
+ * rule, for the same reason, as the type_leaves/val_leaves pair below.
+ */
+#define AIR_MAX_SLOT_LEAVES 16
+
+static bool
+slot_leaves(struct air_ctx *c, struct air_val *v, nir_def **out, unsigned *n)
+{
+ if (!v)
+ return true;
+ switch (v->kind) {
+ case AIR_VAL_AGG:
+ for (unsigned i = 0; i < v->num_elems; i++)
+ if (!slot_leaves(c, v->elems[i], out, n))
+ return false;
+ return true;
+ case AIR_VAL_PTR:
+ if (!v->ptr.offset || nir_src_is_const(nir_src_for_ssa(v->ptr.offset)))
+ return true;
+ break;
+ case AIR_VAL_SSA:
+ if (!v->def)
+ return false;
+ break;
+ default:
+ /* a pointer chosen at run time, or a cell nothing wrote: neither has a
+ * form here, and both are named by the caller
+ */
+ return false;
+ }
+ if (*n >= AIR_MAX_SLOT_LEAVES)
+ return false;
+ if (out)
+ out[*n] = v->kind == AIR_VAL_SSA ? v->def : v->ptr.offset;
+ (*n)++;
+ return true;
+}
+
+/* The caller's tree, rebuilt for the function being translated: `got` holds one
+ * load_param per leaf slot_leaves found, in that order, and `k` walks it.
+ */
+static struct air_val *
+rebind_slot_contents(struct air_ctx *c, struct air_val *v, nir_def **got,
+ unsigned ngot, unsigned *k)
+{
+ if (!v)
+ return v;
+ switch (v->kind) {
+ case AIR_VAL_AGG: {
+ struct air_val *n = air_val_agg(c, v->num_elems);
+ for (unsigned i = 0; i < v->num_elems; i++) {
+ if (!v->elems[i])
+ continue;
+ if (!(n->elems[i] = rebind_slot_contents(c, v->elems[i], got, ngot, k)))
+ return NULL;
+ }
+ return n;
+ }
+ case AIR_VAL_PTR: {
+ struct air_ptr p = v->ptr;
+ if (!p.offset)
+ return v;
+ nir_src src = nir_src_for_ssa(p.offset);
+ if (nir_src_is_const(src)) {
+ /* a constant of the caller is still a def of the caller's impl */
+ p.offset = nir_imm_int(&c->b, nir_src_as_uint(src));
+ return air_val_ptr(c, p);
+ }
+ if (*k >= ngot)
+ return NULL;
+ p.offset = got[(*k)++];
+ return air_val_ptr(c, p);
+ }
+ case AIR_VAL_SSA:
+ if (*k >= ngot)
+ return NULL;
+ return air_val_ssa(c, got[(*k)++]);
+ default:
+ return NULL;
+ }
+}
+
+/* The defs a slot argument must pass, or n = 0 for an argument that is not one.
+ * One function so that the three places that have to agree -- the parameter
+ * count, the parameter declarations and the call's sources -- ask the same
+ * question of the same value.
+ */
+static bool
+slot_arg_leaves(struct air_ctx *c, struct air_val *v, nir_def **out, unsigned *n)
+{
+ *n = 0;
+ if (!v || v->kind != AIR_VAL_PTR || v->ptr.class != AIR_PTR_SLOT)
+ return true;
+ if (v->ptr.binding >= c->num_slots)
+ return false;
+ if (v->ptr.offset &&
+ (!nir_src_is_const(nir_src_for_ssa(v->ptr.offset)) ||
+ nir_src_as_uint(nir_src_for_ssa(v->ptr.offset)) != 0))
+ return false; /* a member of a slot, not the whole of it */
+ return slot_leaves(c, c->slots[v->ptr.binding].val, out, n);
+}
+
+static struct air_val *
+rebind_here(struct air_ctx *c, struct air_val *v, unsigned arg)
+{
+ if (!v)
+ return v;
+ if (v->kind == AIR_VAL_AGG) {
+ struct air_val *n = air_val_agg(c, v->num_elems);
+ for (unsigned i = 0; i < v->num_elems; i++) {
+ if (!v->elems[i])
+ continue;
+ if (!(n->elems[i] = rebind_here(c, v->elems[i], arg)))
+ return NULL;
+ }
+ return n;
+ }
+ if (v->kind != AIR_VAL_PTR || !v->ptr.offset)
+ return v;
+ nir_src src = nir_src_for_ssa(v->ptr.offset);
+ if (!nir_src_is_const(src)) {
+ air_error(c, "intrinsic: a call passing a pointer at an offset computed at run "
+ "time as argument %u, whose value belongs to the caller's own "
+ "function", arg);
+ return NULL;
+ }
+ struct air_ptr p = v->ptr;
+ p.offset = nir_imm_int(&c->b, nir_src_as_uint(src));
+ return air_val_ptr(c, p);
+}
/* A function of the module, translated as a nir_function so that
* nir_inline_functions can fold it into its callers afterwards.
@@ -1401,18 +4149,52 @@ translate_callee(struct air_ctx *c, const struct air_function *f,
* callee to its call site -- so it is not cached. Each of the four modules
* that needs this has exactly one call site, counted beforehand.
*
- * A pointer carrying a byte offset is refused instead: that offset is an
- * nir_def of the CALLER's impl, and using it here would be the same
- * cross-function def/use that the globals used to cause.
+ * A pointer carrying a byte offset that is a CONSTANT is bound the same way,
+ * with the number copied into a def of this function's own impl
+ * (rebind_here); one computed at run time is refused, because using the
+ * caller's def here is the cross-function def/use that the globals used to
+ * cause.
*/
bool specialised = false;
for (unsigned i = 0; i < f->num_args; i++) {
- bool ptr = argv && argv[i] && argv[i]->kind == AIR_VAL_PTR;
- if (ptr && argv[i]->ptr.offset) {
- air_error(c, "intrinsic: a call passing a pointer at a computed offset as "
- "argument %u, which would use the caller's own value here", i);
+ /* A struct holding a pointer counts as one too: CoreImage passes a
+ * texture and a sampler as metal::texture2d and metal::sampler, which
+ * are one-field structs wrapping the address-space-1 and -2 pointers.
+ * Passing such a struct as a NIR value would hand the callee something
+ * it then takes extractvalue of, and extractvalue of a value that is not
+ * an aggregate here is exactly the error this used to produce.
+ */
+ bool ptr = argv && argv[i] && (argv[i]->kind == AIR_VAL_PTR ||
+ val_holds_ptr(argv[i]));
+ /* A constant byte offset is rebuilt in the callee's own builder
+ * (rebind_here) and a run-time one on the pointer ITSELF is passed as a
+ * parameter (ptr_offset_to_pass). What is left -- a run-time offset on a
+ * pointer buried inside a struct -- is refused.
+ */
+ if (ptr && !ptr_offset_to_pass(argv[i]) && val_offset_is_computed(argv[i])) {
+ air_error(c, "intrinsic: a call passing a pointer at an offset computed at run "
+ "time as argument %u, whose value belongs to the caller's own "
+ "function", i);
return NULL;
}
+ /* The same reason, for the two pointers that are values: a chosen
+ * pointer's condition is a def of the CALLER's impl, and a slot's
+ * soundness rests on every use being in one block of one function.
+ */
+ if (argv && argv[i] && argv[i]->kind == AIR_VAL_PTR_SELECT) {
+ air_error(c, "intrinsic: a call passing a pointer chosen at run time as "
+ "argument %u", i);
+ return NULL;
+ }
+ /* A SLOT used to be refused outright here, as a latent cross-function
+ * def/use nothing asked for. Something does ask for it now -- the four
+ * `disparity_refinement_*` of CIPortraitBlurStitchableV3 pass a
+ * `coreimage::Sampler` they rebuilt in a local, and 126 modules of the
+ * image do the same thing -- so it goes through rebind_here, which makes
+ * the callee a slot of its own and copies the resolutions into it. What
+ * is still refused, and by rebind_slot_contents rather than here, is a
+ * slot holding a plain VALUE: that is a def of the caller's impl.
+ */
is_param[i] = !ptr;
specialised |= ptr;
}
@@ -1429,7 +4211,48 @@ translate_callee(struct air_ctx *c, const struct air_function *f,
const struct air_type *rt = f->fn_type->elem; /* FUNCTION: the return type */
bool has_ret = rt && rt->kind != AIR_TYPE_VOID;
- unsigned nparams = f->num_args + (has_ret ? 1 : 0);
+ /* A struct return takes one out-parameter per member rather than one for the
+ * whole value. Four of QuartzCore's vertex shaders are nothing but a wrapper
+ * around such a function -- `tail call fastcc`, then extractvalue of each
+ * field -- and before this they stopped at the call with "an aggregate where
+ * a number belongs", which was emit_return inside the callee asking for the
+ * returned struct as a register.
+ */
+ bool ret_struct = has_ret && rt->kind == AIR_TYPE_STRUCT;
+ unsigned nret = 0;
+ if (ret_struct) {
+ for (unsigned i = 0; i < rt->num_members; i++)
+ nret += ret_member_is_number(rt->members[i]);
+ } else if (has_ret) {
+ nret = 1;
+ }
+ /* one parameter per NUMBER in each argument, not one per argument: see
+ * type_leaves. Counted before fn exists, because fn->params is sized from it.
+ */
+ unsigned nargparams = 0;
+ for (unsigned i = 0; i < f->num_args; i++) {
+ if (!is_param[i]) {
+ if (ptr_offset_to_pass(argv ? argv[i] : NULL)) {
+ nargparams++;
+ continue;
+ }
+ /* a slot's value leaves, which are the caller's defs: slot_arg_leaves */
+ unsigned ns = 0;
+ if (!slot_arg_leaves(c, argv ? argv[i] : NULL, NULL, &ns)) {
+ air_error(c, "intrinsic: a call passing a local that holds a pointer as "
+ "argument %u, whose contents this cannot hand across a "
+ "function boundary", i);
+ return NULL;
+ }
+ nargparams += ns;
+ continue;
+ }
+ unsigned n = 0;
+ if (!type_leaves(c, f->args[i].type, NULL, &n))
+ return NULL;
+ nargparams += n;
+ }
+ unsigned nparams = nargparams + nret;
nir_function *fn = nir_function_create(c->nir, f->base.name ? f->base.name : "callee");
fn->num_params = nparams;
fn->params = rzalloc_array(c->nir, nir_parameter, MAX2(nparams, 1));
@@ -1437,11 +4260,19 @@ translate_callee(struct air_ctx *c, const struct air_function *f,
unsigned idx = 0;
const struct glsl_type *ret_glsl = NULL;
- if (has_ret) {
- unsigned bits = air_nir_bit_size(c, rt);
- unsigned comps = air_nir_components(rt);
- ret_glsl = glsl_vector_type(bits == 16 ? GLSL_TYPE_FLOAT16 :
- bits == 64 ? GLSL_TYPE_UINT64 : GLSL_TYPE_UINT, comps);
+ if (ret_struct) {
+ for (unsigned i = 0; i < rt->num_members; i++) {
+ if (!ret_member_is_number(rt->members[i]))
+ continue;
+ fn->params[idx++] = (nir_parameter){
+ .num_components = 1, .bit_size = 32,
+ .is_return = true,
+ .mode = nir_var_function_temp,
+ .type = param_glsl_type(c, rt->members[i]),
+ };
+ }
+ } else if (has_ret) {
+ ret_glsl = param_glsl_type(c, rt);
fn->params[idx++] = (nir_parameter){
.num_components = 1, .bit_size = 32,
.is_return = true,
@@ -1450,13 +4281,36 @@ translate_callee(struct air_ctx *c, const struct air_function *f,
};
}
for (unsigned i = 0; i < f->num_args; i++) {
- if (!is_param[i])
+ if (!is_param[i]) {
+ /* a bound pointer whose byte offset is a value: see ptr_offset_to_pass */
+ if (ptr_offset_to_pass(argv ? argv[i] : NULL)) {
+ fn->params[idx++] = (nir_parameter){ .num_components = 1, .bit_size = 32 };
+ continue;
+ }
+ /* a slot's value leaves. Each parameter is declared from its own def's
+ * shape and the source is passed unconverted, so the two match by
+ * construction rather than by a width this would have to choose.
+ */
+ nir_def *sv[AIR_MAX_SLOT_LEAVES];
+ unsigned ns = 0;
+ if (!slot_arg_leaves(c, argv ? argv[i] : NULL, sv, &ns))
+ return NULL; /* counted above, so this cannot fail; not assumed */
+ for (unsigned k = 0; k < ns; k++)
+ fn->params[idx++] = (nir_parameter){
+ .num_components = (uint8_t)sv[k]->num_components,
+ .bit_size = (uint8_t)sv[k]->bit_size,
+ };
continue;
- const struct air_type *at = f->args[i].type;
- fn->params[idx++] = (nir_parameter){
- .num_components = (uint8_t)air_nir_components(at),
- .bit_size = (uint8_t)air_nir_bit_size(c, at),
- };
+ }
+ const struct air_type *lv[AIR_MAX_ARG_LEAVES];
+ unsigned n = 0;
+ if (!type_leaves(c, f->args[i].type, lv, &n))
+ return NULL; /* counted above, so this cannot fail; not assumed */
+ for (unsigned k = 0; k < n; k++)
+ fn->params[idx++] = (nir_parameter){
+ .num_components = (uint8_t)air_nir_components(lv[k]),
+ .bit_size = (uint8_t)air_nir_bit_size(c, lv[k]),
+ };
}
fn->num_params = idx; /* the pointers took no parameter */
@@ -1472,6 +4326,12 @@ translate_callee(struct air_ctx *c, const struct air_function *f,
struct hash_table *sv = c->values, *sb = c->blocks,
*sp = c->phi_cursors, *sr = c->phi_regs;
nir_deref_instr *saved_ret = c->ret_deref;
+ nir_deref_instr **saved_ret_fields = c->ret_fields;
+ unsigned saved_num_ret_fields = c->num_ret_fields;
+ /* a constant of the caller is a def of the caller's impl, so the place
+ * constants are built belongs to the function being translated, like the
+ * values table above */
+ nir_cursor saved_consts = c->const_cursor;
c->fn = f;
c->values = _mesa_pointer_hash_table_create(c->mem_ctx);
@@ -1484,25 +4344,106 @@ translate_callee(struct air_ctx *c, const struct air_function *f,
* structured by default, so a callee crashed on its first branch -- which
* is what the corpus's reader count caught, 456 falling to 454. */
c->b.impl->structured = false;
+ c->const_cursor = nir_before_impl(c->b.impl);
c->ret_deref = NULL;
+ c->ret_fields = NULL;
+ c->num_ret_fields = 0;
/* this function's own globals, built in its own builder */
materialise_globals(c);
bool ok = true;
- if (has_ret) {
- /* The caller passes the deref's own def and the callee casts it back,
- * which is the convention nir_functions.c documents: load_param is
- * replaced by the call's source at inline time and nir_opt_deref removes
- * the cast afterwards.
- */
+ /* The caller passes each deref's own def and the callee casts it back, which
+ * is the convention nir_functions.c documents: load_param is replaced by the
+ * call's source at inline time and nir_opt_deref removes the cast afterwards.
+ */
+ if (ret_struct) {
+ c->ret_fields = rzalloc_array(c->mem_ctx, nir_deref_instr *,
+ MAX2(rt->num_members, 1));
+ c->num_ret_fields = rt->num_members;
+ unsigned p = 0;
+ for (unsigned i = 0; i < rt->num_members; i++) {
+ if (!ret_member_is_number(rt->members[i]))
+ continue;
+ c->ret_fields[i] =
+ nir_build_deref_cast(&c->b, nir_load_param(&c->b, p++),
+ nir_var_function_temp,
+ param_glsl_type(c, rt->members[i]), 0);
+ }
+ } else if (has_ret) {
c->ret_deref = nir_build_deref_cast(&c->b, nir_load_param(&c->b, 0),
nir_var_function_temp, ret_glsl, 0);
}
- unsigned pidx = has_ret ? 1 : 0;
+ unsigned pidx = nret;
for (unsigned i = 0; i < f->num_args && ok; i++) {
if (!is_param[i]) {
- air_set(c, &f->args[i], argv[i]); /* the caller's own resolution */
+ if (ptr_offset_to_pass(argv[i])) {
+ /* the binding from the caller, the offset from a parameter */
+ struct air_ptr pp = argv[i]->ptr;
+ pp.offset = nir_load_param(&c->b, pidx++);
+ air_set(c, &f->args[i], air_val_ptr(c, pp));
+ continue;
+ }
+ /* A SLOT: a new slot of the same type for this function, filled with
+ * the caller's resolutions rebuilt here and its values read out of the
+ * parameters declared above, in the order slot_leaves walked them.
+ */
+ if (argv[i] && argv[i]->kind == AIR_VAL_PTR &&
+ argv[i]->ptr.class == AIR_PTR_SLOT) {
+ nir_def *sv[AIR_MAX_SLOT_LEAVES];
+ unsigned ns = 0;
+ if (!slot_arg_leaves(c, argv[i], sv, &ns)) {
+ ok = false;
+ break;
+ }
+ for (unsigned k = 0; k < ns; k++)
+ sv[k] = nir_load_param(&c->b, pidx++);
+ unsigned used = 0;
+ const struct air_type *st = c->slots[argv[i]->ptr.binding].type;
+ struct air_val *inner =
+ rebind_slot_contents(c, c->slots[argv[i]->ptr.binding].val,
+ sv, ns, &used);
+ if (used != ns || (!inner && c->slots[argv[i]->ptr.binding].val)) {
+ air_error(c, "intrinsic: a call passing a local that holds a pointer as "
+ "argument %u, rebuilt with %u of its %u parameters",
+ i, used, ns);
+ ok = false;
+ break;
+ }
+ struct air_ptr sp = {
+ .class = AIR_PTR_SLOT,
+ .binding = new_slot(c, st, NULL),
+ .pointee = argv[i]->ptr.pointee ? argv[i]->ptr.pointee : st,
+ };
+ c->slots[sp.binding].val = inner;
+ air_set(c, &f->args[i], air_val_ptr(c, sp));
+ continue;
+ }
+ /* the caller's own resolution, with its offsets rebuilt here */
+ struct air_val *bound = rebind_here(c, argv[i], i);
+ if (!bound && argv[i]) {
+ ok = false;
+ break;
+ }
+ air_set(c, &f->args[i], bound);
+ continue;
+ }
+ const struct air_type *at = f->args[i].type;
+ if (at->kind == AIR_TYPE_STRUCT || at->kind == AIR_TYPE_ARRAY) {
+ nir_def *lv[AIR_MAX_ARG_LEAVES];
+ unsigned need = 0, taken = 0;
+ if (!type_leaves(c, at, NULL, &need)) {
+ ok = false;
+ break;
+ }
+ for (unsigned k = 0; k < need; k++)
+ lv[k] = nir_load_param(&c->b, pidx++);
+ struct air_val *v = val_from_leaves(c, at, lv, &taken);
+ if (!v) {
+ ok = false;
+ break;
+ }
+ air_set(c, &f->args[i], v);
continue;
}
air_set(c, &f->args[i], air_val_ssa(c, nir_load_param(&c->b, pidx++)));
@@ -1516,7 +4457,10 @@ translate_callee(struct air_ctx *c, const struct air_function *f,
c->blocks = sb;
c->phi_cursors = sp;
c->phi_regs = sr;
+ c->const_cursor = saved_consts;
c->ret_deref = saved_ret;
+ c->ret_fields = saved_ret_fields;
+ c->num_ret_fields = saved_num_ret_fields;
return ok ? fn : NULL;
}
@@ -1565,36 +4509,174 @@ air_call_in_module(struct air_ctx *c, const struct air_inst *in, const char *nam
const struct air_type *rt = f->fn_type->elem; /* FUNCTION: the return type */
bool has_ret = rt && rt->kind != AIR_TYPE_VOID;
+ bool ret_struct = has_ret && rt->kind == AIR_TYPE_STRUCT;
nir_variable *tmp = NULL;
+ /* One local per member for a struct return, in the member order
+ * translate_callee made its parameters from; the ones with no parameter stay
+ * NULL and the aggregate's element for them stays NULL too, which is the
+ * same thing an extractvalue of an undef field already reads.
+ */
+ nir_variable **fields = ret_struct ?
+ rzalloc_array(c->mem_ctx, nir_variable *, MAX2(rt->num_members, 1)) : NULL;
nir_call_instr *call = nir_call_instr_create(c->nir, fn);
unsigned idx = 0;
- if (has_ret) {
- unsigned bits = air_nir_bit_size(c, rt);
- unsigned comps = air_nir_components(rt);
- const struct glsl_type *t =
- glsl_vector_type(bits == 16 ? GLSL_TYPE_FLOAT16 :
- bits == 64 ? GLSL_TYPE_UINT64 : GLSL_TYPE_UINT, comps);
- tmp = nir_local_variable_create(c->b.impl, t, "callret");
+ if (ret_struct) {
+ for (unsigned i = 0; i < rt->num_members; i++) {
+ if (!ret_member_is_number(rt->members[i]))
+ continue;
+ fields[i] = nir_local_variable_create(c->b.impl,
+ param_glsl_type(c, rt->members[i]),
+ "callret");
+ call->params[idx++] =
+ nir_src_for_ssa(&nir_build_deref_var(&c->b, fields[i])->def);
+ }
+ } else if (has_ret) {
+ tmp = nir_local_variable_create(c->b.impl, param_glsl_type(c, rt), "callret");
call->params[idx++] = nir_src_for_ssa(&nir_build_deref_var(&c->b, tmp)->def);
}
- /* operand 0 is the callee; the arguments follow it */
+ /* operand 0 is the callee; the arguments follow it. One source per NUMBER
+ * inside each argument, in the order translate_callee made its parameters
+ * (type_leaves): a struct or an array passed by value is not one NIR value.
+ */
for (unsigned i = 0; i < f->num_args; i++) {
- if (!is_param[i])
+ if (!is_param[i]) {
+ /* a bound pointer's byte offset, if it is a value: ptr_offset_to_pass.
+ * Narrowed to the 32 bits air_ptr documents an offset as and the
+ * parameter declares, because a gep of an i64 index can leave a wider
+ * def behind and a call source must match its parameter exactly.
+ */
+ nir_def *off = ptr_offset_to_pass(argv[i]);
+ if (off) {
+ call->params[idx++] = nir_src_for_ssa(nir_u2u32(&c->b, off));
+ continue;
+ }
+ /* a slot's value leaves, in the order translate_callee declared them
+ * (slot_leaves), and unconverted because each parameter was declared
+ * from the def that fills it
+ */
+ nir_def *sv[AIR_MAX_SLOT_LEAVES];
+ unsigned ns = 0;
+ if (!slot_arg_leaves(c, argv[i], sv, &ns))
+ return air_error(c, "intrinsic: a call passing a local that holds a pointer "
+ "as argument %u, whose contents this cannot hand across "
+ "a function boundary", i);
+ for (unsigned k = 0; k < ns; k++)
+ call->params[idx++] = nir_src_for_ssa(sv[k]);
continue;
- nir_def *a = air_get_ssa(c, in->operands[i + 1]);
- if (!a)
- return air_error(c, "intrinsic: argument %u of %s is not a value this can pass",
- i, name);
- call->params[idx++] = nir_src_for_ssa(a);
+ }
+ nir_def *lv[AIR_MAX_ARG_LEAVES];
+ unsigned n = 0;
+ if (!val_leaves(c, f->args[i].type, argv[i], lv, &n))
+ return false; /* val_leaves has said what the argument was */
+ for (unsigned k = 0; k < n; k++)
+ call->params[idx++] = nir_src_for_ssa(lv[k]);
}
+ /* The two sides derive the count from the same type with the same function,
+ * so they agree -- and the agreement is checked rather than trusted, because
+ * a call left one source short is a null nir_src that nir_instr_insert
+ * dereferences while adding uses: a segfault, not a message.
+ */
+ if (idx != call->num_params)
+ return air_error(c, "intrinsic: a call of %s built with %u sources for %u "
+ "parameters", name, idx, call->num_params);
nir_builder_instr_insert(&c->b, &call->instr);
- if (has_ret)
+ if (ret_struct) {
+ struct air_val *agg = air_val_agg(c, rt->num_members);
+ for (unsigned i = 0; i < rt->num_members; i++)
+ if (fields[i])
+ agg->elems[i] = air_val_ssa(c, nir_load_var(&c->b, fields[i]));
+ air_set(c, &in->base, agg);
+ } else if (has_ret) {
air_set(c, &in->base, air_val_ssa(c, nir_load_var(&c->b, tmp)));
+ }
return true;
}
+/* The air.visible_input node for argument i, or NULL. The nodes carry a
+ * parameter number as their first operand rather than being in order, so they
+ * are searched by it.
+ */
+static const struct air_md *
+arg_md(const struct air_md *inputs, unsigned i)
+{
+ for (unsigned k = 0; k < (inputs ? inputs->num_ops : 0); k++) {
+ const struct air_md *n = inputs->ops[k];
+ uint64_t which;
+ if (n && md_int(n, 0, &which) && which == i)
+ return n;
+ }
+ return NULL;
+}
+
+/* A texture, a sampler or a constant buffer: an argument that is BOUND rather
+ * than passed, at the binding the caller names.
+ *
+ * Eleven of the corpus's library functions are CoreImage's sampling helpers,
+ * and they take what CoreImage hands a stitched function: a texture, a sampler
+ * and a transform. Apple passes the first two as one-field structs named
+ * metal::texture2d and metal::sampler -- the field being exactly the
+ * address-space-1 or -2 pointer an entry point's texture or sampler argument
+ * is -- and the transform as a bare constant-space pointer. A texture is not a
+ * value in this frontend (section 7.4), so each of the three resolves to a
+ * binding and the runtime binds a texture, a sampler and a constant buffer
+ * there.
+ *
+ * Keyed on the struct's name, which is Apple's and is in these modules, and
+ * not on "a struct with one pointer in it": the second would quietly accept
+ * anything shaped alike.
+ *
+ * *kind is what the argument turned out to be, so a caller that was told by a
+ * graph which resource an argument is can check that against the type instead
+ * of trusting either one alone. Returns NULL without an error when the type is
+ * none of the three -- then the argument is a value and the caller supplies it.
+ */
+static struct air_val *
+bind_resource_argument(struct air_ctx *c, const struct air_type *t, unsigned arg,
+ const struct air_md *md, unsigned binding,
+ enum air_resource_kind *kind)
+{
+ const char *sname = t->kind == AIR_TYPE_STRUCT ? t->name : NULL;
+ bool is_tex = sname && !strncmp(sname, "struct.metal::texture", 21);
+ bool is_samp = sname && !strcmp(sname, "struct.metal::sampler");
+
+ if (is_tex || is_samp) {
+ if (t->num_members != 1) {
+ air_error(c, "library: %s has %u members, not the one pointer this expects",
+ sname, t->num_members);
+ return NULL;
+ }
+ struct air_ptr p = {
+ .class = is_tex ? AIR_PTR_TEXTURE : AIR_PTR_SAMPLER,
+ .binding = binding,
+ };
+ *kind = is_tex ? AIR_RESOURCE_TEXTURE : AIR_RESOURCE_SAMPLER;
+ add_resource(c, *kind, binding, 0, true, false, arg, md);
+ /* the value is the struct, so the pointer goes inside an aggregate of
+ * one: the body reaches it with extractvalue
+ */
+ struct air_val *agg = air_val_agg(c, 1);
+ agg->elems[0] = air_val_ptr(c, p);
+ return agg;
+ }
+
+ if (t->kind == AIR_TYPE_POINTER && t->addrspace == 2) {
+ /* the transform, as a constant buffer the runtime fills. With the
+ * argument's own metadata, so the resource carries Apple's name for it --
+ * "float4x4" for the transform these functions take, which is what lets a
+ * harness put something meaningful there rather than the byte pattern.
+ */
+ struct air_ptr p = { .class = AIR_PTR_UBO, .binding = binding, .pointee = t->elem };
+ *kind = AIR_RESOURCE_CONSTANT_BUFFER;
+ add_resource(c, *kind, binding, 256, true, false, arg, md);
+ return air_val_ptr(c, p);
+ }
+
+ *kind = AIR_RESOURCE_DEVICE_BUFFER; /* not a bound resource: a value */
+ return NULL;
+}
+
/* An air.visible library function, inside an entry point we generate for it.
*
* A library function is not a shader. Metal links one into a caller at
@@ -1619,7 +4701,8 @@ air_call_in_module(struct air_ctx *c, const struct air_inst *in, const char *nam
* points.
*/
static bool
-wrap_library_function(struct air_ctx *c, const struct air_md *out_md)
+wrap_library_function(struct air_ctx *c, const struct air_md *out_md,
+ const struct air_md *in_md)
{
const struct air_function *f = c->fn;
@@ -1637,16 +4720,54 @@ wrap_library_function(struct air_ctx *c, const struct air_md *out_md)
MAX2(f->num_args, 1));
bool *is_param = rzalloc_array(c->mem_ctx, bool, MAX2(f->num_args, 1));
+ /* Eleven of the corpus's library functions are CoreImage's sampling helpers,
+ * and they take what CoreImage hands a stitched function: a texture, a
+ * sampler and a transform. Apple passes the first two as one-field structs
+ * named metal::texture2d and metal::sampler -- the field being exactly the
+ * address-space-1 or -2 pointer an entry point's texture or sampler
+ * argument is -- and the transform as a bare constant-space pointer. Those
+ * three shapes are bound rather than loaded: a texture is not a value in
+ * this frontend (section 7.4), so the wrapper resolves each to a binding
+ * and the harness binds its own texture, sampler and constant buffer there.
+ *
+ * Keyed on the struct's name, which is Apple's and is in these modules, and
+ * not on "a struct with one pointer in it": the second would quietly accept
+ * anything shaped alike.
+ */
+ unsigned next_tex = 0, next_samp = 0, next_ubo = 0;
for (unsigned i = 0; i < f->num_args; i++) {
const struct air_type *t = f->args[i].type;
+ const char *sname = t->kind == AIR_TYPE_STRUCT ? t->name : NULL;
+ bool is_tex = sname && !strncmp(sname, "struct.metal::texture", 21);
+ bool is_samp = sname && !strcmp(sname, "struct.metal::sampler");
+ bool is_ubo = t->kind == AIR_TYPE_POINTER && t->addrspace == 2;
+
+ if (is_tex || is_samp || is_ubo) {
+ /* The wrapper numbers these itself, in the order the arguments come:
+ * nothing tells it otherwise, where a stitched graph is given each
+ * one's bind index by the runtime (stitch_graph).
+ */
+ enum air_resource_kind got;
+ argv[i] = bind_resource_argument(c, t, i, arg_md(in_md, i),
+ is_tex ? next_tex : is_samp ? next_samp
+ : next_ubo, &got);
+ if (!argv[i])
+ return false; /* bind_resource_argument has said why */
+ next_tex += is_tex;
+ next_samp += is_samp;
+ next_ubo += is_ubo;
+ is_param[i] = false;
+ continue;
+ }
+
const struct air_type *e = t->kind == AIR_TYPE_VECTOR ? t->elem : t;
bool numeric = e && (e->kind == AIR_TYPE_FLOAT || e->kind == AIR_TYPE_HALF ||
e->kind == AIR_TYPE_INT);
if (!numeric || (t->kind != AIR_TYPE_VECTOR && t->kind != AIR_TYPE_FLOAT &&
t->kind != AIR_TYPE_HALF && t->kind != AIR_TYPE_INT))
- return air_error(c, "library: argument %u is neither a scalar nor a vector, "
- "and this wrapper passes each argument in a 16-byte slot "
- "of one buffer", i);
+ return air_error(c, "library: argument %u is none of a scalar, a vector, a "
+ "metal::texture, a metal::sampler or a constant pointer, "
+ "which is all this wrapper can supply", i);
if (air_nir_components(t) * air_nir_bit_size(c, t) > 16 * 8)
return air_error(c, "library: argument %u does not fit the wrapper's 16-byte "
"slot", i);
@@ -1682,21 +4803,527 @@ wrap_library_function(struct air_ctx *c, const struct air_md *out_md)
nir_call_instr *call = nir_call_instr_create(c->nir, fn);
unsigned idx = 0;
call->params[idx++] = nir_src_for_ssa(&nir_build_deref_var(&c->b, tmp)->def);
- for (unsigned i = 0; i < f->num_args; i++)
- call->params[idx++] = nir_src_for_ssa(argv[i]->def);
+ for (unsigned i = 0; i < f->num_args; i++) {
+ /* only the arguments translate_callee made parameters: a texture, a
+ * sampler or a constant pointer was bound into the callee instead, and
+ * has no SSA def to pass. Passing one anyway inserts a call whose source
+ * has no value, which NIR dereferences while adding uses -- a segfault
+ * inside nir_instr_insert rather than a message.
+ */
+ if (!is_param[i]) {
+ /* no argument this wrapper supplies has a run-time offset -- they are
+ * bindings and 16-byte slots -- but the parameter would exist if one
+ * did, so the source is pushed here rather than left to the count
+ * check below to catch
+ */
+ nir_def *off = ptr_offset_to_pass(argv[i]);
+ if (off) {
+ call->params[idx++] = nir_src_for_ssa(nir_u2u32(&c->b, off));
+ continue;
+ }
+ /* and no argument this wrapper supplies is a slot either, for the same
+ * reason: a slot is something a module's own code allocates
+ */
+ nir_def *sv[AIR_MAX_SLOT_LEAVES];
+ unsigned ns = 0;
+ if (!slot_arg_leaves(c, argv[i], sv, &ns))
+ return air_error(c, "library: argument %u is a local holding a pointer "
+ "whose contents cannot be passed", i);
+ for (unsigned k = 0; k < ns; k++)
+ call->params[idx++] = nir_src_for_ssa(sv[k]);
+ continue;
+ }
+ nir_def *a = argv[i] ? argv[i]->def : NULL;
+ if (!a)
+ return air_error(c, "library: argument %u has no value to pass", i);
+ call->params[idx++] = nir_src_for_ssa(a);
+ }
+ /* Every argument here is a scalar or a vector -- the loop above refuses
+ * anything else -- so each takes exactly one parameter and this holds. It is
+ * checked because translate_callee decides the parameters and this decides
+ * the sources, and the two would otherwise agree only by inspection.
+ */
+ if (idx != call->num_params)
+ return air_error(c, "library: a call built with %u sources for %u parameters",
+ idx, call->num_params);
nir_builder_instr_insert(&c->b, &call->instr);
struct air_ptr out = { .class = AIR_PTR_SSBO, .binding = 1, .pointee = rt };
- if (!air_store_ptr(c, out, nir_load_var(&c->b, tmp), 16))
+ if (!air_store_ptr(c, out, nir_load_var(&c->b, tmp), 16))
+ return false;
+
+ /* the generated block ends the way a translated one does: this impl is
+ * unstructured until nir_lower_goto_ifs runs over it
+ */
+ nir_goto(&c->b, c->b.impl->end_block);
+ return true;
+}
+
+/* -- function stitching ----------------------------------------------------
+ *
+ * A graph of air.visible functions compiled into one entry point, which is
+ * what Metal does with these functions and what CoreImage actually uses
+ * (newLibraryWithStitchedDescriptor:, docs/air.md section 5, "The stitching
+ * ABI, before any of it is built"). The wrapper above runs one body; this runs
+ * a DAG of them, each node's arguments coming from another node's result, from
+ * an argument of the stitched function, or from a bound resource.
+ *
+ * A node's arguments are the callee's OWN arguments, one for one, and not the
+ * CoreImage-level ones: `_pf_sobelH` takes a `coreimage::Sampler` in Apple's
+ * source and five arguments in the module, and the flattening between them is
+ * CoreImage's -- its StitchedFunctionDescriptor::MetalArgumentInfo has
+ * `ci_sampler_builder_args_start` and `..._end`, a RANGE of Metal arguments for
+ * one of its own. So the graph this takes is the Metal-level one, which is the
+ * level MTLFunctionStitchingFunctionNode.arguments is at.
+ */
+static const struct air_function *
+visible_function_of(const struct air_module *m, const struct air_md **entry)
+{
+ if (module_stage(m, entry) != MESA_SHADER_NONE)
+ return NULL;
+ const struct air_md *e = *entry;
+ if (!e->num_ops || !e->ops[0] || e->ops[0]->kind != AIR_MD_VALUE ||
+ e->ops[0]->value->kind != AIR_VALUE_FUNCTION)
+ return NULL;
+ return (const struct air_function *)e->ops[0]->value;
+}
+
+/* One function a graph's nodes can name: read once, however many nodes call
+ * it, so that two nodes calling the same function meet the same
+ * air_function and translate_callee's cache applies to the second.
+ */
+struct stitch_fn {
+ const struct air_function *f;
+ const struct air_md *entry; /* its air.visible node */
+ struct air_module *m;
+ struct air_global_plan *plan; /* its module's globals, planned */
+};
+
+static bool
+stitch_prepare(struct air_ctx *c, struct stitch_fn *sf, const char *name,
+ struct stitch_fn *done, unsigned num_done)
+{
+ /* A function two nodes call is read once: the second node then meets the
+ * same air_function, so translate_callee's cache gives it the same
+ * nir_function and the body is translated once however often it is called.
+ */
+ for (unsigned i = 0; i < num_done; i++) {
+ if (done[i].f && done[i].f->base.name && !strcmp(done[i].f->base.name, name)) {
+ *sf = done[i];
+ return true;
+ }
+ }
+
+ const struct air_stitch_function *src = NULL;
+ for (unsigned i = 0; i < c->options->num_stitch_functions; i++) {
+ const struct air_stitch_function *k = &c->options->stitch_functions[i];
+ if (k->name && name && !strcmp(k->name, name)) {
+ src = k;
+ break;
+ }
+ }
+ if (!src)
+ return air_error(c, "stitch: the graph names %s, which is not one of the %u "
+ "functions given to stitch from", name ? name : "(nothing)",
+ c->options->num_stitch_functions);
+
+ const char *why = NULL;
+ sf->m = air_module_read(c->mem_ctx, src->bitcode, src->size, &why);
+ if (!sf->m)
+ return air_error(c, "stitch: %s: reader: %s", name, why ? why : "?");
+ sf->f = visible_function_of(sf->m, &sf->entry);
+ if (!sf->f)
+ return air_error(c, "stitch: %s is not an air.visible function of its module, so "
+ "it is not a function a graph can call", name);
+ if (sf->f->base.name && strcmp(sf->f->base.name, name))
+ return air_error(c, "stitch: the module given for %s holds %s instead", name,
+ sf->f->base.name);
+
+ /* its module's globals, planned in its own module: plan_globals indexes by
+ * the module's own global numbering, so the plan belongs to the module and
+ * not to the shader, and translate_callee materialises from whichever plan
+ * is current when it runs.
+ */
+ const struct air_module *saved_m = c->m;
+ struct air_global_plan *saved_plan = c->global_plan;
+ c->m = sf->m;
+ c->global_plan = NULL;
+ plan_globals(c);
+ sf->plan = c->global_plan;
+ if (!c->error[0])
+ run_static_initializers(c); /* in the entry point, before any call */
+ c->m = saved_m;
+ c->global_plan = saved_plan;
+ return !c->error[0];
+}
+
+/* The air.visible_input node list of a function, which arg_md searches. */
+static const struct air_md *
+stitch_inputs_md(const struct stitch_fn *sf)
+{
+ return sf->entry->num_ops > 2 ? sf->entry->ops[2] : NULL;
+}
+
+/* Its result's metadata: air.visible_output, which carries Apple's own name
+ * for the type and is what the result buffer is labelled with.
+ */
+static const struct air_md *
+stitch_output_md(const struct stitch_fn *sf)
+{
+ const struct air_md *rn = sf->entry->num_ops > 1 ? sf->entry->ops[1] : NULL;
+ return rn && rn->num_ops ? rn->ops[0] : NULL;
+}
+
+/* The result type of a node, which is what an argument fed from it receives. */
+static const struct air_type *
+stitch_result_type(const struct stitch_fn *sf)
+{
+ return sf->f->fn_type->elem;
+}
+
+static bool
+stitch_graph(struct air_ctx *c)
+{
+ const struct air_stitch_graph *g = c->options->stitch;
+ if (!g->num_nodes)
+ return air_error(c, "stitch: a graph with no nodes");
+ if (g->output_node >= g->num_nodes)
+ return air_error(c, "stitch: the output node is %u of %u nodes", g->output_node,
+ g->num_nodes);
+
+ struct stitch_fn *fns = rzalloc_array(c->mem_ctx, struct stitch_fn, g->num_nodes);
+ nir_def **value = rzalloc_array(c->mem_ctx, nir_def *, g->num_nodes);
+
+ /* The stitched function's own arguments, and the resources its nodes bind.
+ * The argument slots are the library wrapper's -- argument i at byte 16*i of
+ * device buffer 0, the result in device buffer 1 -- on purpose: a one-node
+ * graph then has to produce exactly the words the wrapper produces for the
+ * same function, which is a comparison that already has a reference.
+ */
+ unsigned num_inputs = 0;
+ for (unsigned n = 0; n < g->num_nodes; n++)
+ for (unsigned a = 0; a < g->nodes[n].num_args; a++)
+ if (g->nodes[n].args[a].kind == AIR_STITCH_INPUT)
+ num_inputs = MAX2(num_inputs, g->nodes[n].args[a].index + 1);
+
+ add_resource(c, AIR_RESOURCE_DEVICE_BUFFER, 0, 16 * MAX2(num_inputs, 1),
+ true, false, ~0u, NULL);
+
+ for (unsigned n = 0; n < g->num_nodes; n++) {
+ const struct air_stitch_node *node = &g->nodes[n];
+ if (!stitch_prepare(c, &fns[n], node->function, fns, n))
+ return false;
+ const struct air_function *f = fns[n].f;
+
+ if (node->num_args != f->num_args)
+ return air_error(c, "stitch: node %u gives %s %u arguments and it takes %u",
+ n, node->function, node->num_args, f->num_args);
+
+ struct air_val **argv = rzalloc_array(c->mem_ctx, struct air_val *,
+ MAX2(f->num_args, 1));
+ bool *is_param = rzalloc_array(c->mem_ctx, bool, MAX2(f->num_args, 1));
+
+ for (unsigned a = 0; a < f->num_args; a++) {
+ const struct air_type *t = f->args[a].type;
+ struct air_stitch_arg src = node->args[a];
+ enum air_resource_kind want;
+
+ switch (src.kind) {
+ case AIR_STITCH_TEXTURE:
+ case AIR_STITCH_SAMPLER:
+ case AIR_STITCH_BUFFER: {
+ want = src.kind == AIR_STITCH_TEXTURE ? AIR_RESOURCE_TEXTURE :
+ src.kind == AIR_STITCH_SAMPLER ? AIR_RESOURCE_SAMPLER :
+ AIR_RESOURCE_CONSTANT_BUFFER;
+ enum air_resource_kind got;
+ argv[a] = bind_resource_argument(c, t, a,
+ arg_md(stitch_inputs_md(&fns[n]), a),
+ src.index, &got);
+ if (!argv[a])
+ return c->error[0] ? false :
+ air_error(c, "stitch: node %u binds argument %u of %s as a resource "
+ "and its type is not a texture, a sampler or a "
+ "constant pointer", n, a, node->function);
+ /* The graph says which resource it is and the module's own type
+ * says which it can be. Both are checked against each other
+ * because a bind index that reached the wrong KIND of resource
+ * would still compile and still read bytes.
+ */
+ if (got != want)
+ return air_error(c, "stitch: node %u binds argument %u of %s as a %s and "
+ "the module declares it a %s", n, a, node->function,
+ want == AIR_RESOURCE_TEXTURE ? "texture" :
+ want == AIR_RESOURCE_SAMPLER ? "sampler" : "buffer",
+ got == AIR_RESOURCE_TEXTURE ? "texture" :
+ got == AIR_RESOURCE_SAMPLER ? "sampler" : "buffer");
+ is_param[a] = false;
+ continue;
+ }
+
+ case AIR_STITCH_NODE: {
+ if (src.index >= n)
+ return air_error(c, "stitch: node %u takes the result of node %u, which "
+ "is not an earlier node: a graph is given in "
+ "dependency order", n, src.index);
+ const struct air_type *rt = stitch_result_type(&fns[src.index]);
+ if (air_nir_components(rt) != air_nir_components(t) ||
+ air_nir_bit_size(c, rt) != air_nir_bit_size(c, t))
+ return air_error(c, "stitch: node %u takes node %u's result into an "
+ "argument of a different shape (%ux%u into %ux%u)",
+ n, src.index, air_nir_components(rt),
+ air_nir_bit_size(c, rt), air_nir_components(t),
+ air_nir_bit_size(c, t));
+ argv[a] = air_val_ssa(c, value[src.index]);
+ is_param[a] = true;
+ continue;
+ }
+
+ case AIR_STITCH_INPUT: {
+ const struct air_type *e = t->kind == AIR_TYPE_VECTOR ? t->elem : t;
+ bool numeric = e && (e->kind == AIR_TYPE_FLOAT || e->kind == AIR_TYPE_HALF ||
+ e->kind == AIR_TYPE_INT);
+ if (!numeric || (t->kind != AIR_TYPE_VECTOR && t->kind != AIR_TYPE_FLOAT &&
+ t->kind != AIR_TYPE_HALF && t->kind != AIR_TYPE_INT))
+ return air_error(c, "stitch: node %u takes the stitched function's "
+ "argument %u into argument %u of %s, which is "
+ "neither a scalar nor a vector", n, src.index, a,
+ node->function);
+ if (air_nir_components(t) * air_nir_bit_size(c, t) > 16 * 8)
+ return air_error(c, "stitch: argument %u does not fit the 16-byte slot "
+ "the stitched function's arguments are read from",
+ src.index);
+ struct air_ptr p = { .class = AIR_PTR_SSBO, .binding = 0, .pointee = t };
+ nir_def *v = air_load_ptr(c, air_ptr_add(c, p, nir_imm_int(&c->b,
+ 16 * src.index)),
+ t, 16);
+ if (!v)
+ return false;
+ argv[a] = air_val_ssa(c, v);
+ is_param[a] = true;
+ continue;
+ }
+
+ case AIR_STITCH_GRID_POSITION:
+ /* In the API and not implemented, which is not the same as
+ * forgotten: no case here dispatches more than one thread, so a
+ * grid position would be translated and never measured, and an
+ * untested path is worse than a refusal.
+ */
+ return air_error(c, "stitch: node %u takes thread_position_in_grid, which "
+ "is not implemented", n);
+ }
+ return air_error(c, "stitch: node %u argument %u has an unknown source", n, a);
+ }
+
+ /* its module is current while its body is translated: the globals the
+ * body reads are at the offsets ITS plan decided
+ */
+ const struct air_module *saved_m = c->m;
+ struct air_global_plan *saved_plan = c->global_plan;
+ c->m = fns[n].m;
+ c->global_plan = fns[n].plan;
+ nir_function *fn = translate_callee(c, f, argv, is_param);
+ c->m = saved_m;
+ c->global_plan = saved_plan;
+ if (!fn)
+ return false; /* translate_callee has said why */
+
+ const struct air_type *rt = stitch_result_type(&fns[n]);
+ if (!rt || rt->kind == AIR_TYPE_VOID)
+ return air_error(c, "stitch: node %u calls %s, which returns nothing, so no "
+ "other node can take its result", n, node->function);
+ const struct air_type *re = rt->kind == AIR_TYPE_VECTOR ? rt->elem : rt;
+ if (!re || (re->kind != AIR_TYPE_FLOAT && re->kind != AIR_TYPE_HALF &&
+ re->kind != AIR_TYPE_INT))
+ return air_error(c, "stitch: node %u calls %s, whose result is neither a "
+ "scalar nor a vector", n, node->function);
+
+ unsigned bits = air_nir_bit_size(c, rt);
+ unsigned comps = air_nir_components(rt);
+ const struct glsl_type *gt =
+ glsl_vector_type(bits == 16 ? GLSL_TYPE_FLOAT16 :
+ bits == 64 ? GLSL_TYPE_UINT64 : GLSL_TYPE_UINT, comps);
+ nir_variable *tmp = nir_local_variable_create(c->b.impl, gt, "node");
+
+ nir_call_instr *call = nir_call_instr_create(c->nir, fn);
+ unsigned idx = 0;
+ call->params[idx++] = nir_src_for_ssa(&nir_build_deref_var(&c->b, tmp)->def);
+ for (unsigned a = 0; a < f->num_args; a++) {
+ if (!is_param[a]) {
+ /* as in the wrapper above: a bound pointer takes no parameter
+ * unless its byte offset is a value (ptr_offset_to_pass)
+ */
+ nir_def *off = ptr_offset_to_pass(argv[a]);
+ if (off) {
+ call->params[idx++] = nir_src_for_ssa(nir_u2u32(&c->b, off));
+ continue;
+ }
+ nir_def *sv[AIR_MAX_SLOT_LEAVES];
+ unsigned ns = 0;
+ if (!slot_arg_leaves(c, argv[a], sv, &ns))
+ return air_error(c, "stitch: node %u argument %u is a local holding a "
+ "pointer whose contents cannot be passed", n, a);
+ for (unsigned k = 0; k < ns; k++)
+ call->params[idx++] = nir_src_for_ssa(sv[k]);
+ continue;
+ }
+ nir_def *v = argv[a] ? argv[a]->def : NULL;
+ if (!v)
+ return air_error(c, "stitch: node %u argument %u has no value to pass", n, a);
+ call->params[idx++] = nir_src_for_ssa(v);
+ }
+ /* as in the wrapper above: one parameter per argument holds here because
+ * every argument is a scalar or a vector, and is checked anyway
+ */
+ if (idx != call->num_params)
+ return air_error(c, "stitch: node %u built with %u sources for %u parameters",
+ n, idx, call->num_params);
+ nir_builder_instr_insert(&c->b, &call->instr);
+ value[n] = nir_load_var(&c->b, tmp);
+ }
+
+ /* the result buffer, declared once the output node's own metadata is in
+ * hand: it carries Apple's air.arg_type_name for the result, which is how a
+ * harness knows what the bytes coming back are
+ */
+ const struct air_type *rt = stitch_result_type(&fns[g->output_node]);
+ add_resource(c, AIR_RESOURCE_DEVICE_BUFFER, 1, 16, false, true, ~0u,
+ stitch_output_md(&fns[g->output_node]));
+ struct air_ptr out = { .class = AIR_PTR_SSBO, .binding = 1, .pointee = rt };
+ if (!air_store_ptr(c, out, value[g->output_node], 16))
return false;
- /* the generated block ends the way a translated one does: this impl is
- * unstructured until nir_lower_goto_ifs runs over it
- */
+ c->info->entry_point = g->name ? g->name : "stitched";
+ c->nir->info.name = ralloc_strdup(c->nir, c->info->entry_point);
nir_goto(&c->b, c->b.impl->end_block);
return true;
}
+/* A phi of POINTERS, resolved rather than carried in a register.
+ *
+ * A pointer here is *(which resource, which byte offset)* decided while
+ * translating, so it has no bits to put in a register and a phi of two of them
+ * is not a value NIR can hold. It need not be: LLVM duplicates a
+ * `getelementptr` onto two paths and phis the copies back, so both operands
+ * name the same member -- 207 of the pointer phis in Resident Evil 4's library
+ * are exactly that, and not one names two different members.
+ *
+ * Refused when they name different resources. This COULD go through the
+ * AIR_VAL_PTR_SELECT machinery that already loads both sides and selects the
+ * values, but nothing in reach needs it, a texture or a sampler is not a value
+ * that path can load, and an untested path is worse than a refusal.
+ *
+ * ONE RESOURCE AT DIFFERENT OFFSETS is not that, and is carried. Its offset is
+ * a 32-bit number like any phi's value, so it gets a register: each edge
+ * stores the offset its operand has, and the phi is the resource at the
+ * register's value. That is what LLVM's duplicated GEP needs when the copies
+ * are two defs (mediaserverd's kernelComputeStage1: the same member of the
+ * same buffer, computed in each of two blocks, which same_ptr cannot prove
+ * equal), and what a pointer stepped round a loop needs -- an operand over a
+ * back edge is then stored when the edge is, and checked to name the same
+ * resource there (the phi stores below), not assumed to.
+ *
+ * `live` is why this is sound rather than optimistic: the blocks are
+ * walked in reverse post order, so an operand coming in over a BACK edge has
+ * not been translated yet. Skipping an operand merely because it is missing
+ * would accept a loop-carried pointer by ignoring the edge that changes it, so
+ * a missing operand from a live block is a refusal and only an edge the entry
+ * cannot take at all is passed over.
+ *
+ * `live` is narrower than CFG reachability and that is the point: a block
+ * reached only through a branch whose condition is decided while translating
+ * is not entered, so a phi between two different resources has ONE operand
+ * and resolves. That is how a shader specialised on its function constants
+ * gets through here (translate_body, and docs/air.md section 3), and nothing
+ * about this function had to change for it -- only what it is told is live.
+ */
+/* An `undef` or `poison` operand: the phi may be anything over that edge, so
+ * it is whatever its other operands name. LLVM leaves one when it splits a
+ * pointer into a phi per path and only one path uses each (MPS's
+ * cnnConvArray_*: two threadgroup pointers, each undef over the other's edge),
+ * and counting it as a resource of its own refused a phi that names one.
+ */
+static bool
+undef_operand(const struct air_value *v)
+{
+ if (!v || v->kind != AIR_VALUE_CONSTANT)
+ return false;
+ const enum air_const_kind k = ((const struct air_constant *)v)->ckind;
+ return k == AIR_CONST_UNDEF || k == AIR_CONST_POISON;
+}
+
+static bool
+resolve_pointer_phi(struct air_ctx *c, const struct air_inst *in, const bool *live,
+ const int *taken_succ, unsigned my_block)
+{
+ struct air_val *first = NULL;
+ bool carried = false; /* an operand over a loop edge, or offsets same_ptr cannot equate */
+ for (unsigned k = 0; k < in->num_operands; k++) {
+ if (!in->targets[k] || !live[in->targets[k]->index])
+ continue;
+ /* AND THE EDGE, not just the block. A block whose branch folded still
+ * EXECUTES -- it is live -- but one of its two edges does not exist, and
+ * a phi on the far side of the one that does not must not count the
+ * operand arriving over it. Getting this wrong is not academic: it is
+ * exactly what the fifteen texture phis of fixed_frag_lph_cph look like,
+ * where `br i1 %144, label %null, label %join` folds to the null side
+ * and `%join`'s phi still saw an operand from the branching block. The
+ * refusal then said "a phi between pointers that name different places"
+ * about a shader that names one.
+ */
+ int taken = taken_succ[in->targets[k]->index];
+ if (taken >= 0 && taken != (int)my_block)
+ continue;
+ if (undef_operand(in->operands[k]))
+ continue;
+ struct hash_entry *e = _mesa_hash_table_search(c->values, in->operands[k]);
+ if (!e) {
+ carried = true; /* over a loop edge: stored, and checked, when the edge is */
+ continue;
+ }
+ struct air_val *v = e->data;
+ if (!v || v->kind != AIR_VAL_PTR)
+ return air_error(c, "instruction: a phi of pointers one of whose operands is "
+ "%s", v && v->kind == AIR_VAL_PTR_SELECT
+ ? "itself a pointer chosen at run time"
+ : "not a resolved pointer");
+ if (!first)
+ first = v;
+ else if (first->ptr.class != v->ptr.class || first->ptr.binding != v->ptr.binding ||
+ first->ptr.is_image != v->ptr.is_image)
+ return air_error(c, "instruction: a phi between pointers that name different "
+ "places, which would have to be chosen at run time");
+ else if (!same_ptr(&first->ptr, &v->ptr))
+ carried = true;
+ }
+ if (!first && carried)
+ return air_error(c, "instruction: a phi of pointers whose operand comes in over "
+ "a loop edge, so which resource it is depends on the "
+ "iteration"); /* every live operand over a back edge: nothing to name the resource by */
+ if (!first)
+ return air_error(c, "instruction: a phi of pointers with no reachable operand");
+ if (!carried) {
+ air_set(c, &in->base, first);
+ return true;
+ }
+ /* memory alone has an offset a register can carry: a texture, a sampler or a
+ * slot is a handle, and an argument buffer's offset is resolved to a member
+ * while translating, which a value known only at run time cannot be
+ */
+ const enum air_ptr_class cl = first->ptr.class;
+ if (first->ptr.is_image || (cl != AIR_PTR_UBO && cl != AIR_PTR_SSBO && cl != AIR_PTR_SHARED &&
+ cl != AIR_PTR_SCRATCH && cl != AIR_PTR_CONSTANT_DATA))
+ return air_error(c, "instruction: a phi of a pointer that is not into memory (class %u) "
+ "and changes round a loop or is computed twice", (unsigned)cl);
+ nir_builder entry = nir_builder_at(nir_before_impl(c->b.impl));
+ nir_def *reg = nir_decl_reg(&entry, 1, 32, 0);
+ _mesa_hash_table_insert(c->phi_regs, in, reg);
+ struct air_ptr p = first->ptr;
+ p.offset = nir_load_reg(&c->b, reg);
+ air_set(c, &in->base, air_val_ptr(c, p));
+ return true;
+}
+
static bool
translate_body(struct air_ctx *c)
{
@@ -1704,17 +5331,40 @@ translate_body(struct air_ctx *c)
/* phis first: each becomes a register, loaded where the phi is and stored
* at the end of every predecessor (docs/air.md: NIR's own
- * nir_lower_goto_ifs does the same to make an unstructured CFG safe)
+ * nir_lower_goto_ifs does the same to make an unstructured CFG safe) --
+ * except a phi of POINTERS, which has no bits and is resolved where it
+ * stands (resolve_pointer_phi). It gets no register, and the store loop at
+ * the end skips a phi that has none.
*/
nir_builder entry = nir_builder_at(nir_before_impl(c->b.impl));
for (unsigned bi = 0; bi < f->num_blocks; bi++) {
const struct air_block *blk = &f->blocks[bi];
for (unsigned i = 0; i < blk->num_insts; i++) {
const struct air_inst *in = blk->insts[i];
- if (in->op != AIR_OP_PHI)
+ if (in->op != AIR_OP_PHI || in->base.type->kind == AIR_TYPE_POINTER)
continue;
+ /* A phi of an AGGREGATE, named here rather than by whatever notices
+ * the consequence further down. A struct is a tree of values in this
+ * translator and never a NIR value, so there is no register to carry
+ * one across an edge; the honest form would be one phi per member,
+ * which this does not build. Until it was named, a register of ZERO
+ * bits was declared for it and the module failed later with
+ * "a returned struct that was never built" -- true of the module, and
+ * not what is wrong with it.
+ */
+ if (in->base.type->kind == AIR_TYPE_STRUCT ||
+ in->base.type->kind == AIR_TYPE_ARRAY)
+ return air_error(c, "instruction: a phi of %s, which NIR has no register for",
+ in->base.type->kind == AIR_TYPE_STRUCT ? "a struct"
+ : "an array");
+ /* A phi of a width NIR has no register for is exact: the register
+ * is the next width up and a phi only COPIES, so a canonical value
+ * in is a canonical value out (narrow_int_bits). What may be done
+ * with it afterwards is narrow_int_gate's business, and every use
+ * of it goes through there.
+ */
nir_def *reg = nir_decl_reg(&entry, air_nir_components(in->base.type),
- air_nir_bit_size(c, in->base.type), 0);
+ air_narrow_bits(c, in->base.type), 0);
_mesa_hash_table_insert(c->phi_regs, in, reg);
}
}
@@ -1727,8 +5377,50 @@ translate_body(struct air_ctx *c)
unsigned *order = rzalloc_array(c->mem_ctx, unsigned, MAX2(f->num_blocks, 1));
unsigned num_order = reverse_post_order(c, f, order);
+ /* WHICH BLOCKS ARE ENTERED, which is not the same as which are reachable
+ * in the CFG. A branch whose condition is already decided while translating
+ * -- a function constant, through the program-scope variable the module's
+ * air.static_init copies it into -- has one edge and not two, and the block
+ * on the other side is never entered. It is then not translated either,
+ * because translating it would build instructions for a path that does not
+ * exist and, worse, would make a phi of pointers see two operands where the
+ * shader has one (resolve_pointer_phi).
+ *
+ * This is filled as the walk goes rather than before it: the condition is a
+ * translated value, so it cannot be known earlier. Reverse post order is
+ * what makes that work -- every block but a loop header is reached after
+ * all of its predecessors, so by the time a block is visited every forward
+ * edge into it has been decided. A block nothing live branches to stays
+ * false and is skipped; it gets no NIR block, no phi cursor, and the phi
+ * store loop at the end already passes over an edge whose source has no
+ * cursor.
+ *
+ * It is also strictly narrower than what stood here, which marked every
+ * block of `order` live: a module that folds nothing gets exactly the old
+ * behaviour, because then every edge of every branch is taken.
+ */
+ bool *live = rzalloc_array(c->mem_ctx, bool, MAX2(f->num_blocks, 1));
+ if (num_order)
+ live[order[0]] = true;
+
+ /* THE ONE SUCCESSOR A BLOCK ACTUALLY BRANCHES TO, per block, or -1 for "all
+ * of them". Recorded rather than derived, because it is the only thing a
+ * phi on the far side can use: a block whose terminator folded still
+ * EXECUTES -- it is live -- and the operand arriving over an edge it does
+ * not take must not be counted.
+ *
+ * It is the taken successor and not the untaken ones because a folded
+ * SWITCH prunes every case but one, and one number says that as cheaply as
+ * it says a branch's single pruned edge.
+ */
+ int *taken_succ = rzalloc_array(c->mem_ctx, int, MAX2(f->num_blocks, 1));
+ for (unsigned i = 0; i < f->num_blocks; i++)
+ taken_succ[i] = -1;
+
for (unsigned oi = 0; oi < num_order; oi++) {
unsigned bi = order[oi];
+ if (!live[bi])
+ continue;
const struct air_block *blk = &f->blocks[bi];
nir_block *nb = nir_block_of(c, blk);
c->b.cursor = nir_after_block(nb);
@@ -1738,6 +5430,11 @@ translate_body(struct air_ctx *c)
const struct air_inst *in = blk->insts[i];
if (in->op != AIR_OP_PHI)
continue;
+ if (in->base.type->kind == AIR_TYPE_POINTER) {
+ if (!resolve_pointer_phi(c, in, live, taken_succ, bi))
+ return false;
+ continue;
+ }
struct hash_entry *e = _mesa_hash_table_search(c->phi_regs, in);
air_set(c, &in->base,
air_val_ssa(c, nir_load_reg(&c->b, e->data)));
@@ -1747,7 +5444,12 @@ translate_body(struct air_ctx *c)
const struct air_inst *in = blk->insts[i];
if (in->op == AIR_OP_PHI)
continue;
- if (!translate_instruction(c, in)) {
+ /* c->error as well as the return value: a refusal raised by something
+ * that cannot report one upwards -- air_nir_bit_size, handed a width
+ * NIR does not have -- must stop the walk here rather than let the
+ * next instruction be built on top of it.
+ */
+ if (!translate_instruction(c, in) || c->error[0]) {
/* say where, so the corpus report points at a line of the module */
char where[64];
snprintf(where, sizeof(where), " (block %u, instruction %u, opcode %u)",
@@ -1778,24 +5480,147 @@ translate_body(struct air_ctx *c)
break;
case AIR_OP_BR:
if (term->num_targets == 1) {
+ live[term->targets[0]->index] = true;
nir_goto(&c->b, nir_block_of(c, term->targets[0]));
} else {
nir_def *cond = air_get_ssa(c, term->operands[0]);
if (!cond)
return false;
- if (term->targets[0] == term->targets[1])
+ bool taken;
+ if (term->targets[0] == term->targets[1]) {
+ live[term->targets[0]->index] = true;
nir_goto(&c->b, nir_block_of(c, term->targets[0]));
- else
+ } else if (cond_const(c, cond, &taken)) {
+ /* ONE EDGE, because the condition is a constant here and not
+ * merely foldable later. The other block is left out of `live`
+ * and never translated, which is what lets a phi of pointers on
+ * the far side resolve; emitting nir_goto_if with a constant
+ * and trusting nir_opt_dead_cf would be too late for that, and
+ * would also leave the dead block's instructions translated.
+ */
+ struct air_block *to = term->targets[taken ? 0 : 1];
+ live[to->index] = true;
+ taken_succ[bi] = (int)to->index;
+ nir_goto(&c->b, nir_block_of(c, to));
+ } else {
+ live[term->targets[0]->index] = true;
+ live[term->targets[1]->index] = true;
nir_goto_if(&c->b, nir_block_of(c, term->targets[0]), cond,
nir_block_of(c, term->targets[1]));
+ }
+ }
+ break;
+ case AIR_OP_SWITCH: {
+ /* NIR has no switch, so the cases become a chain of nir_goto_if:
+ * each test gets its own block, and the last one falls to the
+ * default. nir_lower_goto_ifs structures the result like every other
+ * branch this translator emits.
+ *
+ * What makes the chain safe is where the phi stores already are: all
+ * of this block's successors' stores sit at phi_cursors[blk], above
+ * the chain, and each successor's phis have registers of their own --
+ * so storing on an edge the chain does not take writes a register
+ * that the block actually entered never reads. That is the same
+ * property the conditional br above relies on, and the reason a
+ * switch needs no per-edge bookkeeping.
+ */
+ nir_def *cond = air_get_ssa(c, term->operands[0]);
+ if (!cond)
+ return false;
+ /* A SWITCH WHOSE SELECTOR IS ALREADY DECIDED takes one case, exactly
+ * as a branch does.
+ *
+ * What stood here said the opposite, and said why: "the only switches
+ * in this corpus are the texture/blend/colour function dispatches of
+ * the GL-on-Metal uber shaders, where the selector is a uchar function
+ * constant and folding it would delete thirty-odd blocks at once -- a
+ * much larger change in what is compiled than the branch fold, for no
+ * module that the branch fold does not already reach." The first half
+ * is true and the last clause is FALSE, and fixed_frag_lph_cph is the
+ * counterexample: its asg77 filter case loads through the weights
+ * pointer, that pointer is `select fc_has_asg_77_weights, null, %14`
+ * and folds to NULL, and the case is reachable only through a switch
+ * on `fc_filter0`. Leaving the switch alone therefore translated a
+ * block the shader never enters and refused the module for a null
+ * dereference inside it -- a true sentence about code that does not
+ * run. The branch fold could not reach that; only this can.
+ *
+ * The default is targets[0] and the cases follow, so a selector that
+ * matches no case takes the default -- which is what makes folding
+ * total rather than a guess: there is always exactly one answer.
+ */
+ uint64_t sel;
+ if (uint_const(c, cond, &sel)) {
+ const struct air_block *to = term->targets[0]; /* the default */
+ for (unsigned k = 1; k < term->num_targets; k++) {
+ nir_def *cv = air_get_ssa(c, term->operands[k]);
+ uint64_t cval;
+ if (!cv)
+ return false;
+ if (uint_const(c, cv, &cval) && cval == sel) {
+ to = term->targets[k];
+ break;
+ }
+ }
+ live[to->index] = true;
+ taken_succ[bi] = (int)to->index;
+ nir_goto(&c->b, nir_block_of(c, (struct air_block *)to));
+ break;
+ }
+ for (unsigned k = 0; k < term->num_targets; k++)
+ live[term->targets[k]->index] = true;
+ nir_block *deflt = nir_block_of(c, term->targets[0]);
+ unsigned n = term->num_targets - 1;
+ for (unsigned k = 0; k < n; k++) {
+ nir_def *val = air_get_ssa(c, term->operands[1 + k]);
+ if (!val)
+ return false;
+ nir_block *dest = nir_block_of(c, term->targets[1 + k]);
+ bool last = k + 1 == n;
+ nir_block *next = deflt;
+ if (!last) {
+ next = nir_block_create(c->nir);
+ exec_list_push_tail(&c->b.impl->body, &next->cf_node.node);
+ next->cf_node.parent = &c->b.impl->cf_node;
+ }
+ /* a last case that goes where the default goes is not a branch at
+ * all, and nir_goto_if with one block on both sides is what the
+ * conditional br above refuses for the same reason
+ */
+ if (dest == next)
+ nir_goto(&c->b, dest);
+ else
+ nir_goto_if(&c->b, dest, nir_ieq(&c->b, cond, val), next);
+ if (!last)
+ c->b.cursor = nir_after_block(next);
}
+ if (n == 0)
+ nir_goto(&c->b, deflt);
break;
+ }
case AIR_OP_UNREACHABLE:
nir_goto(&c->b, c->b.impl->end_block);
break;
default:
return air_error(c, "instruction: block %u ends with opcode %u", bi, term->op);
}
+
+ /* The same reason as inside the instruction loop, for the TERMINATOR,
+ * which is translated here and not there. Without this a refusal raised
+ * while building a switch's case constants was only noticed at the first
+ * instruction of the next block in reverse-post-order, and the report
+ * then pointed at that instruction: resolve_sample_fragment's 3-bit
+ * switch in block 0 was attributed to a shufflevector in block 26, which
+ * is a location that sends the reader to the wrong line of the module.
+ */
+ if (c->error[0]) {
+ char where[64];
+ snprintf(where, sizeof(where), " (block %u, terminator, opcode %u)",
+ bi, term->op);
+ if (strlen(c->error) + strlen(where) < sizeof(c->error))
+ strcat(c->error, where);
+ return false;
+ }
}
/* the phi stores, now that every value exists */
@@ -1807,13 +5632,32 @@ translate_body(struct air_ctx *c)
continue;
struct hash_entry *re = _mesa_hash_table_search(c->phi_regs, in);
if (!re)
- continue; /* a phi in an unreachable block */
+ continue; /* a phi in an unreachable block, or one of pointers */
for (unsigned k = 0; k < in->num_operands; k++) {
struct hash_entry *ce = _mesa_hash_table_search(c->phi_cursors, in->targets[k]);
if (!ce)
continue; /* an edge from a block the entry cannot reach */
c->b.cursor = *(nir_cursor *)ce->data;
- nir_def *val = air_get_ssa(c, in->operands[k]);
+ nir_def *val;
+ if (in->base.type->kind == AIR_TYPE_POINTER && undef_operand(in->operands[k]))
+ continue; /* the register holds whatever it held: undef over this edge */
+ if (in->base.type->kind == AIR_TYPE_POINTER) {
+ /* a carried pointer phi (resolve_pointer_phi): the operand's offset, once
+ * it is known to name the phi's resource -- over a loop edge this is the
+ * first time it is seen at all
+ */
+ struct hash_entry *pe = _mesa_hash_table_search(c->values, &in->base);
+ struct hash_entry *oe = _mesa_hash_table_search(c->values, in->operands[k]);
+ const struct air_val *pv = pe ? pe->data : NULL, *ov = oe ? oe->data : NULL;
+ if (!pv || !ov || ov->kind != AIR_VAL_PTR || ov->ptr.class != pv->ptr.class ||
+ ov->ptr.binding != pv->ptr.binding || ov->ptr.is_image != pv->ptr.is_image)
+ return air_error(c, "instruction: a phi of pointers whose operand over "
+ "an edge names another resource, so which one it is "
+ "depends on the path taken");
+ val = ov->ptr.offset ? nir_u2u32(&c->b, ov->ptr.offset) : nir_imm_int(&c->b, 0);
+ } else {
+ val = air_get_ssa(c, in->operands[k]);
+ }
if (!val)
return false;
nir_store_reg(&c->b, val, re->data);
@@ -1850,11 +5694,388 @@ function_constant_index(struct air_ctx *c, const struct air_global *g)
return -1;
}
+/* The NAME and the required-ness of a function constant, by index: the other
+ * direction from function_constant_index, and needed where only the index is
+ * in hand -- the load path, which has a binding and not a global.
+ *
+ * `!{ptr @g, !"uint", !"name", i32 index, i1 required}`: the name is the last
+ * string before the first integer, and the required flag the boolean after it.
+ * Matched that way rather than by position, because the two keys this reads
+ * are the two whose places are attested across the corpus and nothing says the
+ * node cannot grow an operand.
+ */
+static const char *
+function_constant_decl(struct air_ctx *c, unsigned index, bool *required)
+{
+ const struct air_named_md *n = named_md(c->m, "air.function_constants");
+ for (unsigned i = 0; i < (n ? n->num_ops : 0); i++) {
+ const struct air_md *node = n->ops[i];
+ if (!node || node->kind != AIR_MD_NODE)
+ continue;
+ const char *name = NULL;
+ for (unsigned k = 0; k < node->num_ops; k++) {
+ const char *str = md_str(node, k);
+ if (str) {
+ name = str;
+ continue;
+ }
+ uint64_t idx;
+ if (!md_int(node, k, &idx))
+ continue;
+ if (idx != index)
+ break; /* a different constant: on to the next node */
+ if (required) {
+ uint64_t req = 0;
+ *required = md_int(node, k + 1, &req) && req != 0;
+ }
+ return name;
+ }
+ }
+ return NULL;
+}
+
/* Decide what every global is: once per shader, because this appends inline
* samplers to info->resources and lays out the shared and scratch memory. It
* records rather than builds -- making the value needs a builder, and which
* function's builder matters (see materialise_globals).
*/
+/* The 64-bit `constexpr sampler` word, field by field (nir_air.h has where the
+ * layout comes from and why it took three sources). Bit 63 marks the one-word
+ * form; the two-word form of AIR 2.6 carries a LOD bias in the second word's
+ * low 16 bits and no module of this corpus uses it, so it is reported rather
+ * than decoded.
+ */
+struct air_sampler_state
+air_decode_sampler_state(uint64_t w)
+{
+#define FIELD(lo, n) ((unsigned)((w >> (lo)) & ((1ull << (n)) - 1)))
+ struct air_sampler_state s = {
+ .address = { FIELD(0, 3), FIELD(3, 3), FIELD(6, 3) },
+ .mag_filter = FIELD(9, 2),
+ .min_filter = FIELD(11, 2),
+ .mip_filter = FIELD(13, 2),
+ .pixel_coord = FIELD(15, 1) != 0,
+ .compare_func = FIELD(16, 4),
+ .max_anisotropy = FIELD(20, 4) + 1,
+ .lod_min = FIELD(24, 16),
+ .lod_max = FIELD(40, 16),
+ .border_color = FIELD(56, 2),
+ .reduction = FIELD(58, 2),
+ };
+#undef FIELD
+ return s;
+}
+
+/* Does this value, or any constant built out of it, name that global?
+ *
+ * A constant expression is where a global's address escapes without any
+ * instruction mentioning it: `getelementptr (%struct, ptr @g, 0, 3)` is a
+ * CONSTANT in the bitcode, and the instruction that uses it names the
+ * expression and not @g. A scan that only compared instruction operands would
+ * therefore have called @_ZL25fc_fragment_shader_config -- whose every field
+ * is reached through exactly that shape -- a global nothing but whole loads
+ * touch, and folded a byte of it over the struct the shader actually reads.
+ */
+static bool
+const_mentions(const struct air_value *v, const struct air_value *target,
+ unsigned depth)
+{
+ if (!v || depth == 0)
+ return false;
+ if (v == target)
+ return true;
+ const struct air_constant *k = air_value_as_constant(v);
+ if (!k)
+ return false;
+ /* ON THE KIND, and not on whichever count happens to be non-zero.
+ * air_constant shares `num_elems` between an AGGREGATE, whose members are
+ * in `elems`, and DATA, whose numbers are in `data` and whose `elems` is
+ * NULL (air_module.c, CST_CODE_DATA) -- so walking `elems` whenever
+ * num_elems is set dereferences NULL on every string and every
+ * `<4 x i8>` data constant. That is what it did: 42 modules of the corpus
+ * died in here with SIGSEGV, every one of them a module whose
+ * air.static_init made this run at all.
+ */
+ switch (k->ckind) {
+ case AIR_CONST_AGGREGATE:
+ for (unsigned i = 0; i < k->num_elems; i++)
+ if (const_mentions(k->elems[i], target, depth - 1))
+ return true;
+ return false;
+ case AIR_CONST_EXPR:
+ for (unsigned i = 0; i < k->num_operands; i++)
+ if (const_mentions(k->operands[i], target, depth - 1))
+ return true;
+ return false;
+ default:
+ /* INT, FLOAT, NULL, UNDEF, POISON, DATA: numbers, and no value in them */
+ return false;
+ }
+}
+
+/* THE RESERVED SLOT A NULL TEXTURE LIVES IN -- road B of docs/air.md section 3,
+ * the frontend's half of it.
+ *
+ * `air.get_null_texture_2d` and its fifteen siblings are what a Metal shader
+ * asks for when the pipeline bound no image, and there is no index here that
+ * means "nothing bound": a texture is Metal's own `[[texture(n)]]`, resolved
+ * while translating, which is what lets a sample be one instruction with no
+ * descriptor indirection. So one index past the module's own is reserved for
+ * it and the resource is flagged, for the runtime to bind a 1x1 texture of
+ * zeros at (air_resource.null_texture in nir_air.h, which carries what that
+ * rests on).
+ *
+ * ONE PAST THE HIGHEST DECLARED INDEX, and that is deliberately NOT Apple's
+ * rule. Apple uses the texture COUNT -- createTextureHeapIndexArray stores the
+ * SmallVector's own Size (0x1bbe988b4) -- which is the same number only when
+ * the indices run 0..n-1. For a module that declares 0, 1 and 5, Apple's rule
+ * gives 3, which is a slot nothing declared but which a later version of the
+ * same shader could; one past the highest gives 6, which cannot collide with
+ * anything the module asked for. The safe one is chosen over the faithful one
+ * because a collision here binds a zero texture over a real image in silence.
+ *
+ * REFUSED BY NAME when it does not fit, rather than wrapped. AIR_MAX_TEXTURES
+ * is freedreno's own 16 (view_seqno[16] in fd6_texture_state), and
+ * fixed_frag_lph_cph declares 0..14 -- so its reserved slot is 15, the last
+ * one there is, and it fits with nothing to spare. A module that declares
+ * sixteen textures gets a refusal here and not an index of 16: the runtime's
+ * own check would otherwise refuse it too, but with the sentence "binds a
+ * texture at index 16", which blames the module for a slot THIS function
+ * added. That is the fourth fixed array of the day found smaller than
+ * something real, and the only one where the array's own maximum is the
+ * reserved slot itself.
+ */
+int
+air_null_texture_slot(struct air_ctx *c)
+{
+ if (c->null_texture_index >= 0)
+ return c->null_texture_index;
+
+ struct air_shader_info *info = c->info;
+ int highest = -1;
+ for (unsigned i = 0; i < info->num_resources; i++) {
+ const struct air_resource *r = &info->resources[i];
+ if (r->kind == AIR_RESOURCE_TEXTURE || r->kind == AIR_RESOURCE_IMAGE)
+ highest = MAX2(highest, (int)r->index);
+ }
+ unsigned slot = (unsigned)(highest + 1);
+ if (slot >= AIR_MAX_TEXTURES) {
+ air_error(c, "intrinsic: a null texture needs a reserved slot and this module "
+ "declares textures up to %d, so the slot would be %u -- past the %u "
+ "this GPU's descriptor table has (AIR_MAX_TEXTURES)",
+ highest, slot, AIR_MAX_TEXTURES);
+ return -1;
+ }
+
+ info->resources = reralloc(c->mem_ctx, info->resources, struct air_resource,
+ info->num_resources + 1);
+ /* Not through add_resource: there is no metadata node behind this one, and
+ * the two "no group"/"no argument buffer" fields have to be written rather
+ * than left zero -- a zeroed raster_order_group reads as group 0 and a
+ * zeroed argbuf_set as "member 0 of the set at Metal index 0", which is the
+ * same care plan_globals takes for an inline sampler.
+ */
+ info->resources[info->num_resources++] = (struct air_resource) {
+ .kind = AIR_RESOURCE_TEXTURE, .index = slot, .read = true,
+ .null_texture = true, .param = ~0u, .name = "(null texture)",
+ .raster_order_group = -1, .argbuf_set = ~0u, .argbuf_id = ~0u,
+ };
+ c->null_texture_index = (int)slot;
+ return (int)slot;
+}
+
+/* Which program-scope variables are a VALUE rather than memory.
+ *
+ * This is the whole of what "specialise the shader on its function constants"
+ * needs from the frontend, and it is a census rather than an optimisation: the
+ * GL-on-Metal uber shaders decide 58 function constants in an air.static_init
+ * constructor, copy them into program-scope variables, and then branch on
+ * those variables fifteen times to choose between a bound texture and a null
+ * one. plan_globals puts such a variable in SCRATCH, so what the body sees is
+ * a load_scratch -- opaque -- and every one of those branches survives
+ * translation, which is what left the two modules with fifteen phis between
+ * two different textures (docs/air.md section 3). Metal does not compile them
+ * that way: it specialises at pipeline creation, and this frontend already has
+ * every function constant as an immediate (AIR_PTR_FUNCTION_CONSTANT). The
+ * only thing in the way was the round trip through memory.
+ *
+ * A global qualifies only when all four hold, and each one is a way of being
+ * wrong that was considered rather than a box ticked:
+ *
+ * - its type is a scalar integer of 8, 16, 32 or 64 bits. Not a float, not a
+ * vector, not an aggregate: `bits` has to BE the value, and a narrower or
+ * odd width would go through air_narrow_bits and stop being the thing a
+ * later load wants.
+ * - every instruction that names it names it as the whole pointer of a load
+ * or of a store, and the type of that load or store is the global's own.
+ * A getelementptr, a call argument, a stored VALUE, a phi, a select: all
+ * disqualify, because then some other access reaches the same bytes and
+ * the value and the memory could disagree.
+ * - no constant expression and no other global's initialiser mentions it, so
+ * its address does not escape into something this scan cannot follow.
+ * - exactly one store, and that store is in a function whose section is
+ * air.static_init. One store is what makes the value independent of where
+ * in the shader the load is; the section is what makes it run before the
+ * body, which run_static_initializers inlines at the top of the entry
+ * point. A variable written by the body is a variable, and stays memory.
+ *
+ * What this does NOT decide is whether the value is a constant at all. That is
+ * known only when the store is translated, so `fold` means "a load of this may
+ * be answered" and `known` means "and here is the answer"; a store of
+ * something that is not constant simply never sets `known`, and the scratch
+ * underneath answers as it always did.
+ *
+ * AND IT DECIDES NOTHING ABOUT WHOSE VALUE IT IS. The constant comes from
+ * air_load_ptr's AIR_PTR_FUNCTION_CONSTANT case, which answers with
+ * air_constant_value.bits and does not look at `supplied` -- and
+ * air_module_function_constants registers every constant a module DECLARES
+ * with bits = 0 and supplied = false, so in every harness in tools/air this
+ * folds a shader on zeros that nobody set. That is not a new invention: the
+ * unfolded shader took the same branch for the same reason, at run time, off
+ * an immediate the same case produced. What changes is where the consequence
+ * lives -- a block that is not compiled rather than a branch that is not
+ * taken -- so the specialisation is exactly as good as the constants the
+ * runtime hands in and no better.
+ *
+ * It matters most for the one kind of constant that is allowed to be absent.
+ * The law (`!{ptr, !"type", !"name", i32 index, i1 required}`) is that a
+ * declaration ending `i1 false` is always guarded by
+ * air.is_function_constant_defined and one ending `i1 true` never is: 58 of
+ * 58 in fixed_frag_lph_cph, 24 required and unguarded against 34 optional and
+ * guarded, no exceptions. Fourteen of that module's fifteen texture
+ * predicates are REQUIRED, so Metal obliges the pipeline to supply them and
+ * folding them is specialisation. The fifteenth, fc_has_lut_tex at index 23,
+ * is optional -- and its constructor copies the RAW load into the variable
+ * rather than the guarded one, so the module itself reads an unsupplied
+ * constant as whatever the implementation gives. This passes that reading
+ * through unchanged instead of deciding it, which is the only honest thing to
+ * do from here: the decision belongs with whoever fills `supplied` in.
+ */
+static void
+plan_global_folding(struct air_ctx *c)
+{
+ for (unsigned i = 0; i < c->m->num_globals; i++) {
+ struct air_global_plan *pl = &c->global_plan[i];
+ const struct air_global *g = c->m->globals[i];
+ if (pl->skip || pl->class != AIR_PTR_SCRATCH)
+ continue;
+
+ const struct air_type *t = g->value_type;
+ if (!t || t->kind != AIR_TYPE_INT ||
+ (t->bits != 8 && t->bits != 16 && t->bits != 32 && t->bits != 64))
+ continue;
+
+ bool ok = true;
+ unsigned stores = 0;
+
+ for (unsigned k = 0; ok && k < c->m->num_globals; k++) {
+ if (k != i && const_mentions(c->m->globals[k]->init, &g->base, 8))
+ ok = false;
+ }
+
+ for (unsigned fi = 0; ok && fi < c->m->num_functions; fi++) {
+ const struct air_function *f = c->m->functions[fi];
+ bool is_init = f->section && !strcmp(f->section, "air.static_init");
+ for (unsigned bi = 0; ok && bi < f->num_blocks; bi++) {
+ const struct air_block *blk = &f->blocks[bi];
+ for (unsigned ii = 0; ok && ii < blk->num_insts; ii++) {
+ const struct air_inst *in = blk->insts[ii];
+ for (unsigned oi = 0; ok && oi < in->num_operands; oi++) {
+ const struct air_value *v = in->operands[oi];
+ if (v == &g->base) {
+ if (in->op == AIR_OP_LOAD && oi == 0 &&
+ in->base.type == t) {
+ continue;
+ }
+ if (in->op == AIR_OP_STORE && oi == 1 &&
+ in->operands[0]->type == t && is_init) {
+ stores++;
+ continue;
+ }
+ ok = false;
+ } else if (const_mentions(v, &g->base, 8)) {
+ ok = false;
+ }
+ }
+ }
+ }
+ }
+
+ if (ok && stores == 1)
+ pl->fold = true;
+ }
+}
+
+/* The global this operand IS -- not one it points into. Only a global used
+ * directly as a load's or a store's pointer can be folded, and
+ * plan_global_folding has already refused every other shape, so a linear scan
+ * over the module's globals answers it exactly.
+ */
+static struct air_global_plan *
+foldable_global_plan(struct air_ctx *c, const struct air_value *v)
+{
+ if (!c->global_plan || !v || v->kind != AIR_VALUE_GLOBAL)
+ return NULL;
+ for (unsigned i = 0; i < c->m->num_globals; i++) {
+ if (&c->m->globals[i]->base == v)
+ return c->global_plan[i].fold ? &c->global_plan[i] : NULL;
+ }
+ return NULL;
+}
+
+/* A constant's bytes as the module lays the type out (air_type_alloc_size, member
+ * offsets): what a read-only global holds, for nir->constant_data. False for what
+ * is not bytes -- a constant expression (a pointer's value), a type with no size.
+ */
+static bool
+write_constant(struct air_ctx *c, const struct air_type *t, const struct air_value *v, uint8_t *dst)
+{
+ const struct air_constant *k = air_value_as_constant(v);
+ if (!k)
+ return false;
+ const uint64_t size = air_type_alloc_size(c->m, t);
+ switch (k->ckind) {
+ case AIR_CONST_NULL:
+ case AIR_CONST_UNDEF:
+ case AIR_CONST_POISON:
+ memset(dst, 0, size);
+ return true;
+ case AIR_CONST_INT:
+ case AIR_CONST_FLOAT: {
+ const unsigned bytes = t->kind == AIR_TYPE_INT ? (t->bits + 7) / 8 : t->kind == AIR_TYPE_HALF || t->kind == AIR_TYPE_BFLOAT ? 2 :
+ t->kind == AIR_TYPE_FLOAT ? 4 : t->kind == AIR_TYPE_DOUBLE ? 8 : 0;
+ if (bytes == 0 || bytes > 8)
+ return false;
+ memcpy(dst, &k->u, bytes); /* little-endian, as the GPU reads it */
+ return true;
+ }
+ case AIR_CONST_DATA:
+ case AIR_CONST_AGGREGATE: {
+ if (t->kind != AIR_TYPE_ARRAY && t->kind != AIR_TYPE_VECTOR && t->kind != AIR_TYPE_STRUCT)
+ return false;
+ memset(dst, 0, size);
+ for (unsigned i = 0; i < k->num_elems; i++) {
+ const struct air_type *et = t->kind == AIR_TYPE_STRUCT ? t->members[i] : t->elem;
+ /* a vector's elements are packed at the element's size, an array's at its stride */
+ const uint64_t at = t->kind == AIR_TYPE_STRUCT ? air_struct_member_offset(c->m, t, i)
+ : t->kind == AIR_TYPE_VECTOR ? i * (uint64_t)((air_type_alloc_size(c->m, et)))
+ : i * air_type_alloc_size(c->m, et);
+ if (k->ckind == AIR_CONST_DATA) {
+ const uint64_t eb = air_type_alloc_size(c->m, et);
+ if (eb > 8)
+ return false;
+ memcpy(dst + at, &k->data[i], eb);
+ } else if (!write_constant(c, et, k->elems[i], dst + at)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ default:
+ return false;
+ }
+}
+
static void
plan_globals(struct air_ctx *c)
{
@@ -1908,6 +6129,12 @@ plan_globals(struct air_ctx *c)
.kind = AIR_RESOURCE_SAMPLER, .index = next, .read = true,
.param = ~0u, .name = name,
.inline_sampler = true, .sampler_state = word,
+ .sampler = air_decode_sampler_state(word),
+ /* not through add_resource, so the "no group" value has to be
+ * written here: a zeroed field would read as group 0, and a zeroed
+ * argbuf_set as "member 0 of the set at Metal index 0" */
+ .raster_order_group = -1,
+ .argbuf_set = ~0u, .argbuf_id = ~0u,
};
} else if (g->addrspace == 3) {
/* a static threadgroup array */
@@ -1917,6 +6144,59 @@ plan_globals(struct air_ctx *c)
pl->offset = c->shared_size;
pl->has_offset = true;
c->shared_size += air_type_alloc_size(c->m, g->value_type);
+ } else if (g->is_const && g->init && type_has_pointer(g->value_type, 8)) {
+ /* A READ-ONLY GLOBAL WHOSE INITIALISER HOLDS A POINTER, which scratch
+ * cannot hold any more than an alloca can: a pointer here is a
+ * resolution and not bytes. It becomes a translation-time slot
+ * (AIR_PTR_SLOT), pre-filled from its own initialiser by
+ * materialise_globals -- in whatever builder is asking, because the
+ * values inside it are defs and a def belongs to one impl.
+ *
+ * The shape this is for is one line of Metal:
+ *
+ * constexpr sampler unused_sampler;
+ *
+ * which Apple's compiler emits as
+ * `constant %"struct.metal::sampler" { ptr addrspace(2) @__air_sampler_state }`
+ * and QuartzCore's two YCbCr uber shaders then `select` against, before
+ * loading the sampler back out of whichever side won. Put in scratch,
+ * that global was bytes nothing ever wrote -- its initialiser holds a
+ * pointer, so there was nothing to write -- and a load through it
+ * would have read uninitialised memory and called it a sampler. The
+ * refusal it produced instead ("a pointer loaded from memory") was
+ * right, and this is what answers it.
+ *
+ * `is_const` is what makes the slot sound rather than convenient: the
+ * value cannot change, so one fill per function is the whole story,
+ * and AIR_OP_STORE refuses a store into a slot that has an
+ * initialiser rather than quietly overwriting one.
+ */
+ pl->class = AIR_PTR_SLOT;
+ pl->binding = new_slot(c, g->value_type, g->init);
+ } else if (g->is_const && g->init && !type_has_pointer(g->value_type, 8) &&
+ air_type_alloc_size(c->m, g->value_type) > 0) {
+ /* A READ-ONLY GLOBAL WITH ITS DATA -- a table, `constant float4 at[6]
+ * = {...}` -- is the shader's constant data (nir->constant_data, read
+ * by load_constant), its bytes written here from the initialiser. As a
+ * program-scope variable it went to scratch, where nothing writes the
+ * initialiser: sokol's bufferless triangle read its corners as zeros
+ * and drew nothing.
+ */
+ unsigned align = MAX2(g->align, air_type_abi_align(c->m, g->value_type));
+ uint32_t at = ALIGN_POT(c->nir->constant_data_size, MAX2(align, 4));
+ uint64_t size = air_type_alloc_size(c->m, g->value_type);
+ uint8_t *data = rzalloc_size(c->nir, at + size);
+ if (c->nir->constant_data_size)
+ memcpy(data, c->nir->constant_data, c->nir->constant_data_size);
+ if (!write_constant(c, g->value_type, g->init, data + at)) {
+ air_error(c, "interface: the constant global %s has an initialiser that is not plain data", name);
+ return;
+ }
+ c->nir->constant_data = data;
+ c->nir->constant_data_size = at + size;
+ pl->class = AIR_PTR_CONSTANT_DATA;
+ pl->offset = at;
+ pl->has_offset = true;
} else {
/* a program-scope variable: private to the invocation, written by
* the static initializers below
@@ -1929,6 +6209,9 @@ plan_globals(struct air_ctx *c)
c->scratch_size += air_type_alloc_size(c->m, g->value_type);
}
}
+
+ /* after the loop, because it reads the class every global ended up with */
+ plan_global_folding(c);
}
/* Build the globals as values of the function being translated now. Called for
@@ -1953,6 +6236,30 @@ materialise_globals(struct air_ctx *c)
};
air_set(c, &g->base, air_val_ptr(c, p));
}
+
+ /* THEN the slots, in a pass of their own, because an initialiser may name
+ * another global: `constant %"struct.metal::sampler" { ptr addrspace(2)
+ * @__air_sampler_state }` is a slot whose contents is the value of a global
+ * further down the list. Filling it inside the loop above read whichever
+ * globals happened to come first and left the rest unresolved, which came
+ * out as "a load through a null pointer" from inside a callee -- a true
+ * sentence about a pointer this function had not finished building.
+ *
+ * Re-filled per function rather than once, for the reason the offsets above
+ * are rebuilt per function: the values inside a slot are defs, and a def
+ * belongs to the impl that made it.
+ */
+ for (unsigned i = 0; i < c->m->num_globals; i++) {
+ const struct air_global_plan *pl = &c->global_plan[i];
+ if (pl->skip || pl->class != AIR_PTR_SLOT)
+ continue;
+ if (pl->binding >= c->num_slots || !c->slots[pl->binding].init)
+ continue;
+ struct air_val *v = air_get(c, c->slots[pl->binding].init);
+ if (!v)
+ return; /* air_get has said why */
+ c->slots[pl->binding].val = v;
+ }
}
/* The functions of @llvm.global_ctors, inlined at the top of the entry point:
@@ -1966,10 +6273,32 @@ run_static_initializers(struct air_ctx *c)
const struct air_function *f = c->m->functions[i];
if (f->is_declaration || !f->section || strcmp(f->section, "air.static_init"))
continue;
- if (f->num_args || f->num_blocks != 1) {
- air_error(c, "interface: a static initializer with arguments or branches");
+ if (f->num_args) {
+ air_error(c, "interface: a static initializer with arguments");
return;
}
+ if (f->num_blocks != 1) {
+ /* A static initializer with branches cannot be translated in place:
+ * the CFG walk owns the impl's start block, so running it a second
+ * time in the same impl would map two AIR blocks onto it. It becomes
+ * a nir_function instead and the entry point calls it, which is the
+ * machinery a call of another function in the module already uses,
+ * and nir_inline_functions folds it back in afterwards. The globals
+ * it writes are at the offsets plan_globals decided, which is one
+ * plan for the whole module, so the callee's stores and the entry
+ * point's loads reach the same scratch.
+ *
+ * These are the GL-on-Metal fragment shaders, whose initializer
+ * decides 58 function constants and has a switch in it -- which is
+ * also why this could not be reached before the reader learnt to read
+ * one.
+ */
+ nir_function *fn = translate_callee(c, f, NULL, NULL);
+ if (!fn)
+ return; /* translate_callee has said why */
+ nir_builder_instr_insert(&c->b, &nir_call_instr_create(c->nir, fn)->instr);
+ continue;
+ }
/* its instructions, in place, without its ret */
const struct air_block *blk = &f->blocks[0];
for (unsigned k = 0; k < blk->num_insts; k++) {
@@ -1982,8 +6311,141 @@ run_static_initializers(struct air_ctx *c)
}
}
+/*
+ * WHICH `device` BUFFERS THE TRANSLATED SHADER STORES THROUGH, out of the IR itself.
+ *
+ * A second reading of a fact the metadata already carries, taken because a runtime
+ * makes a decision on it that is wrong in SILENCE when the fact is wrong: an
+ * unaligned `device` buffer can be served by uploading a copy, exactly as a
+ * `constant` one already is, only if nothing writes it -- a copy of a buffer the
+ * shader writes would throw the writes away and the draw would look as though it had
+ * worked. `air.write`/`air.read_write` answers a different question, which is what
+ * Metal lets the shader do: `device T *` is declared writable whether or not the body
+ * ever stores through it, and in RenderBox's mask shaders none of them does.
+ *
+ * The reading is exact for every module that gets this far, and that is a consequence
+ * of refusals above rather than an assumption about stores. A store through a pointer
+ * chosen at run time is refused ("a store through a chosen pointer, which would have
+ * to write both of them"); a pointer written INTO memory, from where it could be
+ * fetched back and stored through, is refused ("a store of a pointer into memory");
+ * and a pointer whose binding did not resolve never becomes an AIR_PTR_SSBO at all.
+ * So every write to a device buffer that a translated module can perform is a
+ * store_ssbo or an ssbo_atomic carrying the immediate binding this translator put
+ * there -- and this runs after nir_inline_functions, so a callee's store is in the
+ * entry point's own IR by now.
+ *
+ * Where that argument does not reach, the answer is "written" and not "unknown": a
+ * store this scan cannot attribute to a declared buffer marks every device buffer in
+ * the module, because the alternative to over-refusing one draw is dropping a write.
+ */
+static void
+device_writes_from_ir(nir_shader *nir, struct air_shader_info *info)
+{
+ nir_foreach_function_impl(impl, nir) {
+ nir_foreach_block(block, impl) {
+ nir_foreach_instr(instr, block) {
+ if (instr->type != nir_instr_type_intrinsic)
+ continue;
+
+ nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
+ unsigned src;
+ switch (intr->intrinsic) {
+ case nir_intrinsic_store_ssbo:
+ src = 1;
+ break;
+ case nir_intrinsic_ssbo_atomic:
+ case nir_intrinsic_ssbo_atomic_swap:
+ src = 0;
+ break;
+ default:
+ continue;
+ }
+
+ bool named = false;
+ if (nir_src_is_const(intr->src[src])) {
+ unsigned binding = nir_src_as_uint(intr->src[src]);
+ for (unsigned i = 0; i < info->num_resources; i++) {
+ struct air_resource *r = &info->resources[i];
+ if (r->kind == AIR_RESOURCE_DEVICE_BUFFER && r->index == binding) {
+ r->ir_write = true;
+ named = true;
+ }
+ }
+ }
+ if (named)
+ continue;
+
+ for (unsigned i = 0; i < info->num_resources; i++)
+ if (info->resources[i].kind == AIR_RESOURCE_DEVICE_BUFFER)
+ info->resources[i].ir_write = true;
+ }
+ }
+ }
+}
+
/* -- the entry point ---------------------------------------------------- */
+/* Everything the caller is handed in INFO, copied out of the scratch context
+ * that is freed before air_to_nir returns: see the comment inside. Both
+ * returns that hand back a shader call it -- the interface-only one used to
+ * skip it, and handed out resource tables already freed, which a thread
+ * allocating at the same time overwrote: Resident Evil 4's argument encoders
+ * came back with every member id 0, wrote nothing, and failed every frame.
+ */
+static void
+keep_info(void *mem_ctx, struct air_shader_info *info)
+{
+ /* The info's arrays live under mem_ctx, not under the scratch context --
+ * and so must every STRING in them, which is the part this used to get
+ * wrong. `air_md_string` returns a NUL-terminated copy allocated as a
+ * child of the metadata node it came from, so every `air.arg_name` and
+ * `air.arg_type_name` in here pointed into the module, which is under
+ * `ctx`, which is freed three lines below. The caller was handed
+ * pointers into freed memory and mostly got away with it, because the
+ * bytes usually survive; what it looks like when they do not is
+ * `--interface` printing `name=on=1` for an attribute whose name is
+ * `texcoord1`, which is how this was found -- vattrcheck.py counted four
+ * attributes in a three-attribute shader because two garbled lines each
+ * carried a stray tab.
+ *
+ * vertex_attributes was not copied at all, which was the same bug with
+ * no bytes left to get away with.
+ */
+ if (info->num_resources) {
+ info->resources = ralloc_memdup(mem_ctx, info->resources,
+ info->num_resources * sizeof(*info->resources));
+ for (unsigned i = 0; i < info->num_resources; i++) {
+ info->resources[i].name = ralloc_strdup(mem_ctx, info->resources[i].name);
+ info->resources[i].type_name =
+ ralloc_strdup(mem_ctx, info->resources[i].type_name);
+ }
+ }
+ if (info->num_varyings) {
+ info->varyings = ralloc_memdup(mem_ctx, info->varyings,
+ info->num_varyings * sizeof(*info->varyings));
+ for (unsigned i = 0; i < info->num_varyings; i++)
+ info->varyings[i].name = ralloc_strdup(mem_ctx, info->varyings[i].name);
+ }
+ if (info->num_vertex_attributes) {
+ info->vertex_attributes =
+ ralloc_memdup(mem_ctx, info->vertex_attributes,
+ info->num_vertex_attributes * sizeof(*info->vertex_attributes));
+ for (unsigned i = 0; i < info->num_vertex_attributes; i++) {
+ struct air_vertex_attribute *a = &info->vertex_attributes[i];
+ a->name = ralloc_strdup(mem_ctx, a->name);
+ a->type_name = ralloc_strdup(mem_ctx, a->type_name);
+ a->function_constant = ralloc_strdup(mem_ctx, a->function_constant);
+ }
+ }
+ if (info->num_notes) {
+ info->notes = ralloc_memdup(mem_ctx, info->notes,
+ info->num_notes * sizeof(*info->notes));
+ for (unsigned i = 0; i < info->num_notes; i++)
+ info->notes[i] = ralloc_strdup(mem_ctx, info->notes[i]);
+ }
+ info->entry_point = ralloc_strdup(mem_ctx, info->entry_point);
+}
+
nir_shader *
air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_point,
const struct air_to_nir_options *options,
@@ -1996,6 +6458,9 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_po
.mem_ctx = ctx,
.options = options,
.info = info ? info : &local_info,
+ /* -1 and not 0: slot 0 is a real texture index, so a zeroed field would
+ * read as "the null texture already lives at index 0" */
+ .null_texture_index = -1,
};
memset(c.info, 0, sizeof(*c.info));
@@ -2022,8 +6487,14 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_po
* whose only job is to call it (wrap_library_function).
*/
bool wrap = false;
+ /* A stitched graph is the third thing this can be asked for: not this
+ * module as a shader, and not this module's function wrapped, but a graph
+ * of library functions compiled into one entry point, of which this
+ * module's function is one node like any other (stitch_graph).
+ */
+ bool stitch = options && options->stitch;
if (stage == MESA_SHADER_NONE) {
- if (!options || !options->library_wrapper) {
+ if (!options || !(options->library_wrapper || options->stitch)) {
if (error)
*error = ralloc_strdup(mem_ctx,
"stage: air.visible, a library function (not yet)");
@@ -2032,6 +6503,12 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_po
}
stage = MESA_SHADER_COMPUTE;
wrap = true;
+ } else if (stitch) {
+ if (error)
+ *error = ralloc_strdup(mem_ctx, "stitch: the module given is an entry point and "
+ "not an air.visible function");
+ ralloc_free(ctx);
+ return NULL;
}
if (!entry->num_ops || !entry->ops[0] || entry->ops[0]->kind != AIR_MD_VALUE ||
@@ -2065,6 +6542,7 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_po
main->is_entrypoint = true;
c.b = nir_builder_at(nir_before_impl(nir_function_impl_create(main)));
c.b.impl->structured = false;
+ c.const_cursor = nir_before_impl(c.b.impl);
c.values = _mesa_pointer_hash_table_create(ctx);
c.globals = _mesa_pointer_hash_table_create(ctx);
@@ -2110,6 +6588,16 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_po
for (unsigned i = 0; i < (in_node ? in_node->num_ops : 0) && !c.error[0]; i++)
translate_input(&c, in_node->ops[i], &next_in_varying);
+ /* Now that every direct argument's Metal index is known, the members of the
+ * argument buffers can be given bindings that cannot collide with them --
+ * and before plan_globals, so that a shader's own `constexpr sampler` still
+ * comes after every sampler the API binds.
+ */
+ if (!c.error[0] && c.num_argbufs)
+ allocate_argbuf_indices(&c);
+ if (!c.error[0] && options && options->interface_only)
+ goto interface_read;
+
/* the entry point's parameters, as the body will read them */
for (unsigned i = 0; i < (wrap ? 0 : c.num_params) && !c.error[0]; i++) {
if (c.params[i])
@@ -2120,23 +6608,38 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_po
if (!c.error[0])
plan_globals(&c);
+ if (!c.error[0] && c.tg_param_plus1) {
+ /* the threadgroup argument past the static arrays, 16-aligned as Metal
+ * aligns threadgroup memory; the value is shared with every use of the
+ * parameter (air_set keeps the pointer), so setting it here moves them all
+ */
+ c.shared_size = ALIGN_POT(c.shared_size, 16);
+ c.info->has_threadgroup_arg = true;
+ c.info->threadgroup_arg_base = c.shared_size;
+ if (c.shared_size)
+ c.params[c.tg_param_plus1 - 1]->ptr.offset = nir_imm_int(&c.b, c.shared_size);
+ }
if (!c.error[0])
materialise_globals(&c);
if (!c.error[0])
run_static_initializers(&c);
if (!c.error[0]) {
- if (wrap) {
+ if (stitch) {
+ stitch_graph(&c);
+ } else if (wrap) {
/* air.visible's second operand is the result node, whose one entry
* carries air.visible_output and the type's name
*/
const struct air_md *rn = entry->num_ops > 1 ? entry->ops[1] : NULL;
- wrap_library_function(&c, rn && rn->num_ops ? rn->ops[0] : NULL);
+ wrap_library_function(&c, rn && rn->num_ops ? rn->ops[0] : NULL,
+ entry->num_ops > 2 ? entry->ops[2] : NULL);
}
else
translate_body(&c);
}
+interface_read:
if (c.error[0]) {
if (error)
*error = ralloc_strdup(mem_ctx, c.error);
@@ -2180,6 +6683,15 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_po
c.nir->info.workgroup_size_variable = true;
}
+ /* the interface is all that was asked for: the passes below are for a body */
+ if (options && options->interface_only) {
+ ralloc_steal(mem_ctx, c.nir);
+ if (info)
+ keep_info(mem_ctx, info);
+ ralloc_free(ctx);
+ return c.nir;
+ }
+
nir_lower_goto_ifs(c.nir);
/* Fold in any function of the module the shader called, and then drop the
@@ -2191,6 +6703,21 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_po
* point, which every consumer of this NIR assumes.
*/
if (_mesa_hash_table_num_entries(c.fns)) {
+ /* A callee with more than one `ret` in it. Every return this translator
+ * emits is a goto to the impl's end block, and nir_lower_goto_ifs turns
+ * the ones that are not the last into nir_jump_return -- which
+ * nir_inline_functions ASSERTS on rather than refusing
+ * (nir_functions.c: "nir_instr_as_jump(instr)->type !=
+ * nir_jump_return"). nir_lower_returns is the pass that removes them,
+ * by making the body's own control flow carry the early exit, and it is
+ * what spirv_to_nir runs before inlining for the same reason.
+ *
+ * It is not hypothetical and it is not rare: RenderBox's blur returns
+ * early when the kernel is one tap wide, and before this the module
+ * aborted the whole air2nir process -- which a report counts as a crash
+ * and not as a refusal, so the measurement was lost as well.
+ */
+ NIR_PASS(_, c.nir, nir_lower_returns);
NIR_PASS(_, c.nir, nir_inline_functions);
NIR_PASS(_, c.nir, nir_opt_deref);
nir_remove_non_entrypoints(c.nir);
@@ -2204,14 +6731,8 @@ air_to_nir(void *mem_ctx, const void *bitcode, size_t size, const char *entry_po
nir_index_ssa_defs(nir_shader_get_entrypoint(c.nir));
if (info) {
- /* the info's arrays live under mem_ctx, not under the scratch context */
- if (info->num_resources)
- info->resources = ralloc_memdup(mem_ctx, info->resources,
- info->num_resources * sizeof(*info->resources));
- if (info->num_varyings)
- info->varyings = ralloc_memdup(mem_ctx, info->varyings,
- info->num_varyings * sizeof(*info->varyings));
- info->entry_point = ralloc_strdup(mem_ctx, info->entry_point);
+ device_writes_from_ir(c.nir, info);
+ keep_info(mem_ctx, info);
}
ralloc_free(ctx);
if (error)
diff --git a/tools/mesa/air/meson.build b/tools/mesa/air/meson.build
index 81be917..e1f1f24 100644
--- a/tools/mesa/air/meson.build
+++ b/tools/mesa/air/meson.build
@@ -13,6 +13,7 @@ files_libair = files(
'air_module.c',
'air_module.h',
'air_print.c',
+ 'air_stitch.c',
'air_private.h',
'air_to_nir.c',
'air_intrinsics.c',
diff --git a/tools/mesa/air/nir_air.h b/tools/mesa/air/nir_air.h
index 5e24918..7a12018 100644
--- a/tools/mesa/air/nir_air.h
+++ b/tools/mesa/air/nir_air.h
@@ -55,11 +55,146 @@ enum air_resource_kind {
AIR_RESOURCE_SAMPLER, /* sampler state at .index */
};
+/* Metal's `constexpr sampler`, as the 64-bit word a shader carries encodes it.
+ *
+ * The layout is not guessed and not read off one binary: the names, the enum
+ * values and the defaults are in Apple's own metal_stdlib header on the device
+ * (`GPUCompiler.framework/.../include/metal/metal_texture`, struct
+ * _sampler_state and _get_sampler_state), the field positions come from the
+ * encoder that writes the word (llvm::air::AIRSamplerStateEncoder::encode in
+ * libGPUCompilerImpl) and the decoder that reads it back
+ * (AIRConstSampler::constructFlatReply in AGXCompilerCore, with the flatbuffers
+ * schema beside it). Three sources, which is what it took: libGPUCompilerImpl
+ * also carries its own AIRAddressMode printer in which zero prints as
+ * clamp_to_border, and the string clamp_to_zero does not occur in that binary
+ * at all -- read alone it gives a plausible and wrong value for the commonest
+ * field.
+ *
+ * Decoding it against the corpus is the check that it is right: the three words
+ * the 456 modules carry come out as sampler(coord::pixel),
+ * sampler(coord::pixel, filter::linear) and
+ * sampler(coord::pixel, filter::linear, address::clamp_to_zero), with every
+ * other field equal to the header's default -- including a lod range of
+ * [0, 65504], which is two separate 16-bit halves that a wrong layout would
+ * not land on.
+ */
+enum air_sampler_address {
+ AIR_SAMPLER_CLAMP_TO_ZERO = 0, /* transparent black, and not a border colour */
+ AIR_SAMPLER_CLAMP_TO_EDGE = 1, /* the default */
+ AIR_SAMPLER_REPEAT = 2,
+ AIR_SAMPLER_MIRRORED_REPEAT = 3,
+ AIR_SAMPLER_CLAMP_TO_BORDER = 4,
+};
+
+enum air_sampler_filter {
+ AIR_SAMPLER_NEAREST = 0, /* the default */
+ AIR_SAMPLER_LINEAR = 1,
+ AIR_SAMPLER_BICUBIC = 2,
+};
+
+enum air_sampler_mip_filter {
+ AIR_SAMPLER_MIP_NONE = 0, /* the default */
+ AIR_SAMPLER_MIP_NEAREST = 1,
+ AIR_SAMPLER_MIP_LINEAR = 2,
+};
+
+struct air_sampler_state {
+ enum air_sampler_address address[3]; /* s, t, r */
+ enum air_sampler_filter mag_filter, min_filter;
+ enum air_sampler_mip_filter mip_filter;
+ bool pixel_coord; /* coord::pixel; the default is normalized */
+ unsigned compare_func;
+ unsigned max_anisotropy; /* 1 to 16 */
+ uint16_t lod_min, lod_max; /* halves, as their bits */
+ unsigned border_color;
+ unsigned reduction; /* AGX does not read this field */
+};
+
+/* HOW MANY TEXTURE SLOTS A SHADER MAY REACH, and it is freedreno's number and
+ * not a round one: fd6_texture_state keeps view_seqno[16] beside textures[32],
+ * and a seventeenth view walks out of the first array (RWA_MAX_TEXTURES in
+ * air_metal.c has the reading, and a peer session found the fragility). The
+ * public contract header cannot include this one -- air_metal.h is standalone
+ * so a driver can use it without NIR -- so the two are kept equal by a
+ * static assertion in air_metal.c rather than by one of them including the
+ * other.
+ *
+ * It is here, in the frontend's half, because the frontend now INVENTS an
+ * index: a null texture is a reserved slot past the module's own (see
+ * air_resource.null_texture), and inventing one that nothing can bind would be
+ * worse than refusing to.
+ */
+#define AIR_MAX_TEXTURES 16
+
struct air_resource {
enum air_resource_kind kind;
- unsigned index; /* Metal's [[buffer(n)]], [[texture(n)]], [[sampler(n)]] */
+ unsigned index; /* Metal's [[buffer(n)]], [[texture(n)]], [[sampler(n)]] --
+ * EXCEPT for an argument-buffer member, where it is the
+ * index this frontend allocated: see argbuf_set below */
unsigned size; /* air.buffer_size when given, else 0 */
bool read, write;
+
+ /* Whether the TRANSLATED SHADER stores through this binding -- a different
+ * fact from `write` above and read out of a different place, kept beside it
+ * rather than folded into it so that a consumer can compare the two.
+ *
+ * `write` is Apple's DECLARATION, out of `air.write`/`air.read_write`: a
+ * `device T *` argument is writable whether or not the body ever stores
+ * through it, and most of them never do. `ir_write` is what the IR the
+ * backend will compile actually contains. A runtime that must know whether
+ * a write can be LOST -- rwa_draw's unaligned `device` buffer, which can be
+ * uploaded like a `constant` one only if nothing writes it -- wants this
+ * one, and wants the declaration as a cross-check rather than as the
+ * answer.
+ *
+ * Only ever set for AIR_RESOURCE_DEVICE_BUFFER: a `constant` buffer cannot
+ * be stored to (air_store_ptr refuses it), and a texture or an image is not
+ * reached through a buffer binding.
+ */
+ bool ir_write;
+
+ /* A TEXTURE SLOT THE SHADER ASKED FOR AND THE APPLICATION DOES NOT BIND: the
+ * null texture (`air.get_null_texture_2d` and its fifteen siblings), which a
+ * Metal shader asks for when a pipeline bound no image. There is no index
+ * that means "nothing bound" -- a texture here is Metal's own
+ * `[[texture(n)]]`, resolved while translating -- so the frontend reserves
+ * one past the module's highest declared index and says so with this flag,
+ * which is Apple's own structure: where its textures are indices into a
+ * heap, its null texture is a reserved index in that heap
+ * (AGCLLVMUserObject::replaceTexturesAndSamplers, docs/air.md section 3).
+ *
+ * WHAT THE RUNTIME OWES IT, and what it must NOT do: create a 1x1
+ * zero-filled texture of the slot's own type and bind it there, and never
+ * demand that the application bind one. `param` is ~0u for it, as for an
+ * inline sampler, because no entry-point parameter carries it.
+ *
+ * AND THE ASSUMPTION IT RESTS ON, written here rather than implied: a zero
+ * texture is the right answer only if Metal's null texture reads as zero,
+ * and nothing this project can reach says that it does -- Apple's compiler
+ * substitutes a handle and leaves the result to the descriptor, so there is
+ * no semantics in it to borrow (docs/air.md section 3 has the reading). What
+ * this buys is not certainty but its LOCATION: the assumption lives in one
+ * 1x1 texture a boot can change, instead of being folded into every sample
+ * the frontend emits.
+ *
+ * WHICH TEXTURES TO MAKE IS NOT ONE PER SLOT. The same slot is sampled as
+ * both 2D and 3D in fixed_frag_lph_cph, so a caller needs one per (type,
+ * element class) PAIR -- rwa_texture_decl.sampled_as reports the set, and
+ * reports an empty one for a slot no instruction reads, which means bind
+ * nothing rather than bind everything.
+ *
+ * AND NOTHING YET SAYS THE TWO MODULES THIS EXISTS FOR ARE CORRECT. They
+ * translate and ir3 compiles them; that is all. Softpipe runs neither (f16
+ * registers, varyings, and an assertion in nir_to_tgsi) and cpuref compares
+ * neither (air.* calls LLVM has no body for, no rasteriser for the
+ * varyings), so the Adreno is the only machine that can say anything about
+ * them. And a pipeline that DRAWS is not evidence either: before this it was
+ * refused outright, so anything it draws is new rather than better. The
+ * reading that would mean something is a difference from the same boot
+ * without it.
+ */
+ bool null_texture;
+
unsigned param; /* the entry point's parameter; ~0u for an inline sampler */
const char *name; /* air.arg_name */
const char *type_name;
@@ -70,7 +205,45 @@ struct air_resource {
* samplers. The word's layout is not decoded yet -- see docs/air.md.
*/
bool inline_sampler;
- uint64_t sampler_state;
+ uint64_t sampler_state; /* the word itself, for a runtime that wants it raw */
+ struct air_sampler_state sampler; /* ... and decoded */
+
+ /* Metal's `[[raster_order_group(n)]]` on this resource: accesses to it from
+ * overlapping fragments happen in rasterization order. -1 when the shader
+ * asked for none. a6xx has no per-resource ordering at all -- see
+ * air_shader_info.raster_order_groups for what the runtime has to do, and
+ * for the note the frontend leaves when a shader asks for this.
+ */
+ int raster_order_group;
+
+ /* A member of an ARGUMENT BUFFER (`air.indirect_buffer`, Metal's
+ * `[[id(n)]]` struct) rather than an argument of its own, and the one place
+ * in this structure where `.index` is not Metal's number: ~0u when this
+ * resource is a direct argument, otherwise the Metal `[[buffer(n)]]` the
+ * SET is bound at and the member's own `[[id(n)]]` inside it.
+ *
+ * Both halves are needed because the two ends name the member differently
+ * and neither name can be derived from the other. The application names it
+ * by (set, id): that is what it passes to MTLArgumentEncoder. A member's own
+ * `air.location_index` is per set -- set 0's member 0 and set 1's member 0
+ * both say 0, and 237 of Resident Evil 4's modules have two sets -- so it is
+ * not a binding. The runtime binds by `.index`, which this frontend
+ * allocates: one flat counter per kind (buffer, texture, sampler), in
+ * (set, member) order, continuing above the highest index any DIRECT
+ * argument of that kind already took, so that a member and an argument
+ * cannot land in one Gallium slot.
+ *
+ * The allocation is over every member the set declares, used or not, so
+ * that it depends on the metadata alone: the runtime's encoder has to answer
+ * `encodedLength` for a set whichever of its members this shader reads, and
+ * a numbering that skipped the unread ones would differ between two shaders
+ * sharing one set. See docs/air.md section 8.2.
+ */
+ unsigned argbuf_set;
+ unsigned argbuf_id;
+ /* ...and where its eight bytes are in the set (air_argbuf_member.offset):
+ * what the runtime reads to find what was encoded there */
+ unsigned argbuf_offset;
};
struct air_varying {
@@ -80,6 +253,84 @@ struct air_varying {
bool flat, centroid, sample, no_perspective;
};
+/* What the shader wants of each component of a vertex attribute. The LLVM type
+ * cannot say whether an integer is signed -- `int4` and `uint4` are both
+ * `<4 x i32>` -- so the sign comes from `air.arg_type_name`, Apple's own
+ * reflection string, and from nowhere else. A runtime that checks an
+ * MTLVertexFormat against this is checking the shader's declared type against
+ * the format's converted type, which is what Metal's own pipeline validation
+ * checks.
+ */
+enum air_attribute_base_type {
+ AIR_ATTRIB_FLOAT, /* `float`, `half` */
+ AIR_ATTRIB_INT, /* `char`, `short`, `int` -- and the fallback with no type name */
+ AIR_ATTRIB_UINT, /* `uchar`, `ushort`, `uint` */
+};
+
+/* One member of a vertex function's `[[stage_in]]` struct: Metal's
+ * `[[attribute(n)]]`.
+ *
+ * AIR does not present the struct at all. Apple's compiler FLATTENS it: every
+ * member is an ordinary parameter of the entry point with its own
+ * `air.vertex_input` node, and the struct type is nowhere in the module. So
+ * there is no layout here and none can be inferred -- an attribute's format,
+ * offset, stride and buffer are the MTLVertexDescriptor's, which lives in the
+ * API and not in the shader. This table is the shader's HALF of that match:
+ * which indices it reads and what it wants each to be.
+ */
+struct air_vertex_attribute {
+ unsigned index; /* Metal's `[[attribute(n)]]`, `air.location_index` */
+ unsigned location; /* VERT_ATTRIB_GENERIC0 + index, the NIR input's location */
+ unsigned components; /* 1 to 4 */
+
+ /* The NIR input's driver_location, and the one field here a runtime cannot
+ * do without: Gallium matches vertex elements to shader inputs BY POSITION,
+ * not by location. ir3's setup_input indexes `so->inputs[]` by
+ * nir_intrinsic_base -- the driver_location nir_lower_io wrote -- and
+ * fd6_vertex_state_create emits VFD_DECODE[i] from `elements[i]` while
+ * emit_vfd_dest emits VFD_DEST_CNTL_INSTR[i] from `vs->inputs[i]`. So
+ * pipe_vertex_element[i] feeds the input whose driver_location is i, and
+ * these are assigned in the order AIR lists the attributes with the UNUSED
+ * ones skipped -- for std_vert0_lpf, 0 is attribute 0 and 1 is attribute 3.
+ * ~0u when `used` is false.
+ */
+ unsigned driver_location;
+
+ /* The width the SHADER reads, 8, 16 or 32. The NIR input is 32 bits wide
+ * whatever this says, and the narrowing is an instruction in the shader --
+ * see docs/air.md: ir3 puts a 16-bit vertex input in the HALF register file
+ * and then reports it in `ir3_shader_variant.inputs[]`, which has no half
+ * flag, as a full register, which is the number fd6_program.cc programs
+ * VFD_DEST_CNTL_INSTR_REGID from. A 32-bit input keeps the hardware fetch
+ * in the only regime either freedreno driver uses.
+ */
+ unsigned bit_size;
+ enum air_attribute_base_type base_type;
+
+ const char *name; /* air.arg_name */
+ const char *type_name; /* air.arg_type_name: "float4", "half4", "uchar4" ... */
+
+ /* False when Apple's metadata marks the parameter `air.arg_unused`: the
+ * attribute is in the shader's interface and nothing reads it, so there is
+ * NO NIR input for it and the runtime need not bind anything. Eight of
+ * QuartzCore's fourteen `[[stage_in]]` vertex shaders have one, and a fetch
+ * set up for a value nothing reads costs a VFD slot and a binding the
+ * application's descriptor may legally not have configured at all.
+ */
+ bool used;
+
+ /* The implicit function-constant predicate that gates this attribute, as
+ * Apple's metadata names it (`vtx_attr_has_texcoords`), or NULL. The
+ * parameter EXISTS either way -- the entry point's signature is fixed --
+ * so the attribute is translated either way; what the predicate decides is
+ * whether the shader reads it, and the value of that predicate comes from
+ * the function constants the pipeline supplies. There is no index to give:
+ * these globals are not in `!air.function_constants`, they are computed by
+ * the module's `air.static_init` constructor from the constants that are.
+ */
+ const char *function_constant;
+};
+
struct air_shader_info {
gl_shader_stage stage; /* MESA_SHADER_NONE for an air.visible function */
const char *entry_point;
@@ -87,17 +338,184 @@ struct air_shader_info {
struct air_resource *resources;
unsigned num_varyings; /* vertex outputs or fragment inputs */
struct air_varying *varyings;
+
+ /* The `[[stage_in]]` members of a vertex function, in the order AIR lists
+ * them (which is the order of the entry point's parameters and NOT the
+ * order of the attribute indices: QuartzCore's std_vert0_lpf declares
+ * 0, 1, 3). Empty for every other stage, and empty for a vertex shader that
+ * reads its vertices out of a buffer indexed by air.vertex_id -- Apple's
+ * three in the GL renderer and twenty-five of QuartzCore's thirty-nine.
+ */
+ unsigned num_vertex_attributes;
+ struct air_vertex_attribute *vertex_attributes;
+
+ /* One bit per Metal attribute index that the shader actually READS: the set
+ * the pipeline's MTLVertexDescriptor must configure. An attribute marked
+ * air.arg_unused is in vertex_attributes and not here. One caveat, which the
+ * translation also leaves as a note: an attribute gated by a function
+ * constant has its bit set whatever the constant says, because the parameter
+ * exists either way -- see air_vertex_attribute.function_constant before
+ * demanding a binding for a bit of this mask.
+ */
+ unsigned vertex_attributes_read;
+
unsigned num_render_targets;
+ /* The render targets a fragment shader READS -- Metal's `[[color(n)]]` argument,
+ * framebuffer fetch -- one bit per target. It is separate from num_render_targets,
+ * which is what the stage writes, because the two need not overlap: QuartzCore's
+ * `inplace_copy` reads target 0 and declares no colour output at all, and its
+ * `path_blit_frag` reads target 2 while writing 0, 1 and 2.
+ */
+ unsigned fbfetch_targets;
+
+ /* The raster order groups the shader declares on its resources, one bit
+ * each; 0 when it declares none.
+ *
+ * a6xx cannot give what this asks for. Metal's guarantee is per RESOURCE --
+ * two fragments that overlap are ordered against each other for the
+ * resources in a group and not for anything else -- and the only knob the
+ * hardware has is per DRAW: GRAS_SC_CNTL.SINGLE_PRIM_MODE, which
+ * freedreno drives from pipe_blend_state.blend_coherent (fd6_emit.cc,
+ * build_prim_mode). So a runtime that sets blend_coherent for any pipeline
+ * whose shader has a non-zero raster_order_groups gives the shader MORE
+ * synchronisation than it asked for, which is safe and slower; there is no
+ * way to give exactly what it asked for, and no way to give less.
+ *
+ * The frontend does not refuse such a shader -- the coarser guarantee is
+ * still a correct one -- but it does not pass over it in silence either:
+ * every resource with a group leaves a note in `notes`, which
+ * `air2nir --report` prints and tools/air/corpus.py counts.
+ */
+ unsigned raster_order_groups;
+
+ /* What was translated with a caveat: a shader this frontend accepted while
+ * giving something other than exactly what the module asked for. Nothing
+ * here is an error -- an error is a refusal and air_to_nir returns NULL for
+ * it -- and nothing here is advice. Each note names a thing the module asked
+ * for and what was done instead, so that it is in the log at translation
+ * time rather than only in a document.
+ */
+ unsigned num_notes;
+ const char **notes;
+
unsigned threadgroup_bytes; /* static threadgroup memory */
+
+ /* A [[threadgroup(0)]] argument, sized by the encoder (setThreadgroupMemoryLength:):
+ * it starts at threadgroup_arg_base bytes, past the static arrays, so the runtime
+ * gives the dispatch threadgroup_arg_base + that length.
+ */
+ bool has_threadgroup_arg;
+ unsigned threadgroup_arg_base;
};
+/* The word, field by field. A runtime that holds only `sampler_state` can call
+ * this; `air_resource.sampler` is what it returns.
+ */
+struct air_sampler_state air_decode_sampler_state(uint64_t word);
+
/* -- the translator --------------------------------------------------------- */
struct air_constant_value {
unsigned index; /* the function constant's index */
uint64_t bits; /* its value, as the bits of its type */
+
+ /*
+ * Whether the PIPELINE actually supplied that value, which is not the same question
+ * as whether the module declares the constant.
+ *
+ * air_module_function_constants() registers every constant a module declares, because
+ * it reads a declaration; a declaration carries no value, so `bits` is zero until a
+ * runtime fills it in. Those two facts together used to make
+ * air.is_function_constant_defined answer TRUE for every constant and then hand the
+ * module a zero -- and 80 of the corpus's 203 function constants exist precisely to
+ * be asked that question, every one of them guarded by that intrinsic with its own
+ * branch for "it was not supplied". Answering yes and giving zero takes the wrong
+ * branch with an invented value.
+ *
+ * rzalloc_array leaves this false, so a harness or a runtime that fills nothing says
+ * "nothing was supplied", which is the truth about it.
+ */
+ bool supplied;
+};
+
+/* -- function stitching ----------------------------------------------------
+ *
+ * Metal's own way of using an air.visible library function: the runtime hands
+ * the device a GRAPH of calls and gets back a function compiled out of them
+ * (-[MTLDevice newLibraryWithStitchedDescriptor:error:], and the SPI variant
+ * CoreImage actually uses). These three structures are that graph and nothing
+ * else: one field per property of Apple's own objects, read out of the
+ * device's shared cache rather than invented here, so that a runtime filling
+ * them in is transcribing a descriptor and not translating between two models.
+ *
+ * MTLFunctionStitchingGraph -> air_stitch_graph (functionName, nodes, outputNode)
+ * MTLFunctionStitchingFunctionNode -> air_stitch_node (name, arguments)
+ * MTLFunctionStitchingInputNode -> AIR_STITCH_INPUT (argumentIndex)
+ * MTLFunctionStitchingInputTexture -> AIR_STITCH_TEXTURE (bindIndex), and so
+ * ...InputSampler, ...InputBuffer on for the other SPI input nodes
+ * ...BuiltinThreadPositionInGrid -> AIR_STITCH_GRID_POSITION
+ *
+ * What is NOT here is the stitching ABI's four functions (load_argument,
+ * copy_from_buffer, store_return_value, destroy). They are in the modules, with
+ * bodies, and the graph says nothing about them: the frontend reads them from
+ * the metadata of each function it stitches (docs/air.md section 5, "The
+ * stitching ABI, before any of it is built").
+ */
+enum air_stitch_arg_kind {
+ AIR_STITCH_NODE, /* another node's result; index is the node */
+ AIR_STITCH_INPUT, /* the stitched function's own argument i */
+ AIR_STITCH_TEXTURE, /* a texture bound at index */
+ AIR_STITCH_SAMPLER, /* a sampler bound at index */
+ AIR_STITCH_BUFFER, /* a constant buffer bound at index */
+ AIR_STITCH_GRID_POSITION, /* thread_position_in_grid */
+};
+
+struct air_stitch_arg {
+ enum air_stitch_arg_kind kind;
+ unsigned index;
+};
+
+struct air_stitch_node {
+ const char *function; /* the air.visible function this node calls */
+ unsigned num_args;
+ const struct air_stitch_arg *args;
+};
+
+struct air_stitch_graph {
+ const char *name; /* the stitched function's name */
+ unsigned num_nodes;
+ const struct air_stitch_node *nodes; /* in dependency order */
+ unsigned output_node; /* whose result the function returns */
+};
+
+/* One function available to a graph: a module of the library, by name. The
+ * nodes of a graph name functions that live in DIFFERENT modules of the same
+ * .metallib -- every one of CoreImage's 135 is a module of its own -- so a
+ * stitched shader is built out of several bitcode blobs, which is why this
+ * exists at all.
+ */
+struct air_stitch_function {
+ const char *name;
+ const void *bitcode;
+ size_t size;
};
+/* A graph written as text, and the functions it names, for the tools that have
+ * to type one: "f(in0); g(n0)" (air_stitch.c says the whole syntax). The error
+ * buffer is the caller's, and gets one sentence with no prefix.
+ */
+const struct air_stitch_graph *air_stitch_parse(void *mem_ctx, const char *text,
+ char *error, size_t size);
+
+/* Every distinct function a graph names, found in an MTLB container by name:
+ * the nodes of one graph live in different modules of one .metallib. Returns
+ * how many, or 0 with *error set.
+ */
+unsigned air_stitch_functions_of(void *mem_ctx, const struct air_stitch_graph *g,
+ const void *data, size_t size,
+ struct air_stitch_function **out,
+ char *error, size_t esize);
+
struct air_to_nir_options {
const struct air_constant_value *constants;
unsigned num_constants;
@@ -112,6 +530,35 @@ struct air_to_nir_options {
* entry point of the corpus and is counted separately.
*/
bool library_wrapper;
+
+ /* Stitch a graph of air.visible functions into one entry point, instead of
+ * translating the module handed to air_to_nir as a shader of its own. The
+ * graph's nodes name functions of `stitch_functions`; the module passed to
+ * air_to_nir must be one of them (its own visible function is then just
+ * another node, and nothing else about it is used).
+ *
+ * The entry point generated is the library wrapper's, extended: argument i
+ * of the STITCHED function is read from device buffer 0 at byte 16*i and the
+ * output node's result is written to device buffer 1. That is deliberate and
+ * not a convenience -- a one-node graph then has to produce exactly the
+ * words `library_wrapper` produces for the same function, which is a
+ * reference that already exists for 78 of the 135 and is what makes the
+ * graphs above it decidable (docs/air.md section 5).
+ */
+ const struct air_stitch_graph *stitch;
+ const struct air_stitch_function *stitch_functions;
+ unsigned num_stitch_functions;
+
+ /* Read the entry point's interface -- its outputs, its inputs and every
+ * member of its argument buffers, with the bindings allocated as a full
+ * translation allocates them -- and stop there: *info is complete, the body
+ * is not translated and the NIR returned has none. For what a runtime has
+ * to answer before any pipeline exists: MTLFunction's
+ * -newArgumentEncoderWithBufferIndex: is asked of functions whose bodies
+ * this frontend cannot translate yet (Resident Evil 4's culling shaders,
+ * air.simd_ballot), and the layout of a set is in the metadata alone.
+ */
+ bool interface_only;
};
/* Translates one AIR module (one function of a .metallib). entry_point may
diff --git a/tools/mesa/ir3-shift-split.patch b/tools/mesa/ir3-shift-split.patch
new file mode 100644
index 0000000..9b49588
--- /dev/null
+++ b/tools/mesa/ir3-shift-split.patch
@@ -0,0 +1,84 @@
+ir3: moving a shift inside an add changes the number when the constant is
+negative.
+
+create_shift() in ir3_nir_lower_io_offsets.c turns the byte offset of an SSBO
+access into an element offset. When the offset has the shape `iadd constant,
+foo` it rewrites `(C + x) >> s` as `(x >> s) + (C >> s)`, so that
+nir_opt_offsets can fold the constant into the instruction's immediate field.
+
+That identity does not hold for a negative constant. `buf[i - 1].field` is
+ordinary code, and LLVM hands it over as `iadd 0xfffffffc, i*24` -- a
+subtraction. `(i*24 - 4) >> 2` borrows from the bits above the two it drops;
+`(i*24 >> 2) + (0xfffffffc >> 2)` adds 0x3fffffff instead of 0xffffffff and
+does not. The two differ by 2^30, which puts the element offset a gigabyte past
+the end of the buffer, and an out-of-range `ldib` returns zero without saying
+so. It also drops the low bits of a constant that is not a whole number of
+elements, which is the same mistake in a smaller way.
+
+Measured on the phone's Adreno 7c+ Gen 3. `fbs_bistochastize_iter` of
+PortraitFilters.metallib reads `grid_buf[id - 1].n` through exactly this shape
+and sums `id + 1` terms when the read returns 0 instead of `id`. With input
+built from the module's own metadata (tools/air/kinput.py, docs/air.md section
+5) its four outputs should all be 1/sqrt(3) -- one term each -- and on the
+device they were:
+
+ word device 1/sqrt(n) Apple's own LLVM on the host
+ 0 0x3f13cd3a 1/sqrt(3) 0x3f13cd3a
+ 1 0x3f000000 1/sqrt(4) 0x3f13cd3a
+ 2 0x3ee4f92e 1/sqrt(5) 0x3f13cd3a
+ 3 0x3ed105ec 1/sqrt(6) 0x3f13cd3a
+
+1/sqrt(3+i) in word i is the signature of the read returning 0 for every
+thread, and 0 is what that address returns. The three fbs_* kernels of the
+corpus carry this shape; the eight MetalFaceMask_* ones do not.
+
+So split the constant out only when it survives being shifted and cannot be a
+subtraction. Everything else keeps the shift outside the add, which is what the
+code did before the folding and is always right.
+
+--- a/src/freedreno/ir3/ir3_nir_lower_io_offsets.c
++++ b/src/freedreno/ir3/ir3_nir_lower_io_offsets.c
+@@ -126,6 +126,32 @@
+ return new_offset;
+ }
+
++/* Whether `(c + x) >> shift` and `(x >> shift) + (c >> shift)` are the same
++ * number for every x, which is what moving the shift inside the add assumes.
++ *
++ * They are not, twice over. A constant whose low `shift` bits are set loses
++ * them to `c >> shift`, so the offset comes out short. Worse, a negative
++ * constant -- a struct field reached through `buf[i - 1]`, which is ordinary
++ * code -- is a subtraction, and `(x - 4) >> 2` borrows from the bits above
++ * while `(x >> 2) + (0xfffffffc >> 2)` adds 0x3fffffff and does not: the two
++ * differ by 2^30 and the SSBO read lands outside the buffer, where a6xx
++ * returns zero without saying so.
++ *
++ * That was measured: fbs_bistochastize_iter reads grid_buf[id - 1].n with
++ * exactly this shape, and on an Adreno 7c+ Gen 3 it read 0 for every thread,
++ * which turned 1/sqrt(3) in all four of its outputs into 1/sqrt(3), 1/sqrt(4),
++ * 1/sqrt(5), 1/sqrt(6) (docs/air.md section 5).
++ *
++ * So split only a constant that is a whole number of elements and cannot be a
++ * subtraction. Everything else keeps the shift outside the add, which is what
++ * the code did before the folding was added and is always right.
++ */
++static bool
++shift_splits_over_iadd(uint32_t c, int shift)
++{
++ return (c & BITFIELD_MASK(shift)) == 0 && c < (1u << 31);
++}
++
+ static nir_def *
+ create_shift(nir_builder *b, nir_def *offset, int shift)
+ {
+@@ -138,7 +164,9 @@
+ nir_alu_instr *offset_instr = nir_instr_as_alu(offset->parent_instr);
+
+ if (offset_instr->op == nir_op_iadd &&
+- nir_src_is_const(offset_instr->src[0].src)) {
++ nir_src_is_const(offset_instr->src[0].src) &&
++ shift_splits_over_iadd(
++ nir_src_as_const_value(offset_instr->src[0].src)->u32, shift)) {
+ nir_def *new_shift = ir3_nir_try_propagate_bit_shift(
+ b, offset_instr->src[1].src.ssa, -shift);
+
diff --git a/tools/mesa/ir3-ssbo-subdword.patch b/tools/mesa/ir3-ssbo-subdword.patch
new file mode 100644
index 0000000..c8fce59
--- /dev/null
+++ b/tools/mesa/ir3-ssbo-subdword.patch
@@ -0,0 +1,121 @@
+ir3: an SSBO is a buffer of dwords to the Gallium driver, so keep sub-dword
+access off it.
+
+An SSBO on a6xx/a7xx is reached through an IBO descriptor, and `ldib`/`stib`
+take an offset in *that descriptor's* texels -- the hardware multiplies the
+index by the texel size the descriptor declares. ir3 does the dividing on the
+other side, in ir3_nir_lower_io_offsets.c's lower_offset_for_ssbo(), and it
+divides by the size of the *instruction*: shift 2 for a 32-bit access, 1 for
+16-bit, 0 for 8-bit. The two only agree when the descriptor's texel is as wide
+as the access.
+
+Turnip keeps them agreeing by writing a descriptor per width and picking one:
+write_buffer_descriptor_addr() lays down FMT6_16_UINT, FMT6_32_UINT and
+FMT6_8_UINT descriptors next to each other, and tu_lower_io()'s
+`descriptor_idx + 1` selects the 8-bit one for an 8-bit load or store.
+
+The Gallium driver writes one descriptor per SSBO slot and it is always
+PIPE_FORMAT_R32_UINT (fd6_image.cc, fd6_ssbo_descriptor()), with nothing that
+selects another. So on that driver a 16-bit SSBO access addresses twice the
+byte offset it was given and an 8-bit one four times, and an 8-bit store is
+worse than mis-addressed: ir3_a6xx.c emits it as `stib.b.typed...u16.4`, which
+is the blob's encoding for an imageBuffer of VK_FORMAT_R8 and means nothing
+against a 32-bit descriptor. The driver also leaves
+ir3_compiler_options::storage_16bit and storage_8bit false, which is already
+how ir3 knows the difference elsewhere -- ir3_nir_lower_ssbo_size() shifts the
+buffer length by 1 or 2 depending on exactly that, "a byte length divided by
+IBO_0_FMT".
+
+Nothing in GL reaches it, which is why it has stood: a GLSL SSBO member is at
+least 32 bits and at least 4-byte aligned, so ir3_mem_access_size_align()'s
+`(bytes & 1)` and `(bytes & 2)` cases never fire. A Metal shader is a different
+matter -- `ushort` and `uchar` fields in a device buffer are ordinary there,
+and the AIR frontend (src/compiler/air) translates them to 16- and 8-bit
+load_ssbo/store_ssbo.
+
+Measured on the phone's Adreno 7c+ Gen 3, two of Apple's compute kernels from
+PortraitFilters.metallib, given input built from each module's own metadata
+(docs/air.md section 5):
+
+ MetalFaceMask_AddSeedPoint one `uchar` store, to byte 28 of the span
+ table. The dispatch finished and byte 28 was
+ unchanged; the CPU reference over the same
+ bytes writes 0x5a there.
+ MetalFaceMask_PropagateParents walks a linked list of spans, leaving when
+ `nextSpanID` is 0xffff. The dispatch never
+ finished. Its `nextSpanID` reads land four
+ bytes apart instead of two, which turns the
+ lists 0, 1, 2, 3 of the input into the cycle
+ 0 -> 1 -> 3 -> 0, and that loop has no end.
+
+The fix is to ask for dwords whenever the descriptor is one, in
+ir3_mem_access_size_align(), where every other size decision is already made.
+nir_lower_mem_access_bit_sizes then does the work: a narrow load becomes a dword
+load at the masked offset plus a shift, and a narrow store becomes the iand/ior
+pair of 32-bit SSBO atomics the pass already keeps for this, which is why
+may_lower_unaligned_stores_to_atomics has to be set. Atomics rather than a plain
+read-modify-write matter here: these kernels have sixteen threads writing `uchar`
+fields that share a dword.
+
+Turnip is unaffected where it sets the options, and is fixed by the same change
+on a device where it cannot (a6xx without 16-bit storage, where an unaligned
+32-bit access is split into 16-bit ones against a 32-bit descriptor).
+
+cb_data was unset at both call sites, so it carries the compiler now; turnip's
+other call (push constants, tu_shader.cc) leaves it null and never sees an SSBO
+intrinsic, and a null is read as "no sub-dword descriptor" either way.
+
+--- a/src/freedreno/ir3/ir3_nir.c
++++ b/src/freedreno/ir3/ir3_nir.c
+@@ -953,6 +953,8 @@
+ uint32_t align_offset, bool offset_is_const,
+ enum gl_access_qualifier access, const void *cb_data)
+ {
++ const struct ir3_compiler *compiler = cb_data;
++
+ align = nir_combined_align(align, align_offset);
+ assert(util_is_power_of_two_nonzero(align));
+
+@@ -970,6 +972,29 @@
+ if (intrin == nir_intrinsic_load_ubo)
+ bit_size = 32;
+
++ /* ldib/stib scale the offset by the texel size of the IBO descriptor, while
++ * lower_offset_for_ssbo() scales it by the size of the access. A driver with
++ * no narrower descriptor to point them at must therefore stay on dwords:
++ * otherwise a 16-bit access reads and writes at twice the byte offset it was
++ * given and an 8-bit one at four times -- and an 8-bit store is emitted as a
++ * typed R8 store, which a 32-bit descriptor does not describe at all.
++ *
++ * nir_lower_mem_access_bit_sizes turns what is left into a dword load plus a
++ * shift, or a store into the pair of 32-bit atomics it keeps for this, so the
++ * narrow access still happens -- it happens through a descriptor that exists.
++ */
++ if ((intrin == nir_intrinsic_load_ssbo || intrin == nir_intrinsic_store_ssbo) &&
++ ((bit_size == 16 && (!compiler || !compiler->options.storage_16bit)) ||
++ (bit_size == 8 && (!compiler || !compiler->options.storage_8bit)))) {
++ bit_size = 32;
++ /* Below a dword of alignment the pass reads with a mask and a shift and
++ * writes with a pair of atomics, and both of those work a dword at a
++ * time -- so ask for one, however many narrow components there were.
++ */
++ if (align < 4)
++ bytes = MIN2(bytes, 4);
++ }
++
+ return (nir_mem_access_size_align){
+ .num_components = MAX2(1, MIN2(bytes / (bit_size / 8), 4)),
+ .bit_size = bit_size,
+@@ -1092,6 +1117,12 @@
+ nir_var_mem_global | nir_var_mem_shared |
+ nir_var_function_temp | nir_var_mem_ssbo,
+ .callback = ir3_mem_access_size_align,
++ /* so the callback can ask which descriptors this driver writes */
++ .cb_data = (void *)so->compiler,
++ /* a narrow store the callback widened becomes an iand/ior pair of 32-bit
++ * atomics; threads that share a dword then do not lose each other's bytes
++ */
++ .may_lower_unaligned_stores_to_atomics = true,
+ };
+
+ progress |= OPT(s, nir_lower_mem_access_bit_sizes, &mem_bit_size_options);