PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Integrate AutoRound Lib (#1723)
Zhenzhong Xu253117eb0542
diff --git a/.azure-pipelines/build-auto-round-lib.yml b/.azure-pipelines/build-auto-round-lib.ymlnew file mode 100644index 000000000..e1631b1ee--- /dev/null+++ b/.azure-pipelines/build-auto-round-lib.yml@@ -0,0 +1,17 @@+trigger:+ branches:+ include:+ - main+ paths:+ include:+ - auto_round_extension/ark/**+ - .azure-pipelines/build-auto-round-lib.yml+ - .azure-pipelines/template/lib-build-template.yml+ exclude:+ - "*.md"+ - "**/*.md"++stages:+ - template: template/lib-build-template.yml+ parameters:+ publishToTestPyPI: truediff --git a/.azure-pipelines/template/lib-build-template.yml b/.azure-pipelines/template/lib-build-template.ymlnew file mode 100644index 000000000..dcf6ec204--- /dev/null+++ b/.azure-pipelines/template/lib-build-template.yml@@ -0,0 +1,126 @@+parameters:+ - name: enableChangeDetection+ type: boolean+ default: false+ - name: publishToTestPyPI+ type: boolean+ default: false++stages:+ - ${{ if eq(parameters.enableChangeDetection, true) }}:+ - stage: DetectArkChanges+ displayName: Detect ark changes+ dependsOn: []+ jobs:+ - job: Detect+ displayName: Detect ark changes in PR+ pool:+ vmImage: "ubuntu-latest"+ steps:+ - checkout: self+ clean: true+ fetchDepth: 0++ - task: Bash@3+ name: SetArkWheelFlag+ displayName: Set ark wheel flag+ inputs:+ targetType: inline+ script: |+ set -euxo pipefail+ target_ref="${SYSTEM_PULLREQUEST_TARGETBRANCH:-refs/heads/main}"+ git fetch origin "+${target_ref}:${target_ref}"+ changed_files="$(git diff --name-only "${target_ref}"...HEAD)"+ echo "$changed_files"+ build_ark_wheel=false+ while IFS= read -r changed_file; do+ case "$changed_file" in+ auto_round_extension/ark/*.md)+ ;;+ auto_round_extension/ark/*)+ build_ark_wheel=true+ break+ ;;+ esac+ done <<< "$changed_files"++ if [[ "$build_ark_wheel" == "true" ]]; then+ echo "ARK code changes detected, need to build ark wheel."+ printf '%s\n' '##vso[task.setvariable variable=BUILD_ARK_WHEEL;isOutput=true]true'+ else+ echo "No ARK code changes detected, skip building ark wheel."+ printf '%s\n' '##vso[task.setvariable variable=BUILD_ARK_WHEEL;isOutput=true]false'+ fi++ - stage: BuildArkWheel+ displayName: Build ark wheel+ dependsOn:+ - ${{ if eq(parameters.enableChangeDetection, true) }}:+ - DetectArkChanges+ ${{ if eq(parameters.enableChangeDetection, true) }}:+ variables:+ BUILD_ARK_WHEEL: $[ stageDependencies.DetectArkChanges.Detect.outputs['SetArkWheelFlag.BUILD_ARK_WHEEL'] ]+ jobs:+ - job: BuildArkWheel+ displayName: Build ark wheel in manylinux+ ${{ if eq(parameters.enableChangeDetection, true) }}:+ condition: and(succeeded(), eq(variables['BUILD_ARK_WHEEL'], 'true'))+ pool: LIB-BUILD-POOL+ timeoutInMinutes: 10+ steps:+ - checkout: self+ clean: true+ fetchTags: true++ - task: Bash@3+ displayName: Build wheel in manylinux container+ inputs:+ targetType: inline+ script: |+ set -euxo pipefail+ docker run --rm \+ --user "$(id -u):$(id -g)" \+ -e HOME=/tmp \+ -v $(Build.SourcesDirectory):/auto-round \+ -w /auto-round/auto_round_extension/ark \+ pytorch/manylinux2_28-builder:xpu-v2.10.0-rc7 \+ bash -c " /opt/python/cp312-cp312/bin/python -m venv venv \+ && source venv/bin/activate \+ && source /opt/intel/oneapi/setvars.sh \+ && pip install -U pip build cmake setuptools \+ && python -m build --wheel --no-isolation \+ && auditwheel repair dist/*.whl --exclude lib* -w wheelhouse/ \+ && ls -lah wheelhouse"++ - task: PublishPipelineArtifact@1+ displayName: Publish wheel artifact+ inputs:+ targetPath: $(Build.SourcesDirectory)/auto_round_extension/ark/wheelhouse+ artifact: auto-round-lib-wheel+ publishLocation: pipeline++ - ${{ if eq(parameters.publishToTestPyPI, true) }}:+ - task: Bash@3+ displayName: Publish wheel to TestPyPI+ condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'), or(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.Reason'], 'BatchedCI')))+ env:+ TWINE_USERNAME: "__token__"+ TWINE_PASSWORD: "$(TESTPYPI_PASSWORD)"+ inputs:+ targetType: inline+ script: |+ set -euxo pipefail+ if [ -z "${TWINE_PASSWORD}" ]; then+ echo "TESTPYPI_PASSWORD is empty, skip publishing."+ exit 1+ fi+ docker run --rm --privileged \+ --user "$(id -u):$(id -g)" \+ -e HOME=/tmp \+ -v $(Build.SourcesDirectory):/auto-round \+ -w /auto-round/auto_round_extension/ark \+ -e TWINE_USERNAME \+ -e TWINE_PASSWORD \+ pytorch/manylinux2_28-builder:xpu-v2.10.0-rc7 \+ bash -c "python -m pip install --upgrade twine \+ && twine upload --repository-url https://test.pypi.org/legacy/ wheelhouse/*.whl"\ No newline at end of filediff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.ymlindex c7951350a..a1fe77af9 100644--- a/.azure-pipelines/template/ut-template.yml+++ b/.azure-pipelines/template/ut-template.yml@@ -29,6 +29,9 @@ parameters:- name: dockerFileNametype: stringdefault: "Dockerfile"+ - name: buildARKWheel+ type: string+ default: "false"steps:@@ -42,26 +45,38 @@ steps:repo: ${{ parameters.repo }}imageSource: ${{ parameters.imageSource }}+ - task: DownloadPipelineArtifact@2+ condition: and(succeeded(), eq(variables['BUILD_ARK_WHEEL'], 'true'))+ inputs:+ artifact: auto-round-lib-wheel+ patterns: '*.whl'+ path: $(Build.SourcesDirectory)/ark_wheel+ displayName: "Download ark wheel"+- ${{ if eq(parameters.imageSource, 'build') }}:- script: |+ if [ "${{ parameters.buildARKWheel }}" == "true" ]; then+ ARK_WHEEL_INSTALL="uv pip install /auto-round/ark_wheel/*.whl --no-deps"+ else+ ARK_WHEEL_INSTALL="uv pip install --index-url https://test.pypi.org/simple/ auto-round-lib --no-deps"+ fi++ docker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round \+ && uv pip install -r .azure-pipelines/scripts/ut/constraints.txt --extra-index-url https://download.pytorch.org/whl/xpu \+ && uv pip install -r auto_round_extension/ark/requirements.txt \+ && ${ARK_WHEEL_INSTALL} \+ && uv pip install -r requirements.txt"+if [ "${{ parameters.utScriptFileName }}" == "run_ut_xpu" ];thendocker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round \- && uv pip install -r .azure-pipelines/scripts/ut/constraints.txt --extra-index-url https://download.pytorch.org/whl/xpu \- && uv pip install auto-round-lib --constraint .azure-pipelines/scripts/ut/constraints.txt \- && uv pip install -r requirements.txt \&& uv pip install -r test/test_ark/requirements.txt \&& uv pip install -r test/test_xpu/requirements.txt --extra-index-url https://download.pytorch.org/whl/xpu \- && uv pip install -r .azure-pipelines/scripts/ut/constraints.txt --extra-index-url https://download.pytorch.org/whl/xpu \&& cd /auto-round && uv pip install . \&& uv pip list"elsedocker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round \- && uv pip install -r .azure-pipelines/scripts/ut/constraints.txt --extra-index-url https://download.pytorch.org/whl/cpu \- && uv pip install auto-round-lib --constraint .azure-pipelines/scripts/ut/constraints.txt \- && uv pip install -r requirements.txt \&& uv pip install -r requirements-cpu.txt \&& uv pip install -r test/test_cpu/requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu \- && uv pip install -r .azure-pipelines/scripts/ut/constraints.txt --extra-index-url https://download.pytorch.org/whl/cpu \&& uv pip list"fidisplayName: "Env Setup"diff --git a/.azure-pipelines/unit-test-xpu.yml b/.azure-pipelines/unit-test-xpu.ymlindex adb04bf27..814595cec 100644--- a/.azure-pipelines/unit-test-xpu.yml+++ b/.azure-pipelines/unit-test-xpu.yml@@ -17,6 +17,8 @@ pr:- .azure-pipelines/unit-test-xpu.yml- .azure-pipelines/scripts/ut/run_ut_xpu.sh- .azure-pipelines/docker/Dockerfile_xpu.devel+ - .azure-pipelines/build-auto-round-lib.yml+ - .azure-pipelines/template/lib-build-template.ymlexclude:- "*.md"- "**/*.md"@@ -34,9 +36,17 @@ variables:REPO: $(Build.Repository.Uri)stages:+ - template: template/lib-build-template.yml+ parameters:+ enableChangeDetection: true+ publishToTestPyPI: false+- stage: Unit_testdisplayName: Unit Test- dependsOn: []+ dependsOn: [DetectArkChanges, BuildArkWheel]+ condition: and(succeeded('DetectArkChanges'), in(dependencies.BuildArkWheel.result, 'Succeeded', 'Skipped'))+ variables:+ BUILD_ARK_WHEEL: $[ stageDependencies.DetectArkChanges.Detect.outputs['SetArkWheelFlag.BUILD_ARK_WHEEL'] ]jobs:- job:displayName: Unit Test@@ -50,6 +60,7 @@ stages:imageTag: $(IMAGE_TAG)dockerFileName: $(DOCKERFILE_NAME)utContainerName: "AutoRoundUnitTest$(NODE_LABEL)"+ buildARKWheel: $(BUILD_ARK_WHEEL)- task: UseDotNet@2displayName: 'Use .NET Core sdk 7.0.x'diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.ymlindex 9c5da664e..b0a361dd4 100644--- a/.azure-pipelines/unit-test.yml+++ b/.azure-pipelines/unit-test.yml@@ -14,12 +14,13 @@ pr:- setup.py- requirements.txt- requirements-cpu.txt+ - .azure-pipelines/build-auto-round-lib.yml- .azure-pipelines/scripts/ut- .azure-pipelines/unit-test.yml+ - .azure-pipelines/template/lib-build-template.yml- .azure-pipelines/template/ut-template.yml- .azure-pipelines/template/docker-template.ymlexclude:- - auto_round_extension/ark- test/test_hpu- test/test_ark- test/test_xpu@@ -41,9 +42,17 @@ variables:REPO: $(Build.Repository.Uri)stages:+ - template: template/lib-build-template.yml+ parameters:+ enableChangeDetection: true+ publishToTestPyPI: false+- stage: Unit_testdisplayName: Unit Test- dependsOn: []+ dependsOn: [DetectArkChanges, BuildArkWheel]+ condition: and(succeeded('DetectArkChanges'), in(dependencies.BuildArkWheel.result, 'Succeeded', 'Skipped'))+ variables:+ BUILD_ARK_WHEEL: $[ stageDependencies.DetectArkChanges.Detect.outputs['SetArkWheelFlag.BUILD_ARK_WHEEL'] ]jobs:- job:timeoutInMinutes: 120@@ -68,6 +77,7 @@ stages:utArtifact: "ut-$(PART)"utTestMode: $(PART)utContainerName: "AutoRoundUnitTest$(NODE_LABEL)"+ buildARKWheel: $(BUILD_ARK_WHEEL)- stage: CoveragedisplayName: "Collect Coverage"diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex 049c35b3d..ade2ca1ce 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -787,7 +787,7 @@ def dynamic_import_inference_linear(backend, config, packing_format=None):import auto_round_kernel as ark # pylint: disable=E0611, E0401except Exception as e:raise ImportError("Please install auto-round-lib for CPU/XPU, e.g., using: pip install auto-round-lib")- import auto_round_extension.ark.qlinear as qlinear+ import auto_round_kernel.qlinear as qlinear # pylint: disable=E0611, E0401if "zp" in backend:return qlinear.QuantLinearGPTQdiff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 112d53629..370baf29a 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -478,9 +478,6 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features,bias=bias,)- # Special handling for auto-round-lib AWQ layers- from auto_round_extension.ark.qlinear import QuantLinearAWQ-if "auto_round_kernel" in layer_backend:return QuantLinear(bits=config["bits"],@@ -491,7 +488,10 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features,bias=bias,weight_dtype=layer.weight.dtype,)- if "awq" in layer_backend and isinstance(QuantLinear, QuantLinearAWQ):+ if (+ "awq" in layer_backend+ and f"{QuantLinear.__module__}.{QuantLinear.__class__.__name__}" == "auto_round_kernel.qlinear.QuantLinearAWQ"+ ):return QuantLinear.from_linear(layer, config["bits"], config["group_size"], init_only=True, has_zero_points=not config["sym"])diff --git a/auto_round_extension/ark/LICENSE b/auto_round_extension/ark/LICENSEnew file mode 100644index 000000000..2bb9ad240--- /dev/null+++ b/auto_round_extension/ark/LICENSE@@ -0,0 +1,176 @@+ Apache License+ Version 2.0, January 2004+ http://www.apache.org/licenses/++ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION++ 1. Definitions.++ "License" shall mean the terms and conditions for use, reproduction,+ and distribution as defined by Sections 1 through 9 of this document.++ "Licensor" shall mean the copyright owner or entity authorized by+ the copyright owner that is granting the License.++ "Legal Entity" shall mean the union of the acting entity and all+ other entities that control, are controlled by, or are under common+ control with that entity. For the purposes of this definition,+ "control" means (i) the power, direct or indirect, to cause the+ direction or management of such entity, whether by contract or+ otherwise, or (ii) ownership of fifty percent (50%) or more of the+ outstanding shares, or (iii) beneficial ownership of such entity.++ "You" (or "Your") shall mean an individual or Legal Entity+ exercising permissions granted by this License.++ "Source" form shall mean the preferred form for making modifications,+ including but not limited to software source code, documentation+ source, and configuration files.++ "Object" form shall mean any form resulting from mechanical+ transformation or translation of a Source form, including but+ not limited to compiled object code, generated documentation,+ and conversions to other media types.++ "Work" shall mean the work of authorship, whether in Source or+ Object form, made available under the License, as indicated by a+ copyright notice that is included in or attached to the work+ (an example is provided in the Appendix below).++ "Derivative Works" shall mean any work, whether in Source or Object+ form, that is based on (or derived from) the Work and for which the+ editorial revisions, annotations, elaborations, or other modifications+ represent, as a whole, an original work of authorship. For the purposes+ of this License, Derivative Works shall not include works that remain+ separable from, or merely link (or bind by name) to the interfaces of,+ the Work and Derivative Works thereof.++ "Contribution" shall mean any work of authorship, including+ the original version of the Work and any modifications or additions+ to that Work or Derivative Works thereof, that is intentionally+ submitted to Licensor for inclusion in the Work by the copyright owner+ or by an individual or Legal Entity authorized to submit on behalf of+ the copyright owner. For the purposes of this definition, "submitted"+ means any form of electronic, verbal, or written communication sent+ to the Licensor or its representatives, including but not limited to+ communication on electronic mailing lists, source code control systems,+ and issue tracking systems that are managed by, or on behalf of, the+ Licensor for the purpose of discussing and improving the Work, but+ excluding communication that is conspicuously marked or otherwise+ designated in writing by the copyright owner as "Not a Contribution."++ "Contributor" shall mean Licensor and any individual or Legal Entity+ on behalf of whom a Contribution has been received by Licensor and+ subsequently incorporated within the Work.++ 2. Grant of Copyright License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ copyright license to reproduce, prepare Derivative Works of,+ publicly display, publicly perform, sublicense, and distribute the+ Work and such Derivative Works in Source or Object form.++ 3. Grant of Patent License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ (except as stated in this section) patent license to make, have made,+ use, offer to sell, sell, import, and otherwise transfer the Work,+ where such license applies only to those patent claims licensable+ by such Contributor that are necessarily infringed by their+ Contribution(s) alone or by combination of their Contribution(s)+ with the Work to which such Contribution(s) was submitted. If You+ institute patent litigation against any entity (including a+ cross-claim or counterclaim in a lawsuit) alleging that the Work+ or a Contribution incorporated within the Work constitutes direct+ or contributory patent infringement, then any patent licenses+ granted to You under this License for that Work shall terminate+ as of the date such litigation is filed.++ 4. Redistribution. You may reproduce and distribute copies of the+ Work or Derivative Works thereof in any medium, with or without+ modifications, and in Source or Object form, provided that You+ meet the following conditions:++ (a) You must give any other recipients of the Work or+ Derivative Works a copy of this License; and++ (b) You must cause any modified files to carry prominent notices+ stating that You changed the files; and++ (c) You must retain, in the Source form of any Derivative Works+ that You distribute, all copyright, patent, trademark, and+ attribution notices from the Source form of the Work,+ excluding those notices that do not pertain to any part of+ the Derivative Works; and++ (d) If the Work includes a "NOTICE" text file as part of its+ distribution, then any Derivative Works that You distribute must+ include a readable copy of the attribution notices contained+ within such NOTICE file, excluding those notices that do not+ pertain to any part of the Derivative Works, in at least one+ of the following places: within a NOTICE text file distributed+ as part of the Derivative Works; within the Source form or+ documentation, if provided along with the Derivative Works; or,+ within a display generated by the Derivative Works, if and+ wherever such third-party notices normally appear. The contents+ of the NOTICE file are for informational purposes only and+ do not modify the License. You may add Your own attribution+ notices within Derivative Works that You distribute, alongside+ or as an addendum to the NOTICE text from the Work, provided+ that such additional attribution notices cannot be construed+ as modifying the License.++ You may add Your own copyright statement to Your modifications and+ may provide additional or different license terms and conditions+ for use, reproduction, or distribution of Your modifications, or+ for any such Derivative Works as a whole, provided Your use,+ reproduction, and distribution of the Work otherwise complies with+ the conditions stated in this License.++ 5. Submission of Contributions. Unless You explicitly state otherwise,+ any Contribution intentionally submitted for inclusion in the Work+ by You to the Licensor shall be under the terms and conditions of+ this License, without any additional terms or conditions.+ Notwithstanding the above, nothing herein shall supersede or modify+ the terms of any separate license agreement you may have executed+ with Licensor regarding such Contributions.++ 6. Trademarks. This License does not grant permission to use the trade+ names, trademarks, service marks, or product names of the Licensor,+ except as required for reasonable and customary use in describing the+ origin of the Work and reproducing the content of the NOTICE file.++ 7. Disclaimer of Warranty. Unless required by applicable law or+ agreed to in writing, Licensor provides the Work (and each+ Contributor provides its Contributions) on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+ implied, including, without limitation, any warranties or conditions+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A+ PARTICULAR PURPOSE. You are solely responsible for determining the+ appropriateness of using or redistributing the Work and assume any+ risks associated with Your exercise of permissions under this License.++ 8. Limitation of Liability. In no event and under no legal theory,+ whether in tort (including negligence), contract, or otherwise,+ unless required by applicable law (such as deliberate and grossly+ negligent acts) or agreed to in writing, shall any Contributor be+ liable to You for damages, including any direct, indirect, special,+ incidental, or consequential damages of any character arising as a+ result of this License or out of the use or inability to use the+ Work (including but not limited to damages for loss of goodwill,+ work stoppage, computer failure or malfunction, or any and all+ other commercial damages or losses), even if such Contributor+ has been advised of the possibility of such damages.++ 9. Accepting Warranty or Additional Liability. While redistributing+ the Work or Derivative Works thereof, You may choose to offer,+ and charge a fee for, acceptance of support, warranty, indemnity,+ or other liability obligations and/or rights consistent with this+ License. However, in accepting such obligations, You may act only+ on Your own behalf and on Your sole responsibility, not on behalf+ of any other Contributor, and only if You agree to indemnify,+ defend, and hold each Contributor harmless for any liability+ incurred by, or claims asserted against, such Contributor by reason+ of your accepting any such warranty or additional liability.++ END OF TERMS AND CONDITIONS\ No newline at end of filediff --git a/auto_round_extension/ark/README.md b/auto_round_extension/ark/README.mdindex 6897a5f0d..a36eef2b3 100644--- a/auto_round_extension/ark/README.md+++ b/auto_round_extension/ark/README.md@@ -37,55 +37,48 @@ AutoRound Kernel provides weight-only linear computational capabilities for LLM## Installation-### Install via pip+### 1. Install via pip```bash-# Install the latest auto-round kernel which may upgrade your PyTorch version automatically-pip install auto-round-lib-# Install auto-round kernel with respective to specific PyTorch version (e.g., v2.8.x)-pip install auto-round-lib torch~=2.8.0+pip install auto-round-lib```-<details>-<summary>Other Installation Methods</summary>--### Install via Script+### 2. Install from Source```bash-curl -fsSL https://raw.githubusercontent.com/intel/auto-round/main/auto_round_extension/ark/install_kernel.py-python3 install_kernel.py+python setup.py bdist_wheel;pip install dist/*```-**Notes:**-Recommend to use this method if you want to keep your current PyTorch and auto-round versions.-This installation script will detect the current environment and install the corresponding auto-round-lib version.-### Install via auto_round-```bash-pip install auto-round-auto-round-lib-install-```+### Validated Hardware Environment+#### CPU based on [Intel 64 architecture or compatible processors](https://en.wikipedia.org/wiki/X86-64):+* Intel Xeon Scalable processor (Granite Rapids)+#### GPU built on Intel's Xe architecture:+* Intel Arc B-Series Graphics (Battlemage)-</details>+### Resources-### Versioning Scheme-The version number of auto-round-lib follows the format:-`{auto-round major version}.{auto-round minor version}.{oneAPI version}.{lib version}`+#### QuantLinear API+ ARK exposes a unified weight-only linear interface through QuantLinear, QuantLinearGPTQ, QuantLinearAWQ, and QuantLinearFP8. Please refer to the [QLinear](auto_round_kernel/qlinear.py) for more integration details.-**For example: v0.10.1.1**-- The first two digits (0.10) correspond to the major and minor version of the auto_round framework.-- The third component (1) represents the major version of Intel oneAPI: `1` indicates support for oneAPI 2025.1 (typically Torch 2.8), `2` indicates support for oneAPI 2025.2 (typically Torch 2.9), and `3` indicates support for oneAPI 2025.3 (typically Torch 2.10).-- The final digit (1) is the patch version of auto-round-lib, reflecting updates, bug fixes, or improvements to the kernel package itself.+ The expected lifecycle is: create the module, load quantized tensors from the checkpoint, call post_init() once to repack weights into the ARK-friendly layout, and then call forward() during inference.-**Version mapping table**+ Minimal usage:+```python+from auto_round_kernel.qlinear import QuantLinear-| auto-round-lib Version | auto-round Version | oneAPI Version | Typical PyTorch Version |-|:----------------------:|:------------------:|:--------------:|:-----------------------:|-| 0.10.1.x | 0.10.x | 2025.1 | 2.8.x |-| 0.10.2.x | 0.10.x | 2025.2 | 2.9.x |-| 0.10.3.x | 0.10.x | 2025.3 | 2.10.x |+qlinear = QuantLinear(+ bits=4,+ group_size=128,+ sym=True,+ in_features=in_features,+ out_features=out_features,+ bias=bias is not None,+ weight_dtype=weight_dtype,+)+# Load qweight, qzeros, scales, and bias from checkpoint.+qlinear.post_init()-**Notes:** oneAPI version is aligned with PyTorch version during auto-round-lib binary build, but oneAPI toolkit is not required in runtime.+# Run inference+y = qlinear(x)+```-### Validated Hardware Environment-#### CPU based on [Intel 64 architecture or compatible processors](https://en.wikipedia.org/wiki/X86-64):-* Intel Xeon Scalable processor (Granite Rapids)-#### GPU built on Intel's Xe architecture:-* Intel Arc B-Series Graphics (Battlemage)\ No newline at end of file+#### A Weight-Only Example+ A runnable end-to-end example is available in [test_weightonly.py](test/test_weightonly.py). It demonstrates how to prepare quantized weights and scales, call repack_quantized_weight to build ARK-packed weights, verify correctness with unpack_weight, and run woqgemm on CPU and XPU.diff --git a/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt b/auto_round_extension/ark/auto_round_kernel/CMakeLists.txtnew file mode 100755index 000000000..1d799d098--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/CMakeLists.txt@@ -0,0 +1,169 @@+cmake_minimum_required(VERSION 3.5...3.31)+project(ark LANGUAGES CXX)++option(ARK_XPU "Build XPU kernels" OFF)+option(ARK_UT "Build XPU kernels UT" OFF)+option(ARK_RESCALE "Exoerimental" OFF)+option(ARK_DNNL_BUILD_SOURCE "Exoerimental" OFF)++if(NOT ARK_XPU OR WIN32)+ set(ARK_DNNL_BUILD_SOURCE ON)+endif()++# ARK_SYCL_TLA defaults to ON when ARK_XPU is enabled+if(ARK_XPU AND NOT DEFINED ARK_SYCL_TLA)+ set(ARK_SYCL_TLA ON CACHE BOOL "Build SYCL TLA (auto-enabled with ARK_XPU)")+else()+ option(ARK_SYCL_TLA "Build SYCL TLA" OFF)+endif()++include(FetchContent)+FetchContent_Declare(+ pybind11+ GIT_REPOSITORY https://github.com/pybind/pybind11.git+ GIT_TAG v3.0.1+)+FetchContent_MakeAvailable(pybind11)++include(FindOpenMP)++set(BTLA_ENABLE_OPENMP ON CACHE BOOL "BesTLA enable compiling OpenMP threading")++if(ARK_DNNL_BUILD_SOURCE)+ FetchContent_Declare(+ dnnl+ GIT_REPOSITORY https://github.com/uxlfoundation/oneDNN.git+ GIT_TAG v3.10.2+ )+ set(libs dnnl)+ set(DNNL_LIBRARY_TYPE "STATIC" CACHE INTERNAL "")+ set(DNNL_CPU_RUNTIME "OMP" CACHE INTERNAL "")+ set(DNNL_CPU_THREADING_RUNTIME "OMP")++ set(DNNL_BUILD_EXAMPLES OFF CACHE INTERNAL "")+ set(DNNL_BUILD_TESTS OFF CACHE INTERNAL "")+ set(DNNL_BUILD_DOC OFF CACHE INTERNAL "")+ set(ONEDNN_BUILD_GRAPH OFF CACHE INTERNAL "")+ set(ONEDNN_ENABLE_WORKLOAD "INFERENCE" CACHE INTERNAL "")+else()+ find_package(DNNL REQUIRED)+ set(libs DNNL::dnnl)+endif()++if(ARK_XPU)+ if(ARK_DNNL_BUILD_SOURCE)+ set(ONEDNN_ENABLE_PRIMITIVE_CPU_ISA "AVX2" CACHE INTERNAL "")+ set(DNNL_GPU_RUNTIME "SYCL" CACHE INTERNAL "")+ FetchContent_MakeAvailable(dnnl)+ endif()+ set(BTLA_SYCL ON CACHE BOOL "BesTLA with SYCL")+ set(PY_NAME auto_round_kernel_xpu)+ set(ARK_TYPE ARK_XPU)+ find_package(IntelSYCL REQUIRED)+ list(APPEND libs IntelSYCL::SYCL_CXX)+else()+ if(ARK_DNNL_BUILD_SOURCE)+ set(DNNL_GPU_RUNTIME "NONE" CACHE INTERNAL "")+ FetchContent_MakeAvailable(dnnl)+ endif()+ set(BTLA_SYCL OFF CACHE BOOL "BesTLA without SYCL")+ set(PY_NAME auto_round_kernel_cpu)+ set(ARK_TYPE ARK_CPU)+endif()++add_subdirectory(bestla)+if(MSVC)+ target_compile_options(bestla INTERFACE /w)+else()+ target_compile_options(bestla INTERFACE -w)+endif()+list(APPEND libs bestla)++file(GLOB SRCS ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)+set(SDPA_GENERATED_SRCS)+set(SDPA_KERNEL_DECLARATIONS)++include_directories(wrapper/include)++# Build flash_attn_wrapper as a separate static library with sycl-tla flags+if(ARK_XPU AND ARK_SYCL_TLA)+ # Fetch sycl-tla headers (header-only, don't build)+ set(SYCL_TLA_GIT_REPOSITORY "https://github.com/luoyu-intel/sycl-tla.git" CACHE STRING "sycl-tla git repository")+ set(SYCL_TLA_GIT_TAG "260409" CACHE STRING "sycl-tla git tag/commit")++ FetchContent_Declare(+ sycl_tla+ GIT_REPOSITORY ${SYCL_TLA_GIT_REPOSITORY}+ GIT_TAG ${SYCL_TLA_GIT_TAG}+ )++ # Map target to device name for -Xs flag+ set(DPCPP_SYCL_TARGET "intel_gpu_bmg_g21" CACHE STRING "SYCL target (intel_gpu_pvc, intel_gpu_bmg_g21)")+ if(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_bmg_g21" OR DPCPP_SYCL_TARGET STREQUAL "bmg")+ set(SYCL_DEVICE_NAME "bmg_g21")+ elseif(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_pvc" OR DPCPP_SYCL_TARGET STREQUAL "pvc")+ set(SYCL_DEVICE_NAME "pvc")+ else()+ set(SYCL_DEVICE_NAME "${DPCPP_SYCL_TARGET}")+ endif()++ # Link flags for sycl-tla (applied to final module)+ set(SYCL_TLA_LINK_FLAGS+ -fsycl+ -fsycl-targets=spir64+ "-Xs" "-device ${SYCL_DEVICE_NAME}"+ -Xspirv-translator+ "-spirv-ext=+SPV_INTEL_split_barrier,+SPV_INTEL_2d_block_io,+SPV_INTEL_subgroup_matrix_multiply_accumulate")++ set(CUTLASS_ENABLE_SYCL ON)+ set(CUTLASS_ENABLE_BENCHMARKS OFF)+ set(CUTLASS_ENABLE_EXAMPLES OFF)+ set(CUTLASS_ENABLE_TESTS OFF)+ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)+ set(CUTLASS_ENABLE_LIBRARY OFF)+ set(CUTLASS_ENABLE_TOOLS OFF)+ set(CUTLASS_ENABLE_GDC_FOR_SM100_DEFAULT+ OFF+ CACHE BOOL "DISABLE CUDA")+ FetchContent_MakeAvailable(sycl_tla)++ set(_sycl_tla_include_dirs+ ${CUTLASS_DIR}/include+ ${CUTLASS_DIR}/applications+ ${CUTLASS_DIR}/tools/util/include+ ${CUTLASS_DIR}/examples/common+ ${CUTLASS_DIR}/examples/06_bmg_flash_attention+ ${CUTLASS_DIR}/examples/12_xe20_moe_gemm_cute_interface+ )+ include(${CMAKE_CURRENT_LIST_DIR}/sdpa_generation.cmake)+endif()++pybind11_add_module(${PY_NAME} ${SRCS} ${HEADERS} ${SDPA_GENERATED_SRCS})+target_compile_features(${PY_NAME} PRIVATE cxx_std_17)+target_compile_definitions(${PY_NAME} PRIVATE PY_NAME=${PY_NAME} ${ARK_TYPE}=1)+if(ARK_RESCALE)+ target_compile_definitions(${PY_NAME} PRIVATE ARK_RESCALE=1)+endif()+if(ARK_XPU AND ARK_SYCL_TLA)+ target_compile_definitions(${PY_NAME} PRIVATE ARK_SYCL_TLA=1 CUTLASS_ENABLE_SYCL=1 SYCL_INTEL_TARGET=1)+ target_include_directories(${PY_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/wrapper/include)+ target_include_directories(${PY_NAME} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/generated/sdpa)+ # Use SYSTEM include directories for sycl-tla to suppress warnings/errors from third-party headers+ # (e.g., std::common_type specialization issues in traits.hpp)+ foreach(_inc_dir IN LISTS _sycl_tla_include_dirs)+ if(EXISTS "${_inc_dir}")+ target_include_directories(${PY_NAME} SYSTEM PRIVATE "${_inc_dir}")+ endif()+ endforeach()+ target_compile_options(${PY_NAME} PRIVATE -fsycl -fno-sycl-instrument-device-code)+ target_link_options(${PY_NAME} PRIVATE ${SYCL_TLA_LINK_FLAGS})+endif()+target_link_libraries(${PY_NAME} PRIVATE ${libs})++if(ARK_UT)+ set(TEST_NAME test_${ARK_TYPE})+ add_executable(${TEST_NAME} wrapper/test/test_main.cpp)+ target_compile_features(${TEST_NAME} PRIVATE cxx_std_17)+ target_compile_definitions(${TEST_NAME} PRIVATE ${ARK_TYPE}=1)+ target_link_libraries(${TEST_NAME} PRIVATE ${libs})+endif()diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.pynew file mode 100644index 000000000..48778aff3--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py@@ -0,0 +1,984 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from typing import Optional+import torch+import sys+++class ARK_DT:+ float64 = 64+ float32 = 32+ float16 = 16+ bfloat16 = 65552+ int2 = 258+ int3 = 259+ int4 = 260+ int5 = 261+ int6 = 262+ int7 = 263+ int8 = 264+ int32 = 288+ float8_e4m3 = 8+ float8_e5m2 = 65544+ float8_e8m0 = 196616+ undef = 0+++def cvt_dtype(dtype):+ if dtype == torch.float32:+ return ARK_DT.float32+ if dtype == torch.float16:+ return ARK_DT.float16+ if dtype == torch.bfloat16:+ return ARK_DT.bfloat16+ if dtype == torch.float8_e4m3fn:+ return ARK_DT.float8_e4m3+ if dtype == torch.float8_e5m2:+ return ARK_DT.float8_e5m2+ if dtype == torch.int8:+ return ARK_DT.int8+ if dtype == torch.int32:+ return ARK_DT.int32+ return ARK_DT.undef+++def cvtstr_dtype(dtype):+ if dtype == "fp32":+ return ARK_DT.float32+ if dtype == "fp16":+ return ARK_DT.float16+ if dtype == "bf16":+ return ARK_DT.bfloat16+ if dtype == "fp8_e4m3":+ return ARK_DT.float8_e4m3+ if dtype == "fp8_e5m2":+ return ARK_DT.float8_e5m2+ if dtype == "fp8_e8m0":+ return ARK_DT.float8_e8m0+ if dtype == "int8":+ return ARK_DT.int8+ if dtype == "int4":+ return ARK_DT.int4+ if dtype == "int2":+ return ARK_DT.int2+ if dtype == "int3":+ return ARK_DT.int3+ if dtype == "int5":+ return ARK_DT.int5+ if dtype == "int6":+ return ARK_DT.int6+ if dtype == "int7":+ return ARK_DT.int7+ if dtype == "int32":+ return ARK_DT.int32+ return ARK_DT.undef+++def get_stream(A: torch.Tensor) -> int:+ if A.device.type == "cpu":+ return 0+ if A.device.type == "xpu":+ return torch.xpu.current_stream().sycl_queue+++def singleton(cls):+ """+ 一个简单的单例模式装饰器+ """+ instances = {} # 存储类与实例的映射关系++ def get_instance(*args, **kwargs):+ # 如果类不在字典中,则创建一个并存入+ if cls not in instances:+ instances[cls] = cls(*args, **kwargs)+ # 如果已经在字典中,直接返回之前创建的那个+ return instances[cls]++ return get_instance+++@singleton+class ARK:+ cpu_lib = None+ xpu_lib = None++ def __init__(self):+ try:+ from . import auto_round_kernel_cpu++ self.cpu_lib = auto_round_kernel_cpu+ except ImportError as e:+ print(f"ARK is unable to load CPU lib: {e}")+ self.cpu_lib = None++ if torch.xpu.is_available():+ try:+ from . import auto_round_kernel_xpu++ self.xpu_lib = auto_round_kernel_xpu+ except ImportError as e:+ print(f"ARK is unable to load XPU lib: {e}")+ self.xpu_lib = None++ def get_lib(self, A: torch.Tensor):+ lib = None+ if A.device.type == "xpu":+ lib = self.xpu_lib+ if A.device.type == "cpu":+ lib = self.cpu_lib+ if lib is None:+ raise NotImplementedError(f"Current device {A.device} is not supported")+ return lib++ # A: mxk, B: nxk, bias: n+ def matmul(self, A: torch.Tensor, B: torch.Tensor, bias: torch.Tensor):+ m = A.shape[0]+ n = B.shape[0]+ k = B.shape[1]+ lib = self.get_lib(A)+ ctype = A.dtype+ if A.device.type == "cpu":+ ctype = torch.float32+ C = torch.zeros(m, n, dtype=ctype, device=A.device)+ stream = get_stream(A)+ lib.matmul(+ stream,+ m,+ n,+ k,+ A.contiguous().data_ptr(),+ cvt_dtype(A.dtype),+ B.contiguous().data_ptr(),+ cvt_dtype(B.dtype),+ C.contiguous().data_ptr(),+ cvt_dtype(C.dtype),+ bias.to(C.dtype).contiguous().data_ptr(),+ True,+ )+ return C++ # A: mxk:s8, B: nxk:s8, return: mxn:s32+ def igemm_s8s8s32(self, A: torch.Tensor, B: torch.Tensor):+ m = A.shape[0]+ n = B.shape[0]+ k = B.shape[1]+ lib = self.get_lib(A)+ if lib is None:+ raise NotImplementedError(f"Current device {A.device} is not supported")+ C = torch.zeros(m, n, dtype=torch.int32, device=A.device)+ stream = get_stream(A)+ lib.matmul(+ stream,+ m,+ n,+ k,+ A.contiguous().data_ptr(),+ cvt_dtype(A.dtype),+ B.contiguous().data_ptr(),+ cvt_dtype(B.dtype),+ C.contiguous().data_ptr(),+ cvt_dtype(C.dtype),+ 0,+ True,+ )+ return C++ # A: mxk:DT, B: nxk:s8, scaleB: n:DT+ # return: mxn:DT+ def woqgemm_s8(self, A: torch.Tensor, B: torch.Tensor, scaleB: torch.Tensor, bias: torch.Tensor):+ m = A.shape[0]+ n = B.shape[0]+ k = B.shape[1]+ lib = self.get_lib(A)++ C = torch.zeros(m, n, dtype=A.dtype, device=A.device)+ stream = get_stream(A)+ lib.woqgemm_s8(+ stream,+ m,+ n,+ k,+ A.contiguous().data_ptr(),+ cvt_dtype(A.dtype),+ B.contiguous().data_ptr(),+ C.contiguous().data_ptr(),+ bias.contiguous().data_ptr(),+ True,+ scaleB.contiguous().data_ptr(),+ )+ return C++ # A: mxk:DT, B: BS:s8, bias: n:DT+ # return: C: mxn:DT+ def woqgemm(+ self,+ A: torch.Tensor,+ B: torch.Tensor,+ bias: torch.Tensor,+ n,+ k,+ groupsize,+ compute_type,+ weight_type,+ scale_type,+ asym,+ ):+ m = A.shape[0]+ lib = self.get_lib(A)+ ct = cvtstr_dtype(compute_type)+ wt = cvtstr_dtype(weight_type)+ st = cvtstr_dtype(scale_type)+ C = torch.zeros(m, n, dtype=A.dtype, device=A.device)+ stream = get_stream(A)+ lib.woqgemm(+ stream,+ m,+ n,+ k,+ A.contiguous().data_ptr(),+ cvt_dtype(A.dtype),+ B.contiguous().data_ptr(),+ C.contiguous().data_ptr(),+ bias.contiguous().data_ptr(),+ groupsize,+ ct,+ wt,+ st,+ asym,+ )+ return C++ # QB: k*n:int8, scaleB: k/blocksize*n:DT+ # return: blob:BS:int8+ def repack_quantized_weight(+ self,+ QB: torch.Tensor,+ scaleB: torch.Tensor,+ zp: torch.Tensor,+ groupsize,+ compute_type,+ weight_type,+ scale_type,+ asym,+ ):+ k = QB.shape[0]+ n = QB.shape[1]+ lib = self.get_lib(QB)+ stream = get_stream(QB)+ ct = cvtstr_dtype(compute_type)+ wt = cvtstr_dtype(weight_type)+ st = cvtstr_dtype(scale_type)+ BS = lib.packed_weight_size(stream, n, k, groupsize, ct, wt, st, asym)+ blob = torch.zeros(BS, dtype=torch.int8, device=QB.device)+ lib.repack_quantized_weight(+ stream,+ QB.contiguous().data_ptr(),+ zp.contiguous().data_ptr(),+ scaleB.contiguous().data_ptr(),+ blob.data_ptr(),+ n,+ k,+ groupsize,+ ct,+ wt,+ st,+ asym,+ )+ return blob++ # QB: blob:BS:int8+ # return: out:nxk:out_dtype+ def unpack_weight(+ self,+ blob: torch.Tensor,+ out_dtype: torch.dtype,+ n,+ k,+ groupsize,+ compute_type,+ weight_type,+ scale_type,+ asym,+ ):+ lib = self.get_lib(blob)+ stream = get_stream(blob)+ ct = cvtstr_dtype(compute_type)+ wt = cvtstr_dtype(weight_type)++ st = cvtstr_dtype(scale_type)+ oshape = (n, k) if blob.device.type == "xpu" else (k, n)+ out = torch.zeros(oshape, dtype=out_dtype, device=blob.device)+ lib.unpack_weight(+ stream,+ blob.data_ptr(),+ out.data_ptr(),+ cvt_dtype(out_dtype),+ n,+ k,+ groupsize,+ ct,+ wt,+ st,+ asym,+ )+ if blob.device.type == "cpu":+ return out.T+ return out++ def sdpa(+ self,+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attn_mask: torch.Tensor | None = None,+ dropout_p: float = 0.0,+ is_causal: bool = False,+ scale: float | None = None,+ ) -> torch.Tensor:+ """Scaled dot-product attention (SDPA) prefill+decode.++ Expects contiguous layouts:+ - query: [B, Hq, Sq, D]+ - key: [B, Hkv, Skv, D]+ - value: [B, Hkv, Skv, D]++ Args:+ - scale: Softmax scale. Uses 1 / sqrt(D) when None.++ Returns:+ - O: [B, Hq, Sq, D] (same dtype as value)+ """+ if query.device.type != "xpu":+ raise NotImplementedError("sdpa is only supported on XPU")++ if query.dtype not in (torch.float16, torch.bfloat16):+ raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")+ if key.dtype != query.dtype or value.dtype != query.dtype:+ raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")++ if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:+ raise ValueError("Q/K/V must be 4D tensors")++ if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():+ raise ValueError("Q/K/V must be contiguous")++ B, Hq, Sq, D = query.shape+ Bk, Hkv, Skv, Dk = key.shape+ Bv, Hkv2, Skv2, Dv = value.shape++ if Bk != B or Bv != B:+ raise ValueError("Batch size mismatch between Q/K/V")+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:+ raise ValueError("K/V shape mismatch")+ if Dk != D:+ raise ValueError("Head dim mismatch between Q and K/V")+ if D not in (64, 128, 96, 192):+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128, 96, 192")++ if dropout_p != 0.0:+ raise NotImplementedError(f"dropout_p must be 0.0 (got {dropout_p}); dropout is not supported")++ if attn_mask is not None:+ if attn_mask.device.type != "xpu":+ raise ValueError("attn_mask must be on XPU")+ if not attn_mask.is_contiguous():+ raise ValueError("attn_mask must be contiguous")+ if attn_mask.dtype != torch.float32:+ raise ValueError(f"attn_mask must be float32 (additive bias), got {attn_mask.dtype}")+ expected_mask_shape = (B, 1, Sq, Skv)+ if attn_mask.shape != expected_mask_shape:+ raise ValueError(f"attn_mask shape must be {expected_mask_shape}, got {tuple(attn_mask.shape)}")++ lib = self.get_lib(query)+ stream = get_stream(query)+ O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=value.dtype)+ lib.sdpa(+ stream,+ query.data_ptr(),+ key.data_ptr(),+ value.data_ptr(),+ O.data_ptr(),+ attn_mask.data_ptr() if attn_mask is not None else 0,+ cvt_dtype(query.dtype),+ cvt_dtype(key.dtype),+ cvt_dtype(O.dtype),+ B,+ Hq,+ Hkv,+ Sq,+ Skv,+ D,+ float(scale) if scale is not None else 1.0 / (D**0.5),+ bool(is_causal),+ )+ return O++ def sage(+ self,+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attn_mask: torch.Tensor | None = None,+ dropout_p: float = 0.0,+ is_causal: bool = False,+ scale: float | None = None,+ enable_gqa: bool = False,+ quant_block_size: int = 64,+ qscale: torch.Tensor = None,+ kscale: torch.Tensor = None,+ ) -> torch.Tensor:+ """SAGE attention prefill+decode.++ Expects contiguous layouts:+ - query: [B, Hq, Sq, D]+ - key: [B, Hkv, Skv, D]+ - value: [B, Hkv, Skv, D]++ Args:+ - scale: Attention scale. Uses 1.0 when None.+ - scale_block_size: Block size for qscale and kscale.++ Returns:+ - O: [B, Hq, Sq, D] (same dtype as value)+ """+ if query.device.type != "xpu":+ raise NotImplementedError("sdpa is only supported on XPU")++ # if query.dtype not in (torch.float16, torch.bfloat16):+ # raise ValueError(f"Q must be float16 or bfloat16, got {query.dtype}")+ # if key.dtype != query.dtype or value.dtype != query.dtype:+ # raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")++ if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:+ raise ValueError("Q/K/V must be 4D tensors")++ if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():+ raise ValueError("Q/K/V must be contiguous")++ B, Hq, Sq, D = query.shape+ Bk, Hkv, Skv, Dk = key.shape+ Bv, Hkv2, Skv2, Dv = value.shape++ if Bk != B or Bv != B:+ raise ValueError("Batch size mismatch between Q/K/V")+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:+ raise ValueError("K/V shape mismatch")+ if Dk != D:+ raise ValueError("Head dim mismatch between Q and K/V")+ if D not in (64, 128):+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")++ lib = self.get_lib(query)+ stream = get_stream(query)+ O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=value.dtype)+ lib.sage(+ stream,+ query.data_ptr(),+ key.data_ptr(),+ value.data_ptr(),+ O.data_ptr(),+ attn_mask.data_ptr() if attn_mask is not None else 0,+ quant_block_size,+ qscale.data_ptr() if qscale is not None else 0,+ kscale.data_ptr() if kscale is not None else 0,+ cvt_dtype(query.dtype),+ cvt_dtype(key.dtype),+ cvt_dtype(O.dtype),+ B,+ Hq,+ Hkv,+ Sq,+ Skv,+ D,+ float(scale) if scale is not None else 1.0,+ bool(is_causal),+ )+ return O++ def sagev1(+ self,+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attn_mask: torch.Tensor | None = None,+ dropout_p: float = 0.0,+ is_causal: bool = False,+ scale: float | None = None,+ enable_gqa: bool = False,+ quant_block_size: int = 64,+ ) -> torch.Tensor:+ """SAGE v1 attention prefill+decode.++ Expects contiguous layouts:+ - query: [B, Hq, Sq, D]+ - key: [B, Hkv, Skv, D]+ - value: [B, Hkv, Skv, D]++ Args:+ - scale: Attention scale. Uses 1.0 when None.+ - quant_block_size: Quantization block size used by the kernel.++ Returns:+ - O: [B, Hq, Sq, D] (same dtype as value)+ """+ if quant_block_size <= 0:+ return self.sdpa(+ query=query,+ key=key,+ value=value,+ attn_mask=attn_mask,+ dropout_p=dropout_p,+ is_causal=is_causal,+ scale=scale,+ )+ if query.device.type != "xpu":+ raise NotImplementedError("sdpa is only supported on XPU")+ if query.dtype not in (torch.float16,):+ raise ValueError(f"Q must be float16, got {query.dtype}")+ if key.dtype != query.dtype or value.dtype != query.dtype:+ raise ValueError(f"K/V dtype must match Q dtype, got K={key.dtype}, V={value.dtype}, Q={query.dtype}")++ if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:+ raise ValueError("Q/K/V must be 4D tensors")++ if not query.is_contiguous() or not key.is_contiguous() or not value.is_contiguous():+ raise ValueError("Q/K/V must be contiguous")++ B, Hq, Sq, D = query.shape+ Bk, Hkv, Skv, Dk = key.shape+ Bv, Hkv2, Skv2, Dv = value.shape++ if Bk != B or Bv != B:+ raise ValueError("Batch size mismatch between Q/K/V")+ if Hkv2 != Hkv or Skv2 != Skv or Dv != Dk:+ raise ValueError("K/V shape mismatch")+ if Dk != D:+ raise ValueError("Head dim mismatch between Q and K/V")+ if D not in (64, 128):+ raise ValueError(f"Unsupported head_dim={D}; supported: 64, 128")++ lib = self.get_lib(query)+ stream = get_stream(query)+ O = torch.empty((B, Hq, Sq, D), device=query.device, dtype=value.dtype)+ lib.sagev1(+ stream,+ query.data_ptr(),+ key.data_ptr(),+ value.data_ptr(),+ O.data_ptr(),+ attn_mask.data_ptr() if attn_mask is not None else 0,+ quant_block_size,+ cvt_dtype(query.dtype),+ cvt_dtype(key.dtype),+ cvt_dtype(value.dtype),+ cvt_dtype(O.dtype),+ B,+ Hq,+ Hkv,+ Sq,+ Skv,+ D,+ float(scale) if scale is not None else 1.0,+ bool(is_causal),+ )+ return O++ def sage_dynquant(+ self,+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ attn_mask: torch.Tensor | None = None,+ dropout_p: float = 0.0,+ is_causal: bool = False,+ scale: float | None = None,+ enable_gqa: bool = False,+ quant_block_size: int = 64,+ ) -> torch.Tensor:+ """SAGE Attention with dynamic INT8 block-wise quantization of Q/K.++ Takes FP16 Q, K, V inputs. Quantizes Q and K to INT8 per-block+ using a fused SYCL kernel, then calls SAGE V1 with INT8 data.+ API is like SDPA but with an extra quant_block_size parameter.++ Args:+ query: [B, Hq, Sq, D] float16+ key: [B, Hkv, Skv, D] float16+ value: [B, Hkv, Skv, D] float16+ quant_block_size: Number of tokens sharing one INT8 scale.+ E.g. 64 means 64 consecutive tokens share one absmax.+ 0 means per-token (block_size=1).++ Returns:+ O: [B, Hq, Sq, D] float16+ """+ if query.device.type != "xpu":+ raise NotImplementedError("sage_dynquant is only supported on XPU")++ if query.dtype not in (torch.float16,):+ raise ValueError(f"Q must be float16, got {query.dtype}")++ B, Hq, Sq, D = query.shape+ _, Hkv, Skv, _ = key.shape++ # block_size=0 means per-token+ block_size = quant_block_size if quant_block_size > 0 else 1++ # SAGE V1 kernel uses K-tile size=32; scale_block_size must be 1 or >=32+ if block_size != 1 and block_size < 32:+ raise ValueError(+ f"quant_block_size={block_size} is not supported. "+ f"Must be 1 (per-token) or >= 32 (e.g. 32, 64, 128, 256)."+ )++ lib = self.get_lib(query)+ stream = get_stream(query)++ # Auto-pad Q and K/V seq lengths to be divisible by block_size+ # so sage_dynquant works as a drop-in replacement for SDPA+ def _ceil_div(a, b):+ return (a + b - 1) // b++ Sq_pad = _ceil_div(Sq, block_size) * block_size+ Skv_pad = _ceil_div(Skv, block_size) * block_size+ need_pad_q = Sq_pad != Sq+ need_pad_kv = Skv_pad != Skv++ if need_pad_q:+ pad_q = Sq_pad - Sq+ query = torch.nn.functional.pad(query, (0, 0, 0, pad_q)) # pad S dim with zeros+ if need_pad_kv:+ pad_kv = Skv_pad - Skv+ key = torch.nn.functional.pad(key, (0, 0, 0, pad_kv))+ value = torch.nn.functional.pad(value, (0, 0, 0, pad_kv))++ # Fused block-wise quantization via SYCL kernel+ # Tensor layout: [B, H, S, D] is contiguous → [B*H*S, D] flattened+ # block_size tokens share one scale → num_blocks = B*H*S / block_size+ # For Q: num_rows = B*Hq*Sq_pad, scale shape = [B, Hq, Sq_pad/block_size, 1]+ q_num_rows = B * Hq * Sq_pad+ q_num_blocks = q_num_rows // block_size+ q_i8 = torch.empty_like(query, dtype=torch.int8)+ q_scale = torch.empty(q_num_blocks, dtype=torch.float32, device=query.device)+ lib.sage_dynamic_quant(+ stream,+ query.data_ptr(),+ q_i8.data_ptr(),+ q_scale.data_ptr(),+ q_num_rows,+ D,+ block_size,+ )+ q_scale = q_scale.reshape(B, Hq, Sq_pad // block_size, 1)++ k_num_rows = B * Hkv * Skv_pad+ k_num_blocks = k_num_rows // block_size+ k_i8 = torch.empty_like(key, dtype=torch.int8)+ k_scale = torch.empty(k_num_blocks, dtype=torch.float32, device=key.device)+ lib.sage_dynamic_quant(+ stream,+ key.data_ptr(),+ k_i8.data_ptr(),+ k_scale.data_ptr(),+ k_num_rows,+ D,+ block_size,+ )+ k_scale = k_scale.reshape(B, Hkv, Skv_pad // block_size, 1)++ # Call SAGE v1 with matching scale_block_size+ out = self.sage(+ q_i8,+ k_i8,+ value,+ attn_mask=attn_mask,+ is_causal=is_causal,+ scale=scale,+ enable_gqa=enable_gqa,+ scale_block_size=block_size,+ qscale=q_scale,+ kscale=k_scale,+ )++ # Slice back to original seq length if padded+ if need_pad_q:+ out = out[:, :, :Sq, :]+ return out++ def moe_gemm(+ self,+ activations: torch.Tensor,+ weights: torch.Tensor,+ num_tokens_per_expert: torch.Tensor,+ *,+ scales: Optional[torch.Tensor] = None,+ ) -> torch.Tensor:+ """MOE GEMM (Mixture of Experts Grouped GEMM).++ Computes grouped GEMM for MOE layers where different experts process+ different numbers of tokens.++ Expects contiguous layouts:+ - activations: [total_tokens, K] (BF16/FP16)+ - weights: [num_experts, K, N] (BF16/FP16, Row major)+ - num_tokens_per_expert: [num_experts] (int32)+ - scales (optional): [num_experts, N] or None++ Returns:+ - outputs: [total_tokens, N] (same dtype as activations)+ """+ if activations.device.type != "xpu":+ raise NotImplementedError("moe_gemm is only supported on XPU")++ if activations.dtype not in (torch.float16, torch.bfloat16):+ raise ValueError(f"activations must be fp16/bf16, got {activations.dtype}")+ if weights.dtype != activations.dtype:+ raise ValueError("weights dtype must match activations dtype")++ if activations.ndim != 2 or weights.ndim != 3:+ raise ValueError("activations must be 2D [total_tokens, K], weights must be 3D [num_experts, K, N]")++ if not activations.is_contiguous() or not weights.is_contiguous():+ raise ValueError("activations and weights must be contiguous")++ if num_tokens_per_expert.dtype != torch.int32:+ num_tokens_per_expert = num_tokens_per_expert.to(torch.int32)++ if not num_tokens_per_expert.is_contiguous():+ num_tokens_per_expert = num_tokens_per_expert.contiguous()++ total_tokens, K = activations.shape+ num_experts, K_w, N = weights.shape # weights are [num_experts, K, N]++ if K != K_w:+ raise ValueError(f"K dimension mismatch: activations K={K}, weights K={K_w}")++ if num_tokens_per_expert.shape[0] != num_experts:+ raise ValueError(+ f"num_tokens_per_expert length {num_tokens_per_expert.shape[0]} != num_experts {num_experts}"+ )++ # Validate total tokens+ expected_total = int(num_tokens_per_expert.sum().item())+ if expected_total != total_tokens:+ raise ValueError(f"Sum of num_tokens_per_expert ({expected_total}) != total_tokens ({total_tokens})")++ lib = self.get_lib(activations)+ stream = get_stream(activations)+ outputs = torch.empty((total_tokens, N), device=activations.device, dtype=activations.dtype)++ scales_ptr = scales.data_ptr() if scales is not None else 0++ lib.moe_gemm(+ stream,+ activations.data_ptr(),+ weights.data_ptr(),+ scales_ptr,+ outputs.data_ptr(),+ cvt_dtype(activations.dtype),+ N,+ K,+ num_tokens_per_expert.data_ptr(),+ num_experts,+ )+ return outputs+++if __name__ == "__main__":+ ark = ARK()+ print(ark.cpu_lib is None, ark.xpu_lib is None)++ def matmul():+ m = n = k = 128+ dt = torch.int8+ device = "cpu"+ has_bias = False+ if dt == torch.int8:+ A = torch.randint(-128, 127, (m, k), dtype=dt, device=device)+ B = torch.randint(-128, 127, (n, k), dtype=dt, device=device)+ C = ark.igemm_s8s8s32(A, B)+ print(C)+ else:+ A = torch.rand(m, k, dtype=dt, device=device) - 0.5+ B = torch.rand(k, n, dtype=dt, device=device) - 0.5+ bias = torch.rand(1, n, dtype=dt, device=device) if has_bias else torch.empty(0)+ C = ark.matmul(A, B, bias)+ ref = torch.matmul(A, B.T)+ if has_bias:+ ref = ref + bias+ dff = abs(C - ref)+ if dt != torch.int8:+ print(dff.max(), dff.mean())+ print(torch.allclose(ref, C, 0.01, 0.1))++ def woq():+ m = n = k = 128+ dt = torch.float32+ device = "cpu"+ A = torch.rand(m, k, dtype=dt, device=device) - 0.5+ bias = torch.rand(1, n, dtype=dt, device=device) + 1000+ B = torch.randint(-128, 127, (n, k), dtype=torch.int8, device=device)+ scaleB = torch.rand(n, 1, dtype=dt, device=device)+ C = ark.woqgemm_s8(A, B, scaleB, bias)+ print(C)+ DB = B.to(dt) * scaleB+ ref = torch.matmul(A, DB.T) + bias+ print(ref)+ dff = abs(C - ref)+ print(dff.max(), dff.mean())++ def pack_unpack():+ m = n = k = 128+ groupsize = 32+ dt = torch.float32+ device = "xpu"+ B = torch.randint(-8, 7, (k, n), dtype=torch.int8, device=device)+ zp = torch.randint(-8, 7, (k // groupsize, n), dtype=torch.int8, device=device)+ scaleB = torch.rand(k // groupsize, n, dtype=dt, device=device) / 100+ blob = ark.repack_quantized_weight(B, scaleB, zp, groupsize, "fp32", "int4", "fp32", False)+ dq = ark.unpack_weight(blob, dt, n, k, groupsize, "fp32", "int4", "fp32", False)+ print(blob, dq)+ scale_re = scaleB.repeat_interleave(repeats=groupsize, dim=0).to(dt)++ DB = B.to(dt) * scale_re+ dff = abs(DB.T - dq)+ print(dff.max(), dff.mean())++ pack_unpack()+++__all__ = ["ARK"]+++# -----------------------------------------------------------------------------+# Compatibility layer+#+# Some callers (e.g. auto_round_extension/ark/qlinear.py) historically imported+# this package as a module and expected certain functions to exist at the module+# level (e.g. ark.repack_quantized_weight, ark.woq_linear).+#+# The current implementation exposes these as methods on the singleton ARK()+# instance. The wrappers below keep backward compatibility without changing the+# compiled extension.+# -----------------------------------------------------------------------------+++def _ark_instance():+ return ARK()+++def check_isa_supported(_isa: str) -> bool:+ # Best-effort: some builds expose ISA checks via native libs; keep safe.+ # Returning False is conservative and avoids misconfiguration.+ return False+++def repack_quantized_weight(*args, **kwargs):+ """Repack quantized weights into ARK/BestLA packed format.++ Supports two call styles:++ 1) New style (recommended):+ repack_quantized_weight(QB, scaleB, zp, groupsize, compute_type, weight_type, scale_type, asym)++ 2) Legacy style used by qlinear.py:+ repack_quantized_weight(QB, scaleB, zp, g_idx, compute_type, weight_type, scale_type, asym, groupsize)+ repack_quantized_weight(QB, scaleB, zp, g_idx, weight_type, compute_type, scale_type, asym, groupsize)+ (g_idx is ignored)+ """++ if kwargs:+ return _ark_instance().repack_quantized_weight(**kwargs)++ if len(args) == 8:+ QB, scaleB, zp, groupsize, compute_type, weight_type, scale_type, asym = args+ elif len(args) == 9:+ QB, scaleB, zp, _g_idx, a4, a5, scale_type, asym, groupsize = args+ # Legacy call sites sometimes swap compute_type/weight_type.+ compute_types = {"fp16", "bf16", "fp32", "fp8_e4m3", "fp8_e5m2", "fp8_e8m0"}+ if isinstance(a4, str) and a4 in compute_types:+ compute_type, weight_type = a4, a5+ else:+ weight_type, compute_type = a4, a5+ else:+ raise TypeError("repack_quantized_weight() expects 8 or 9 positional arguments; " f"got {len(args)}")++ # Some native paths may still expect a valid zp pointer even when asym=False.+ if (zp is None) or (isinstance(zp, torch.Tensor) and zp.numel() == 0):+ if not bool(asym):+ k = QB.shape[0]+ n = QB.shape[1]+ zp = torch.zeros((k // int(groupsize), n), dtype=torch.int8, device=QB.device)+ else:+ zp = torch.empty(0, dtype=torch.int8, device=QB.device)++ return _ark_instance().repack_quantized_weight(+ QB,+ scaleB,+ zp,+ groupsize,+ compute_type,+ weight_type,+ scale_type,+ asym,+ )+++def unpack_weight(+ blob: torch.Tensor, out_dtype: torch.dtype, n, k, groupsize, compute_type, weight_type, scale_type, asym+):+ return _ark_instance().unpack_weight(blob, out_dtype, n, k, groupsize, compute_type, weight_type, scale_type, asym)+++def packed_weight_size(A: torch.Tensor, n, k, groupsize, compute_type, weight_type, scale_type, asym):+ # Keep signature convenient for Python callers; native library needs a stream.+ lib = _ark_instance().get_lib(A)+ stream = get_stream(A)+ ct = cvtstr_dtype(compute_type)+ wt = cvtstr_dtype(weight_type)+ st = cvtstr_dtype(scale_type)+ return lib.packed_weight_size(stream, n, k, groupsize, ct, wt, st, asym)+++def woq_linear(+ A: torch.Tensor,+ packed_B: torch.Tensor,+ bias: torch.Tensor,+ out: torch.Tensor,+ compute_type,+ weight_type,+ scale_type,+ asym,+ groupsize=None,+):+ """Linear helper that writes into a preallocated output tensor."""++ if groupsize is None:+ groupsize = A.shape[-1]++ result = _ark_instance().woqgemm(+ A,+ packed_B,+ bias,+ out.shape[-1],+ A.shape[-1],+ int(groupsize),+ compute_type,+ weight_type,+ scale_type,+ bool(asym),+ )+ out.copy_(result)+ return outdiff --git a/auto_round_extension/ark/auto_round_kernel/ark.cpp b/auto_round_extension/ark/auto_round_kernel/ark.cppnew file mode 100755index 000000000..cd5c1ab03--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/ark.cpp@@ -0,0 +1,283 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include <pybind11/pybind11.h>++#include <iostream>+#include <stdexcept>+#include "bestla/bestla/bestla.h"+typedef uintptr_t torch_ptr;+#if ARK_XPU+#include <sycl/sycl.hpp>+#include "xpu_wrapper.hpp"+#if ARK_SYCL_TLA+// Only include declarations, implementations are in separate .cpp files+#include "sycl_tla_common.hpp"+#include "sycl_tla_moe.hpp"+#include "sycl_tla_sdpa.hpp"+#endif+#else+#include "cpu_wrapper.hpp"+#endif++#include "dnnl_wrapper.hpp"++namespace ark {++static void matmul(torch_ptr stream, int m, int n, int k, torch_ptr A, int Adt, torch_ptr B, int Bdt, torch_ptr C,+ int Cdt, torch_ptr bias, bool BT) {+ auto dt = ark::to_dt((BTLA_DTYPE)Adt);+ auto cdt = dt;+ if (Adt == (int)BTLA_DTYPE::S8) cdt = dnnl::memory::data_type::s32;+#ifdef ARK_XPU+ ark::DnnlWrapper::gemm((sycl::queue*)stream, m, n, k, (void*)A, dt, (void*)B, dt, BT, (void*)C, cdt, (void*)bias);+#else+ CpuWrapper::gemm(m, n, k, (void*)A, (BTLA_DTYPE)Adt, (void*)B, BT, (float*)C, (const float*)bias);+#endif+}++static void woqgemm_s8(torch_ptr stream, int m, int n, int k, torch_ptr A, int ACdt, torch_ptr B, torch_ptr C,+ torch_ptr bias, bool BT, torch_ptr scaleb) {+ auto dt = ark::to_dt((BTLA_DTYPE)ACdt);+ ark::DnnlWrapper::woq_s8((sycl::queue*)stream, m, n, k, (void*)A, (void*)B, BT, (void*)C, dt, (void*)scaleb,+ (void*)bias, k);+}++static void woqgemm(torch_ptr stream, int m, int n, int k, torch_ptr A, int ACdt, torch_ptr BlobB, torch_ptr C,+ torch_ptr bias, int blocksize, int compute_type, int weight_type, int scale_type, bool asym) {+ QuantParam param{n, k, blocksize, compute_type, weight_type, scale_type, asym};+#ifdef ARK_XPU+ XpuWrapper::woq_gemm(m, (void*)A, (void*)BlobB, (void*)C, (void*)bias, (BTLA_DTYPE)ACdt, ¶m,+ (sycl::queue*)stream);+#else+ CpuWrapper::woq_gemm(m, (void*)A, (void*)BlobB, (void*)C, (void*)bias, (BTLA_DTYPE)ACdt, ¶m);+#endif+}++static void repack_quantized_weight(torch_ptr stream, torch_ptr raws8, torch_ptr zp, torch_ptr scale, torch_ptr blob,+ int n, int k, int blocksize, int compute_type, int weight_type, int scale_type,+ bool asym) {+ QuantParam param{n, k, blocksize, compute_type, weight_type, scale_type, asym};+#ifdef ARK_XPU+ XpuWrapper::packq((int8_t*)raws8, (void*)scale, (int8_t*)zp, (int8_t*)blob, ¶m, (sycl::queue*)stream);+#else+ CpuWrapper::packq((int8_t*)raws8, (float*)scale, (int8_t*)zp, (int8_t*)blob, ¶m);+#endif+}++static void unpack_weight(torch_ptr stream, torch_ptr blob, torch_ptr output, int out_type, int n, int k, int blocksize,+ int compute_type, int weight_type, int scale_type, bool asym) {+ QuantParam param{n, k, blocksize, compute_type, weight_type, scale_type, asym};+#ifdef ARK_XPU+ XpuWrapper::unpackq((BTLA_DTYPE)out_type, (int8_t*)blob, (void*)output, ¶m, (sycl::queue*)stream);+#else+ CpuWrapper::unpackq((BTLA_DTYPE)out_type, (int8_t*)blob, (void*)output, ¶m);+#endif+}++static size_t packed_weight_size(torch_ptr stream, int n, int k, int blocksize, int compute_type, int weight_type,+ int scale_type, bool asym) {+ QuantParam param{n, k, blocksize, compute_type, weight_type, scale_type, asym};+#ifdef ARK_XPU+ return XpuWrapper::get_packw_size(¶m);+#else+ return CpuWrapper::get_packw_size(¶m);+#endif+}++#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)++static void sdpa(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask, int q_dtype,+ int k_dtype, int o_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,+ int head_dim, float softmax_scale, bool is_causal) {+ if (k_dtype != q_dtype || o_dtype != q_dtype) {+ throw std::invalid_argument("ark::sdpa: k_dtype and o_dtype must match q_dtype");+ }+ if(q_dtype != (int)BTLA_DTYPE::F16 && q_dtype != (int)BTLA_DTYPE::BF16) {+ throw std::invalid_argument("ark::sdpa: only FP16 and BF16 are supported");+ }+ if (mask && is_causal) {+ throw std::invalid_argument("ark::sdpa: mask and is_causal cannot both be set");+ }+ ark::sdpa_impl((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, (BTLA_DTYPE)(q_dtype),+ batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+}++static void sagev1(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,+ int scale_block_size, int q_dtype, int k_dtype, int v_dtype, int o_dtype, int batch, int num_heads_q,+ int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {+ if (mask && is_causal) {+ throw std::invalid_argument("ark::sagev1: mask and is_causal cannot both be set");+ }+ if (q_dtype != k_dtype || o_dtype != q_dtype || v_dtype != q_dtype) {+ throw std::invalid_argument("ark::sagev1: k_dtype and o_dtype must match q_dtype");+ }+ if (q_dtype != (int)BTLA_DTYPE::F16) {+ throw std::invalid_argument("ark::sagev1: only F16 is supported for q_dtype");+ }+#ifdef ARK_XPU+ XpuWrapper::sagev1((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask, scale_block_size, batch,+ num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+#else+ throw std::runtime_error("ark::sagev1 is only supported on XPU");+#endif+}++static void sage(torch_ptr stream, torch_ptr Q, torch_ptr K, torch_ptr V, torch_ptr O, torch_ptr mask,+ int scale_block_size, torch_ptr qscale, torch_ptr kscale, int q_dtype, int k_dtype, int o_dtype,+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,+ float softmax_scale, bool is_causal) {+ if (mask && is_causal) {+ throw std::invalid_argument("ark::sagev1: mask and is_causal cannot both be set");+ }+ ark::sdpa_impl_qks8_pvhalf((sycl::queue*)stream, (void*)Q, (void*)K, (void*)V, (void*)O, (void*)mask,+ scale_block_size, (void*)qscale, (void*)kscale, batch, num_heads_q, num_heads_kv,+ seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+}++static void moe_gemm_wrapper(torch_ptr stream, torch_ptr activations, torch_ptr weights, torch_ptr scales,+ torch_ptr outputs, int dtype, int N, int K, torch_ptr num_tokens_per_expert,+ int num_experts) {+ ark::moe_gemm((sycl::queue*)stream, (void*)activations, (void*)weights, scales ? (void*)scales : nullptr,+ (void*)outputs, (BTLA_DTYPE)(dtype), N, K, (int*)num_tokens_per_expert, num_experts);+}++static void sage_dynamic_quant(torch_ptr stream, torch_ptr input, torch_ptr output, torch_ptr scale_out, int num_rows,+ int head_dim, int block_size) {+ auto* q = (sycl::queue*)stream;+ auto* in_ptr = (sycl::half*)input;+ auto* out_ptr = (int8_t*)output;+ auto* scale_ptr = (float*)scale_out;++ int num_blocks = num_rows / block_size;+ int elems_per_block = block_size * head_dim;++ // Work-group size: use 256 threads (16 sub-groups × 16 lanes).+ // For block_size=1, head_dim=128: 128 elements / 256 threads < 1, so we use smaller WG.+ // For block_size=64, head_dim=128: 8192 elements / 256 threads = 32 elements per thread.+ constexpr int SG_SIZE = 32;+ constexpr int MAX_Reg = 64;+ constexpr int MAX_WG_SIZE = 512;+ constexpr int Unroll = 8;+ if (elems_per_block > MAX_Reg * MAX_WG_SIZE) {+ int wg_size = (elems_per_block <= 256) ? SG_SIZE : 256;+ // Ensure wg_size is a multiple of SG_SIZE+ wg_size = ((wg_size + SG_SIZE - 1) / SG_SIZE) * SG_SIZE;++ q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ int block_id = item.get_group(0);+ int tid = item.get_local_id(0);+ auto wg = item.get_group();+ auto* block_in = in_ptr + (size_t)block_id * elems_per_block;+ auto* block_out = out_ptr + (size_t)block_id * elems_per_block;++ // Phase 1: compute absmax across entire block+ float local_max = 0.0f;+ sycl::vec<sycl::half, Unroll> local_data, local_max_vec;+ local_max_vec = sycl::vec<sycl::half, Unroll>(0.0f);+ for (int i = tid * Unroll; i < elems_per_block; i += wg_size * Unroll) {+ local_data = *(sycl::vec<sycl::half, Unroll>*)(&block_in[i]);+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data));+ }+ for (int i = 0; i < Unroll; ++i) {+ local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));+ }+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});++ // Compute scale+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;++ // Store scale (one thread writes)+ if (tid == 0) {+ scale_ptr[block_id] = absmax / 127.0f;+ }++ // Phase 2: fused quantize+ for (int i = tid * Unroll; i < elems_per_block; i += wg_size * Unroll) {+#pragma unroll+ for (int j = 0; j < Unroll; ++j) {+ float val = static_cast<float>(block_in[i + j]) * inv_scale;+ int iv = static_cast<int>(val + (val >= 0.0f ? 0.5f : -0.5f));+ iv = sycl::clamp(iv, -127, 127);+ block_out[i + j] = static_cast<int8_t>(iv);+ }+ }+ });+ } else {+ int wg_size = MAX_WG_SIZE;+ q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ int block_id = item.get_group(0);+ int tid = item.get_local_id(0);+ auto wg = item.get_group();+ auto* block_in = in_ptr + (size_t)block_id * elems_per_block;+ auto* block_out = out_ptr + (size_t)block_id * elems_per_block;++ // Phase 1: compute absmax across entire block+ float local_max = 0.0f;+ sycl::vec<sycl::half, Unroll> local_data[MAX_Reg / Unroll], local_max_vec;+ local_max_vec = sycl::vec<sycl::half, Unroll>(0.0f);+ int local_i = 0;+ for (int i = tid * Unroll; i < elems_per_block; i += wg_size * Unroll, local_i++) {+ local_data[local_i] = *(sycl::vec<sycl::half, Unroll>*)&block_in[i];+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));+ }+#pragma unroll+ for (int i = 0; i < Unroll; ++i) {+ local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));+ }+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});++ // Compute scale+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;++ // Store scale (one thread writes)+ if (tid == 0) {+ scale_ptr[block_id] = absmax / 127.0f;+ }++ // Phase 2: fused quantize+ local_i = 0;+ for (int i = tid * Unroll; i < elems_per_block; i += wg_size * Unroll, local_i++) {+ sycl::vec<float, Unroll> val =+ local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();+ val = val * inv_scale;+ val = sycl::round(val);+ val = sycl::clamp(val, -127, 127);+ sycl::vec<int8_t, Unroll> qv = val.template convert<int8_t, sycl::rounding_mode::automatic>();+ *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;+ }+ });+ }+}+#endif // ARK_XPU && ARK_SYCL_TLA++} // namespace ark++PYBIND11_MODULE(PY_NAME, m) {+ m.def("matmul", &ark::matmul);+ m.def("woqgemm_s8", &ark::woqgemm_s8);+ m.def("woqgemm", &ark::woqgemm);+ m.def("packed_weight_size", &ark::packed_weight_size);+ m.def("repack_quantized_weight", &ark::repack_quantized_weight);+ m.def("unpack_weight", &ark::unpack_weight);+#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)+ m.def("sdpa", &ark::sdpa);+ m.def("sagev1", &ark::sagev1);+ m.def("sage", &ark::sage);+ m.def("sage_dynamic_quant", &ark::sage_dynamic_quant);+ m.def("moe_gemm", &ark::moe_gemm_wrapper);+#endif+}\ No newline at end of filediff --git a/auto_round_extension/ark/auto_round_kernel/bestla/.clang-format b/auto_round_extension/ark/auto_round_kernel/bestla/.clang-formatnew file mode 100644index 000000000..c4a0baf58--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/.clang-format@@ -0,0 +1,22 @@+---+# Defaults for all languages.+BasedOnStyle: Google++# Setting ColumnLimit to 0 so developer choices about where to break lines are maintained.+# Developers are responsible for adhering to the 120 character maximum.+ColumnLimit: 120+SortIncludes: false+DerivePointerAlignment: false+# Avoid adding spaces between tokens in GSL_SUPPRESS arguments.+# E.g., don't change "GSL_SUPPRESS(r.11)" to "GSL_SUPPRESS(r .11)".+WhitespaceSensitiveMacros: ["GSL_SUPPRESS"]++# if you want to customize when working locally see https://clang.llvm.org/docs/ClangFormatStyleOptions.html for options.+# See ReformatSource.ps1 for a script to update all source according to the current options in this file.+# e.g. customizations to use Allman bracing and more indenting.+# AccessModifierOffset: -2+# BreakBeforeBraces: Allman+# CompactNamespaces: false+# IndentCaseLabels: true+# IndentWidth: 4+# NamespaceIndentation: Alldiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/CMakeLists.txt b/auto_round_extension/ark/auto_round_kernel/bestla/CMakeLists.txtnew file mode 100644index 000000000..dbf70c5e2--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/CMakeLists.txt@@ -0,0 +1,236 @@+cmake_minimum_required(VERSION 3.12)+project(bestla LANGUAGES CXX VERSION 0.1.0)++if(BTLA_SYCL)+ include(cmake/sycl.cmake)+endif()+include(cmake/FindSIMD.cmake)++file(GLOB headers ${PROJECT_NAME}/*.h ${PROJECT_NAME}/*.hpp)++option(BTLA_ENABLE_OPENMP "Compile OpenMP thread pool if OMP can be found" ON)+option(BTLA_SYCL "Compile OpenMP thread pool if OMP can be found" OFF)++option(BTLA_UT_ALL "Enable all unit tests" OFF)+option(BTLA_UT_DEBUG "Enable debug unit tests" OFF)+option(BTLA_UT_EPILOGUE "Enable unit test for epilogue" OFF)+option(BTLA_UT_PROLOGUE_A "Enable unit test for activation prologue" OFF)+option(BTLA_UT_PROLOGUE_B "Enable unit test for weight prologue" OFF)+option(BTLA_UT_GEMM "Enable unit test for micro gemm kernels" OFF)+option(BTLA_UT_WRAPPER "Enable unit test for parallel gemms" OFF)+option(BTLA_UT_PARALLEL "Enable unit test for parallel set" OFF)+option(BTLA_UT_KERNEL_JIT "Enable unit test for jit kernels" OFF)+option(BTLA_UT_KERNEL_INTRIN "Enable unit test for intrinsic kernels" OFF)+option(BTLA_UT_KERNEL_WRAPPER "Enable unit test for runtime ISA kernels" OFF)+option(BTLA_UT_NOASAN "Disable sanitize" OFF)+option(BTLA_UT_BENCHMARK "Benchmark ON may take a long time to finish all tests" OFF)+option(BTLA_UT_OPENMP "Use OpenMP for UT tests" ON)++include(FetchContent)+FetchContent_Declare(+ xbyak+ GIT_REPOSITORY https://github.com/herumi/xbyak.git+ GIT_TAG v7.06+)+FetchContent_MakeAvailable(xbyak)++add_library(${PROJECT_NAME} INTERFACE)+target_link_libraries(${PROJECT_NAME} INTERFACE xbyak)+add_library(neural_speed::${PROJECT_NAME} ALIAS ${PROJECT_NAME})+target_include_directories(+ ${PROJECT_NAME} INTERFACE+ "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>"+ "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"+)++function(add_isa_def ARG)+ if(${${ARG}_FOUND})+ target_compile_definitions(${PROJECT_NAME} INTERFACE BTLA_${ARG}_FOUND=1)+ else()+ target_compile_definitions(${PROJECT_NAME} INTERFACE BTLA_${ARG}_FOUND=0)+ endif()+endfunction()++foreach (ISA ${ISA_SET})+ add_isa_def(${ISA})+endforeach()++set(sycl_headers)+set(sycl_libs)+if(BTLA_SYCL)+ set(BTLA_UT_NOASAN ON) # hang issue with santizer+ file(GLOB sycl_headers ${PROJECT_NAME}/sycl/*.h ${PROJECT_NAME}/sycl/*.hpp)+ target_compile_definitions(${PROJECT_NAME} INTERFACE BTLA_SYCL)+ list(APPEND sycl_libs IntelSYCL::SYCL_CXX)+ target_compile_options(${PROJECT_NAME} INTERFACE -march=native)+ target_link_libraries(${PROJECT_NAME} INTERFACE ${sycl_libs})+ # add_link_options(-fsycl-targets=spir64 -Xsycl-target-backend "-options -ze-opt-large-register-file")+endif(BTLA_SYCL)++if(BTLA_ENABLE_OPENMP)+ include(FindOpenMP)+ message(STATUS "BesTLA enable OpenMP ThreadPool")+ target_compile_definitions(${PROJECT_NAME} INTERFACE BTLA_USE_OPENMP)+ target_link_libraries(${PROJECT_NAME} INTERFACE OpenMP::OpenMP_CXX)+endif(BTLA_ENABLE_OPENMP)++if(WIN32)+ target_compile_definitions(${PROJECT_NAME} INTERFACE _CRT_SECURE_NO_WARNINGS NOMINMAX)+endif(WIN32)++set(CMAKE_CXX_STANDARD 17)+set(CMAKE_CXX_STANDARD_REQUIRED ON)+target_compile_features(${PROJECT_NAME} INTERFACE cxx_std_17)++if(BTLA_UT_ALL)+ set(BTLA_UT_EPILOGUE ON)+ set(BTLA_UT_PROLOGUE_A ON)+ set(BTLA_UT_PROLOGUE_B ON)+ set(BTLA_UT_GEMM ON)+ set(BTLA_UT_WRAPPER ON)+ set(BTLA_UT_PARALLEL ON)+ set(BTLA_UT_KERNEL_JIT ON)+ set(BTLA_UT_KERNEL_INTRIN ON)+ set(BTLA_UT_KERNEL_WRAPPER ON)+endif(BTLA_UT_ALL)++set(UT_BUILD FALSE)+if(BTLA_UT_DEBUG OR BTLA_UT_PROLOGUE_A OR BTLA_UT_PROLOGUE_B OR BTLA_UT_EPILOGUE OR BTLA_UT_GEMM+OR BTLA_UT_WRAPPER OR BTLA_UT_PARALLEL OR BTLA_UT_KERNEL_JIT OR BTLA_UT_KERNEL_INTRIN+OR BTLA_UT_KERNEL_WRAPPER)+ set(UT_BUILD TRUE)+endif()++function(add_ut_flag UT_OPTION)+ if(${${UT_OPTION}})+ # target_compile_definitions(${PROJECT_NAME}_ut PRIVATE ${UT_OPTION})+ add_compile_definitions(${UT_OPTION})+ endif()+endfunction()++set(benchmark_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}/ut/bestla_benchmark.cpp)+list(APPEND benchmark_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}/ut/sycl_benchmark.cpp)+# Flash attention benchmarks are in separate files to avoid header conflicts+#list(APPEND benchmark_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}/ut/sycl_tla_flash_attn_prefill_bench.cpp)+#list(APPEND benchmark_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}/ut/sycl_tla_flash_attn_decode_bench.cpp)+# MOE GEMM benchmark+#list(APPEND benchmark_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}/ut/sycl_tla_moe_bench.cpp)+++if(UT_BUILD)+ file(GLOB srcs ${PROJECT_NAME}/ut/*.cc ${PROJECT_NAME}/ut/*.cpp) #compile everything even run parts of UTs+ file(GLOB sycl_srcs ${PROJECT_NAME}/ut/sycl*)+ if(NOT BTLA_SYCL)+ list(REMOVE_ITEM srcs ${sycl_srcs})+ endif()+ list(REMOVE_ITEM srcs ${benchmark_srcs})+ file(GLOB ut_headers ${PROJECT_NAME}/ut/*.h)+ include_directories(${PROJECT_NAME})+ add_executable(${PROJECT_NAME}_ut ${srcs} ${headers} ${sycl_headers} ${ut_headers})+ target_compile_options(${PROJECT_NAME}_ut PRIVATE -w)+ if(BTLA_UT_OPENMP)+ target_compile_definitions(${PROJECT_NAME} INTERFACE BTLA_USE_OPENMP)+ target_link_libraries(${PROJECT_NAME}_ut PRIVATE OpenMP::OpenMP_CXX)+ endif()+ if(NOT WIN32)+ if(NOT BTLA_UT_NOASAN)+ target_compile_options(${PROJECT_NAME}_ut PRIVATE -fsanitize=address)+ target_link_options(${PROJECT_NAME}_ut PRIVATE -fsanitize=address)+ endif()+ target_link_options(${PROJECT_NAME}_ut PRIVATE -lpthread)+ else()+ target_link_options(${PROJECT_NAME}_ut PUBLIC /STACK:5242880)+ endif()++ add_ut_flag(BTLA_UT_DEBUG)+ add_ut_flag(BTLA_UT_EPILOGUE)+ add_ut_flag(BTLA_UT_PROLOGUE_A)+ add_ut_flag(BTLA_UT_PROLOGUE_B)+ add_ut_flag(BTLA_UT_GEMM)+ add_ut_flag(BTLA_UT_PARALLEL)+ add_ut_flag(BTLA_UT_WRAPPER)+ add_ut_flag(BTLA_UT_KERNEL_INTRIN)+ add_ut_flag(BTLA_UT_KERNEL_JIT)+ add_ut_flag(BTLA_UT_KERNEL_WRAPPER)+ if(BTLA_SYCL)+ # add_compile_definitions(BTLA_UT_SYCL)+ endif()+ target_link_libraries(${PROJECT_NAME}_ut PRIVATE ${PROJECT_NAME} ${sycl_libs})+endif(UT_BUILD)++if(BTLA_UT_BENCHMARK)+ file(GLOB ut_headers ${PROJECT_NAME}/ut/*.h)+ include_directories(${PROJECT_NAME})+ if(NOT BTLA_SYCL)+ list(REMOVE_ITEM benchmark_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}/ut/sycl_benchmark.cpp)+ endif()+ add_executable(${PROJECT_NAME}_benchmark ${benchmark_srcs} ${headers} ${ut_headers})+ if(BTLA_UT_OPENMP)+ include(FindOpenMP)+ target_compile_definitions(${PROJECT_NAME} INTERFACE BTLA_USE_OPENMP)+ target_link_libraries(${PROJECT_NAME}_benchmark PRIVATE OpenMP::OpenMP_CXX)+ endif()+ if(NOT WIN32)+ target_link_options(${PROJECT_NAME}_benchmark PRIVATE -lpthread)+ else()+ target_link_options(${PROJECT_NAME}_benchmark PUBLIC /STACK:5242880)+ endif()+ target_link_libraries(${PROJECT_NAME}_benchmark PRIVATE ${PROJECT_NAME} ${sycl_libs} dnnl)+ target_compile_options(${PROJECT_NAME}_benchmark PRIVATE -w)+ # Add SYCL target for Intel GPU with XMX/2D block IO support (required for sycl-tla flash attention)+ if(BTLA_SYCL AND ARK_SYCL_TLA)+ # Header-only consumption of sycl-tla (do NOT build sycl-tla as a subproject).+ set(SYCL_TLA_GIT_REPOSITORY "https://github.com/intel/sycl-tla.git" CACHE STRING "sycl-tla git repository")+ set(SYCL_TLA_GIT_TAG "main" CACHE STRING "sycl-tla git tag/commit")++ FetchContent_Declare(+ sycl_tla+ GIT_REPOSITORY ${SYCL_TLA_GIT_REPOSITORY}+ GIT_TAG ${SYCL_TLA_GIT_TAG}+ )+ FetchContent_GetProperties(sycl_tla)+ if(NOT sycl_tla_POPULATED)+ FetchContent_Populate(sycl_tla)+ endif()++ set(_sycl_tla_include_dirs+ ${sycl_tla_SOURCE_DIR}/include+ ${sycl_tla_SOURCE_DIR}/applications+ ${sycl_tla_SOURCE_DIR}/tools/util/include+ ${sycl_tla_SOURCE_DIR}/examples/common+ ${sycl_tla_SOURCE_DIR}/examples/06_bmg_flash_attention+ ${sycl_tla_SOURCE_DIR}/examples/12_xe20_moe_gemm_cute_interface+ )+ foreach(_inc_dir IN LISTS _sycl_tla_include_dirs)+ if(EXISTS "${_inc_dir}")+ target_include_directories(${PROJECT_NAME}_benchmark PRIVATE "${_inc_dir}")+ endif()+ endforeach()++ # AOT compile target for Intel GPUs+ # Use intel_gpu_pvc for Data Center GPU Max series, or intel_gpu_bmg_g21 for Battlemage+ set(DPCPP_SYCL_TARGET "intel_gpu_bmg_g21" CACHE STRING "SYCL target (intel_gpu_pvc, intel_gpu_bmg_g21)")++ # Map target to device name for -Xs flag+ if(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_bmg_g21" OR DPCPP_SYCL_TARGET STREQUAL "bmg")+ set(SYCL_DEVICE_NAME "bmg_g21")+ elseif(DPCPP_SYCL_TARGET STREQUAL "intel_gpu_pvc" OR DPCPP_SYCL_TARGET STREQUAL "pvc")+ set(SYCL_DEVICE_NAME "pvc")+ else()+ set(SYCL_DEVICE_NAME "${DPCPP_SYCL_TARGET}")+ endif()++ target_compile_definitions(${PROJECT_NAME}_benchmark PRIVATE ARK_SYCL_TLA=1 CUTLASS_ENABLE_SYCL=1 SYCL_INTEL_TARGET=1)+ # Compile flags (no AOT, JIT at runtime)+ target_compile_options(${PROJECT_NAME}_benchmark PRIVATE+ -fsycl+ -fno-sycl-instrument-device-code)+ # Link flags: use spir64 (JIT) with device hint and enable required SPIR-V extensions+ target_link_options(${PROJECT_NAME}_benchmark PRIVATE+ -fsycl+ -fsycl-targets=spir64+ "-Xs" "-device ${SYCL_DEVICE_NAME}"+ -Xspirv-translator+ "-spirv-ext=+SPV_INTEL_split_barrier,+SPV_INTEL_2d_block_io,+SPV_INTEL_subgroup_matrix_multiply_accumulate")+ endif()+endif(BTLA_UT_BENCHMARK)diff --git a/auto_round_extension/ark/auto_round_kernel/bestla/CMakePresets.json b/auto_round_extension/ark/auto_round_kernel/bestla/CMakePresets.jsonnew file mode 100644index 000000000..7187120ff--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/CMakePresets.json@@ -0,0 +1,116 @@+{+ "version": 3,+ "configurePresets": [+ {+ "name": "linux-debug",+ "displayName": "Linux Debug",+ "description": "Target the Windows Subsystem for Linux (WSL) or a remote Linux system.",+ "generator": "Ninja",+ "binaryDir": "${sourceDir}/out/build/${presetName}",+ "installDir": "${sourceDir}/out/install/${presetName}",+ "cacheVariables": {+ "CMAKE_BUILD_TYPE": "Debug",+ "BTLA_UT_DEBUG": "ON",+ "BTLA_UT_ALL": "OFF"+ },+ "condition": {+ "type": "equals",+ "lhs": "${hostSystemName}",+ "rhs": "Linux"+ },+ "vendor": { "microsoft.com/VisualStudioRemoteSettings/CMake/1.0": { "sourceDir": "$env{HOME}/.vs/$ms{projectDirName}" } }+ },+ {+ "name": "linux-release-UT",+ "displayName": "linux Release for UT",+ "description": "Run all UT",+ "inherits": "linux-debug",+ "cacheVariables": {+ "CMAKE_BUILD_TYPE": "Release",+ "BTLA_UT_ALL": "ON"+ }+ },+ {+ "name": "linux-release",+ "displayName": "linux Release",+ "description": "Release",+ "inherits": "linux-debug",+ "cacheVariables": { "CMAKE_BUILD_TYPE": "Release" }+ },+ {+ "name": "windows-base",+ "description": "Target Windows with the Visual Studio development environment.",+ "hidden": true,+ "generator": "Ninja",+ "binaryDir": "${sourceDir}/out/build/${presetName}",+ "installDir": "${sourceDir}/out/install/${presetName}",+ "cacheVariables": {+ "CMAKE_C_COMPILER": "cl.exe",+ "CMAKE_CXX_COMPILER": "cl.exe",+ "BTLA_UT_ALL": "OFF"+ },+ "condition": {+ "type": "equals",+ "lhs": "${hostSystemName}",+ "rhs": "Windows"+ }+ },+ {+ "name": "x64-debug",+ "displayName": "x64 Debug",+ "description": "Target Windows (64-bit) with the Visual Studio development environment. (Debug)",+ "inherits": "windows-base",+ "architecture": {+ "value": "x64",+ "strategy": "external"+ },+ "cacheVariables": {+ "CMAKE_BUILD_TYPE": "Debug",+ "BTLA_UT_DEBUG": "ON",+ "BTLA_UT_ALL": "OFF"+ }+ },+ {+ "name": "x64-release",+ "displayName": "x64 Release",+ "description": "Target Windows (64-bit) with the Visual Studio development environment. (RelWithDebInfo)",+ "inherits": "x64-debug",+ "cacheVariables": { "CMAKE_BUILD_TYPE": "Release" }+ },+ {+ "name": "x64-release-UT",+ "displayName": "x64 Release for UT",+ "description": "Target Windows (64-bit) with the Visual Studio development environment. (RelWithDebInfo)",+ "inherits": "x64-release",+ "cacheVariables": { "BTLA_UT_ALL": "ON" }+ },+ {+ "name": "x64-debug-sycl",+ "displayName": "x64 Debug SYCL",+ "description": "x64 Debug SYCL",+ "inherits": "windows-base",+ "architecture": {+ "value": "x64",+ "strategy": "external"+ },+ "cacheVariables": {+ "CMAKE_BUILD_TYPE": "Debug",+ "BTLA_UT_DEBUG": "ON",+ "BTLA_UT_ALL": "OFF",+ "BTLA_SYCL": "ON",+ "BTLA_UT_BENCHMARK": "ON",+ "CMAKE_CXX_COMPILER": "icx",+ "CMAKE_C_COMPILER": "icx"+ }+ },+ {+ "name": "x64-release-sycl",+ "displayName": "x64 Release for SYCL",+ "description": "x64 SYCL",+ "inherits": "x64-debug-sycl",+ "cacheVariables": {+ "CMAKE_BUILD_TYPE": "Release"+ }+ }+ ]+}diff --git a/auto_round_extension/ark/auto_round_kernel/bestla/README.md b/auto_round_extension/ark/auto_round_kernel/bestla/README.mdnew file mode 100644index 000000000..425c3ec83--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/README.md@@ -0,0 +1,90 @@+# BesTLA+BesTLA is a lightweight, header-only acceleration library for high-performance GEMM and related computations on Intel platform. Inspired by Cutlass, it provides high-level template class abstractions for various elements required for computation, and allows flexible kernel construction through template combinations to meet specific needs, maximizing the reuse of existing template classes. Users can also develop custom template classes to expand BesTLA’s computational capabilities. BesTLA includes several different types of template classes, specifically:++- `Launcher`: Schedules computation-related template classes, allowing users to specify their own computation-related template classes, including GemmCore, Prologue, and Epilogue.+- `Parallel`: Specifies data splitting strategy for task distribution among different cores. BesTLA’s default Parallel template class adopts an L2-cache-fusion concept, i.e., each core tries to temporarily store the data it processes in its L2-cache during each round of gemm-tile computation.+- `GemmCore`: A computation-related template class that provides a micro-kernel for performing a tile gemm computation with a specific ISA. It is the most important template class in BesTLA. Currently, GemmCore supports the following ISAs:+ - AVX2: sgemm, u8s8 igemm+ - AVX_VNNI: u8s8&s8s8 igemm+ - AVX512F: sgemm+ - AVX512BW: u8s8 igemm+ - AVX512_VNNI: u8s8 igemm+ - AMX_BF16: bf16 hgemm+ - AMX_INT8: s8s8&u8s8&u8u8&s8u8 igemm+ - AVX512_FP16: fp16 hgemm+- `Prologue`: A computation-related template class that preprocesses (such as data type conversion/padding) input data to meet GemmCore’s input data requirements.+- `Epilogue`: A computation-related template class that post-processes (such as eltwiseop-fusion) the results of gemm-core computations to expand BesTLA’s application scenarios.+BesTLA supports users to configure thread libraries for multi-core parallelism (e.g. openMP), greatly facilitating user integrate BesTLA into their own projects. BesTLA also supports specifying the number of computing-threads at runtime, making the allocation of computing resources more flexible.++# Highlights+## Weight-only+BesTLA provides weight-only linear computational capabilities for LLM inference. We provide a series of Prologues for quantize/compress/serialize/deserialize fp32 weights in different ways. Specifically, the weight-only-quantization configs we support are given in the table below:++| Weight dtype | Compute dtype | Scale dtype | algo |+| ---------------------- | :----------------: | :---------------: | :--------: |+| INT8 | INT8 / BF16 / FP32 | BF16 / FP32 | sym / asym |+| INT4 | INT8 / BF16 / FP32 | BF16 / FP32 | sym / asym |+| INT3 | INT8 / BF16 / FP32 | BF16 / FP32 | sym / asym |+| INT2 | INT8 / BF16 / FP32 | BF16 / FP32 | sym / asym |+| INT5 | INT8 / BF16 / FP32 | BF16 / FP32 | sym / asym |+| INT6 | INT8 / BF16 / FP32 | BF16 / FP32 | sym / asym |+| INT7 | INT8 / BF16 / FP32 | BF16 / FP32 | sym / asym |+| INT1 | INT8 / BF16 / FP32 | BF16 / FP32 | sym / asym |+| FP8 (E4M3, E5M2) | BF16 / FP32 | FP32 / FP8 (E8M0) | sym |+| FP4 (E2M1) | BF16 / FP32 | BF16 / FP32 | sym |+| NF4 | BF16 / FP32 | BF16 / FP32 | sym |++Config description of the table:+| Config | Description |+| ------------- | --------------------------------------------------- |+| Weight dtype | Data type of quantized weight |+| Compute dtype | Data type of BesTLA internal Gemm computation |+| Scale dtype | Data type of scales |+| algo | Quantization algorithm to use(symmetric/asymmetric) |+++## Postop-fusion+BesTLA provides assembly-level postop-fusion through epilogue to minimize the overhead caused by data movement. Specifically, we support the following postop-fusions:++- GELU+- SWISH+- RELU+- EXP+- TANH++## Optimized thread pool for hybrid CPUs+Our thread pool is optimized for hybrid CPUs (client CPUs after Core 11th). It will be much faster than OMP thread pool.+Recommend to use all threads and cores on hybrid CPU: P cores * 2 + E cores.++## Compilation Requirements and Usage+Compile:++- GCC version >= 9.0+- CMake version >= 3.12+- MSVC version >= 1900+- oneAPI version >= 2024.0++Best Performance:++- GCC >= 11.0.0+- MSVC >= 1930+- DPCPP >= 2024.0+++Usage:+```cmake+add_subdirectory(bestla)+target_link_libraries("${YOUR_PROJECT}" neural_speed::bestla)+```++# Benchmark+Build with:+```shell+mkdir build+cd build+cmake .. -DBTLA_UT_BENCHMARK=ON -DBTLA_UT_ALL=ON -DCMAKE_BUILD_TYPE=Release+cmake --build . -j+./bin/bestla_benchmark+```++More template usages, please refer code in [bestla_benchmark](bestla/ut/bestla_benchmark.cpp)diff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla.hnew file mode 100644index 000000000..4471e49d2--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla.h@@ -0,0 +1,109 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include <stdint.h>+enum class BTLA_CODE {+ Success = 0,+ InvalidParam = 1,+ InvalidISA = 2,+ RuntimeError = 4,+ NotSupport = 8,+};+enum class BTLA_ISA : uint8_t {+ NoSIMD = 0,+ AVX,+ AVX2,+ AVX_VNNI,+ AVX512F,+ AVX512BW,+ AVX512_VNNI,+ AVX512_BF16,+ AVX512_FP16,+ AMX_BF16,+ AMX_INT8,+ AMX_FP16,+ ISA_COUNT,+};+enum class BTLA_DTYPE : uint32_t {+ EleBitsMask = 0xff,+ EleBitsShift = 0,+ EleBitsUndef = 0,+ EleBits1 = 1,+ EleBits2 = 2,+ EleBits3 = 3,+ EleBits4 = 4,+ EleBits5 = 5,+ EleBits6 = 6,+ EleBits7 = 7,+ EleBits8 = 8,+ EleBits16 = 16,+ EleBits32 = 32,+ EleBits64 = 64,+ TypeMask = 0xff00,+ TypeShift = 8,+ TypeFloat = 0 << TypeShift,+ TypeInt = 1 << TypeShift,+ SubTypeMask = 0xff0000,+ SubTypeShift = 16,+ SubType0 = 0 << SubTypeShift,+ SubType1 = 1 << SubTypeShift,+ SubType2 = 2 << SubTypeShift,+ SubType3 = 3 << SubTypeShift,+ SubType4 = 4 << SubTypeShift,+ F64 = EleBits64 | TypeFloat,+ F32 = EleBits32 | TypeFloat,+ F16 = EleBits16 | TypeFloat,+ BF16 = EleBits16 | TypeFloat | SubType1,+ F8_E4M3 = EleBits8 | TypeFloat,+ F8_E5M2 = EleBits8 | TypeFloat | SubType1,+ F8_E3M4 = EleBits8 | TypeFloat | SubType2,+ F8_E8M0 = EleBits8 | TypeFloat | SubType3,+ DQ8_BNB = EleBits8 | TypeFloat | SubType4,+ S8 = EleBits8 | TypeInt,+ U8 = EleBits8 | TypeInt | SubType1,+ S1_CLIP = EleBits1 | TypeInt,+ S1 = S1_CLIP,+ S2_CLIP = EleBits2 | TypeInt,+ S2 = S2_CLIP,+ S3_CLIP = EleBits3 | TypeInt,+ S3 = S3_CLIP,+ S4_CLIP = EleBits4 | TypeInt,+ S4 = S4_CLIP,+ S5_CLIP = EleBits5 | TypeInt,+ S5 = S5_CLIP,+ S6_CLIP = EleBits6 | TypeInt,+ S6 = S6_CLIP,+ S7_CLIP = EleBits7 | TypeInt,+ S7 = S7_CLIP,+ F4_E2M1 = EleBits4 | TypeFloat,+ F4_BNB = EleBits4 | TypeFloat | SubType1,+ F4_NF4 = EleBits4 | TypeFloat | SubType2,+ S32 = EleBits32 | TypeInt,+ U32 = EleBits32 | TypeInt | SubType1,+};++enum class BTLA_ELTWISEOP { GELU, SWISH, TANH, EXP, LOW_PRECISION_EXP, RELU, LINEAR };++enum class BTLA_PROLOGUEB_IDS : uint32_t {+ Undef = (uint32_t)-1,+ Begin = 0,+ NormalBegin = Begin,+ WeightPack = NormalBegin,+ NormalEnd,+ KBlockBegin = NormalEnd,+ WeightKBlockNInteger = KBlockBegin,+ WeightKBlockNFloat,+ KBlockEnd,+ End,+};diff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_device.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_device.hnew file mode 100644index 000000000..606cc3359--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_device.h@@ -0,0 +1,809 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include <map>+#include <thread>+#include <vector>+#include "bestla.h"+#include "xbyak/xbyak_util.h"+#include "bestla_utils.h"+#ifdef _WIN32+#include <windows.h>+#define FIXED_CACHE 1+#else+#include <sched.h>+#include <unistd.h>+#define FIXED_CACHE 0+#endif++#define FIXED_CACHE_SIZE ((1 << 20) - (128 << 10))++namespace bestla {++namespace device {++struct X64_ISA {+ int64_t MMX : 1; // 0+ int64_t SSE : 1; // 1+ int64_t SSE2 : 1; // 2+ int64_t SSE3 : 1; // 3+ int64_t SSSE3 : 1; // 4+ int64_t SSE41 : 1; // 5+ int64_t SSE42 : 1; // 6+ int64_t AVX : 1; // 7+ int64_t F16C : 1; // 8+ int64_t FMA : 1; // 9+ int64_t AVX2 : 1; // 10+ int64_t AVX_VNNI : 1; // 11+ int64_t AVX_VNNI_INT8 : 1; // 12+ int64_t AVX_NE_CONVERT : 1; // 13+ int64_t AVX_IFMA : 1; // 14+ int64_t AVX512F : 1; // 15+ int64_t AVX512BW : 1; // 16+ int64_t AVX512CD : 1; // 17+ int64_t AVX512DQ : 1; // 18+ int64_t AVX512ER : 1; // 19+ int64_t AVX512IFMA52 : 1; // 20+ int64_t AVX512PF : 1; // 21+ int64_t AVX512VL : 1; // 22+ int64_t AVX512VPOPCNTDQ : 1; // 23+ int64_t AVX512_4FMAPS : 1; // 24+ int64_t AVX512_4VNNIW : 1; // 25+ int64_t AVX512_BF16 : 1; // 26+ int64_t AVX512_BITALG : 1; // 27+ int64_t AVX512_VBMI : 1; // 28+ int64_t AVX512_VBMI2 : 1; // 29+ int64_t AVX512_VNNI : 1; // 30+ int64_t AVX512_VP2INTERSECT : 1; // 31+ int64_t AVX512_FP16 : 1; // 32+ int64_t AMX_TILE : 1; // 33+ int64_t AMX_BF16 : 1; // 34+ int64_t AMX_INT8 : 1; // 35+ int64_t AMX_FP16 : 1; // 36+ int64_t AMX_COMPLEX : 1; // 37+ int64_t reserved : (64 - 38);+};++class AVX2_Default {+ public:+ static constexpr bool MMX = 1;+ static constexpr bool SSE = 1;+ static constexpr bool SSE2 = 1;+ static constexpr bool SSE3 = 1;+ static constexpr bool SSSE3 = 1;+ static constexpr bool SSE41 = 1;+ static constexpr bool SSE42 = 1;+ static constexpr bool AVX = 1;+ static constexpr bool F16C = 1;+ static constexpr bool FMA = 1;+ static constexpr bool AVX2 = 1;+ static constexpr bool AVX_VNNI = 0;+ static constexpr bool AVX_VNNI_INT8 = 0;+ static constexpr bool AVX_NE_CONVERT = 0;+ static constexpr bool AVX_IFMA = 0;+ static constexpr bool AVX512F = 0;+ static constexpr bool AVX512BW = 0;+ static constexpr bool AVX512CD = 0;+ static constexpr bool AVX512DQ = 0;+ static constexpr bool AVX512ER = 0;+ static constexpr bool AVX512IFMA52 = 0;+ static constexpr bool AVX512PF = 0;+ static constexpr bool AVX512VL = 0;+ static constexpr bool AVX512VPOPCNTDQ = 0;+ static constexpr bool AVX512_4FMAPS = 0;+ static constexpr bool AVX512_4VNNIW = 0;+ static constexpr bool AVX512_BF16 = 0;+ static constexpr bool AVX512_BITALG = 0;+ static constexpr bool AVX512_VBMI = 0;+ static constexpr bool AVX512_VBMI2 = 0;+ static constexpr bool AVX512_VNNI = 0;+ static constexpr bool AVX512_VP2INTERSECT = 0;+ static constexpr bool AVX512_FP16 = 0;+ static constexpr bool AMX_TILE = 0;+ static constexpr bool AMX_BF16 = 0;+ static constexpr bool AMX_INT8 = 0;+ static constexpr bool AMX_FP16 = 0;+ static constexpr bool AMX_COMPLEX = 0;+};++class AVX512_VNNI_Default {+ public:+ static constexpr bool MMX = 1;+ static constexpr bool SSE = 1;+ static constexpr bool SSE2 = 1;+ static constexpr bool SSE3 = 1;+ static constexpr bool SSSE3 = 1;+ static constexpr bool SSE41 = 1;+ static constexpr bool SSE42 = 1;+ static constexpr bool AVX = 1;+ static constexpr bool F16C = 1;+ static constexpr bool FMA = 1;+ static constexpr bool AVX2 = 1;+ static constexpr bool AVX_VNNI = 0;+ static constexpr bool AVX_VNNI_INT8 = 0;+ static constexpr bool AVX_NE_CONVERT = 0;+ static constexpr bool AVX_IFMA = 0;+ static constexpr bool AVX512F = 1;+ static constexpr bool AVX512BW = 1;+ static constexpr bool AVX512CD = 1;+ static constexpr bool AVX512DQ = 1;+ static constexpr bool AVX512ER = 0;+ static constexpr bool AVX512IFMA52 = 0;+ static constexpr bool AVX512PF = 0;+ static constexpr bool AVX512VL = 1;+ static constexpr bool AVX512VPOPCNTDQ = 0;+ static constexpr bool AVX512_4FMAPS = 0;+ static constexpr bool AVX512_4VNNIW = 0;+ static constexpr bool AVX512_BF16 = 0;+ static constexpr bool AVX512_BITALG = 0;+ static constexpr bool AVX512_VBMI = 0;+ static constexpr bool AVX512_VBMI2 = 0;+ static constexpr bool AVX512_VNNI = 1;+ static constexpr bool AVX512_VP2INTERSECT = 0;+ static constexpr bool AVX512_FP16 = 0;+ static constexpr bool AMX_TILE = 0;+ static constexpr bool AMX_BF16 = 0;+ static constexpr bool AMX_INT8 = 0;+ static constexpr bool AMX_FP16 = 0;+ static constexpr bool AMX_COMPLEX = 0;+};++class SapphireRapids {+ public:+ static constexpr bool MMX = 1;+ static constexpr bool SSE = 1;+ static constexpr bool SSE2 = 1;+ static constexpr bool SSE3 = 1;+ static constexpr bool SSSE3 = 1;+ static constexpr bool SSE41 = 1;+ static constexpr bool SSE42 = 1;+ static constexpr bool AVX = 1;+ static constexpr bool F16C = 1;+ static constexpr bool FMA = 1;+ static constexpr bool AVX2 = 1;+ static constexpr bool AVX_VNNI = 0;+ static constexpr bool AVX_VNNI_INT8 = 0;+ static constexpr bool AVX_NE_CONVERT = 0;+ static constexpr bool AVX_IFMA = 0;+ static constexpr bool AVX512F = 1;+ static constexpr bool AVX512BW = 1;+ static constexpr bool AVX512CD = 1;+ static constexpr bool AVX512DQ = 1;+ static constexpr bool AVX512ER = 0;+ static constexpr bool AVX512IFMA52 = 0;+ static constexpr bool AVX512PF = 0;+ static constexpr bool AVX512VL = 1;+ static constexpr bool AVX512VPOPCNTDQ = 0;+ static constexpr bool AVX512_4FMAPS = 0;+ static constexpr bool AVX512_4VNNIW = 0;+ static constexpr bool AVX512_BF16 = 0;+ static constexpr bool AVX512_BITALG = 0;+ static constexpr bool AVX512_VBMI = 0;+ static constexpr bool AVX512_VBMI2 = 0;+ static constexpr bool AVX512_VNNI = 1;+ static constexpr bool AVX512_VP2INTERSECT = 0;+ static constexpr bool AVX512_FP16 = 0;+ static constexpr bool AMX_TILE = 1;+ static constexpr bool AMX_BF16 = 1;+ static constexpr bool AMX_INT8 = 1;+ static constexpr bool AMX_FP16 = 0;+ static constexpr bool AMX_COMPLEX = 0;+};++template <BTLA_ISA ISA_T>+class isa_base {+ public:+ static bool constexpr avx = ISA_T >= BTLA_ISA::AVX;+ static bool constexpr avx2 = ISA_T >= BTLA_ISA::AVX2;+ static bool constexpr avx512f = ISA_T >= BTLA_ISA::AVX512F;+ static bool constexpr avx512bw = ISA_T >= BTLA_ISA::AVX512BW;+ static bool constexpr avx512_vnni = ISA_T >= BTLA_ISA::AVX512_VNNI;+ static bool constexpr avx512_fp16 = ISA_T >= BTLA_ISA::AVX512_FP16;+ static bool constexpr amx_bf16 = ISA_T >= BTLA_ISA::AMX_BF16;+ static bool constexpr amx_int8 = ISA_T >= BTLA_ISA::AMX_INT8;+};++class CpuRuntimeV2 {+ public:+ CpuRuntimeV2() = default;+ inline void adjustPE(int _kernel_id, const float PE_) {+ // printf("Adjust:%d,%f\n",int(isa),PE_);+ PR[_kernel_id] = PR[_kernel_id] * PE_ * 0.7f + PR[_kernel_id] * 0.3f;+ }++ size_t L1_cache_ = 0, L2_cache_ = 0;+ int core_id_ = 0, core_sub_id_ = 0;+ static constexpr int MAX_KERNEL_NUM = 256;+ float PR[MAX_KERNEL_NUM];+};++class CpuDevice {+ public:+ inline int getThreads() { return numthreads; }+ inline int getTotalThreads() { return num_total_threads_; }+ inline int getCores() { return isHybrid() ? getPcoreNum() + getEcoreNum() : numcores; }+ inline uint32_t getL3CacheSize() { return L3Cache; }+ inline uint32_t getL2CacheSize() { return L2Cache; }+ inline uint32_t getL1CacheSize() { return L1Cache; }+ inline uint32_t getL2CacheSize_E() { return E_L2Cache; }+ inline uint32_t getL1CacheSize_E() { return E_L1Cache; }+ inline bool INTEL() { return mIntel; }+ inline bool AVX() { return mHasAVX; }+ inline bool AVX2() { return mHasAVX2; }+ inline bool AVX_VNNI() { return mHasAVX_VNNI; }+ inline bool AVX512F() { return mHasAVX512F; }+ inline bool AVX512BW() { return mHasAVX512BW; }+ inline bool AVX512_VNNI() { return mHasAVX512_VNNI; }+ inline bool AMX_INT8() { return mHasAMX_INT8; }+ inline bool AMX_BF16() { return mHasAMX_BF16; }+ inline bool AVX512_BF16() { return mHasAVX512_BF16; }+ inline bool AVX512_FP16() { return mHasAVX512_FP16; }+ inline bool AMX_FP16() { return mHasAMX_FP16; }+ inline float* const getPE() { return PE; }+ inline int getPcoreNum() { return static_cast<int>(P_core.size()); }+ inline int getEcoreNum() { return static_cast<int>(E_core.size()); }+ inline int getSMTcoreNum() { return isHybrid() ? static_cast<int>(SMT_core.size()) : getCores(); }+ inline int* getPCores() { return P_core.data(); }+ inline int* getECores() { return E_core.data(); }+ inline int* getSMTCores() { return SMT_core.data(); }+#define ADD_FLAG(isa) mHas##isa = _cpu.has(_cpu.t##isa)+ CpuDevice() {+ static Xbyak::util::Cpu _cpu;+ ADD_FLAG(AVX);+ ADD_FLAG(AVX2);+ ADD_FLAG(AVX512F);+ ADD_FLAG(AVX512BW);+ ADD_FLAG(AVX512_VNNI);+ ADD_FLAG(AVX_VNNI);+ ADD_FLAG(AMX_BF16);+ ADD_FLAG(AMX_INT8);+ ADD_FLAG(AVX512_BF16);+ ADD_FLAG(AVX512_FP16);+ ADD_FLAG(AMX_FP16);+ mIntel = _cpu.has(_cpu.tINTEL);+ numcores = _cpu.getNumCores(Xbyak::util::IntelCpuTopologyLevel::CoreLevel);+ numcores = numcores >= 48 ? numcores * 2 : numcores;+ if (mHasAMX_BF16 || mHasAMX_INT8 || mHasAMX_FP16) {+ utils::request_perm_xtile_data();+ }+ static bool p = false;+ {+ uint32_t tmp[4];+ _cpu.getCpuid(7, tmp);+ if (tmp[3] & (1U << 15)) mHybrid = true;+ if (p) printf("!!!Hybrid:%d\t%x\t%x\t%x\t%x!!!\n", mHybrid, tmp[0], tmp[1], tmp[2], tmp[3]);+ }+ num_total_threads_ = numcores * _cpu.getNumCores(Xbyak::util::IntelCpuTopologyLevel::SmtLevel);+ compute_infos_.resize(num_total_threads_);++ if (mHybrid) {+ mClient = true;+ std::vector<int> core_type(num_total_threads_), core_id(num_total_threads_), L1(num_total_threads_),+ L2(num_total_threads_);+ std::map<int, int> core_id_count;+ std::vector<CpuRuntimeV2> cores(num_total_threads_);+ {+ // classify E-core / LPE-core and P-core / smt+ std::vector<std::thread> thdset(num_total_threads_);+ for (size_t i = 0; i < num_total_threads_; i++) {+ thdset[i] = std::thread(+ [&](int tidx) {+ core_bond(tidx);+ Xbyak::util::Cpu cpu;+ L1[tidx] = cpu.getDataCacheSize(0);+ L2[tidx] = cpu.getDataCacheSize(1);+ if (isEcore(cpu))+ core_type[tidx] = 1;+ else+ core_type[tidx] = 2;+ core_id[tidx] = getCoreId(cpu);+ },+ int(i));+ }+ for (size_t i = 0; i < num_total_threads_; i++) {+ thdset[i].join();+ core_id_count[core_id[i]] = core_id_count[core_id[i]] + 1;+ }+ if (p) {+ for (int i = 0; i < num_total_threads_; i++) printf("%d %d\n", core_type[i], core_id[i]);+ for (auto& kv : core_id_count) printf("%d,%d\n", kv.first, kv.second);+ }+ for (int i = 0; i < num_total_threads_; i++) {+ if (core_type[i] == 2) {+ if (core_id_count[core_id[i]] > 0) {+ P_core.push_back(i);+ core_id_count[core_id[i]] = 0;+ } else {+ SMT_core.push_back(i);+ }+ } else {+ if (core_id_count[core_id[i]] == 4) E_core.push_back(i);+ }+ }+ if (p) {+ printf("Pcore:");+ for (auto& i : P_core) printf("%d,", i);+ printf("\nEcore:");+ for (auto& i : E_core) printf("%d,", i);+ printf("\nsmt:");+ for (auto& i : SMT_core) printf("%d,", i);+ printf("\n");+ }+ mHybrid = !(E_core.empty() || P_core.empty()); // in case of bond core by external+ if (!E_core.empty()) {+ E_L1Cache = L1[E_core[0]];+ E_L2Cache = L2[E_core[0]] / 4;+ };+ if (!P_core.empty()) {+ L1Cache = L1[P_core[0]];+ L2Cache = L2[P_core[0]];+ }+ }+ numcores = static_cast<int>(P_core.size() + E_core.size());+ numthreads = static_cast<int>(P_core.size() + E_core.size() + SMT_core.size());++ {+ // set PE+ uint32_t tmp[4];+ _cpu.getCpuid(1, tmp);+ if (p) printf("!!!\t%x\t%x\t%x\t%x!!!\n", tmp[0], tmp[1], tmp[2], tmp[3]);+ const int famliy = (tmp[0] >> 8) & ((1u << 4) - 1); // cpu.extractBit(a[0], 8, 11);+ const int extendedModel = (tmp[0] >> 16) & ((1u << 4) - 1); // cpu.extractBit(a[0], 16, 24);+ {+ for (int i = 0; i < int(BTLA_ISA::ISA_COUNT); i++) PE[i] = 1.0f;+ // CPU identification refer to: https://en.wikichip.org/wiki/intel/cpuid+ if (famliy == 6) switch (extendedModel) {+ case 9: // ALD+ PE[int(BTLA_ISA::AVX2)] = 3.0f;+ PE[int(BTLA_ISA::AVX_VNNI)] = 5.0f;+ PE[int(BTLA_ISA::NoSIMD)] = 3.5f;+ break;+ case 10: // MTL+ PE[int(BTLA_ISA::AVX2)] = 2.2f;+ PE[int(BTLA_ISA::AVX_VNNI)] = 3.0f;+ PE[int(BTLA_ISA::NoSIMD)] = 3.0f;+ break;+ case 11: // RPL+ PE[int(BTLA_ISA::AVX2)] = 1.8f;+ PE[int(BTLA_ISA::AVX_VNNI)] = 2.6f;+ PE[int(BTLA_ISA::NoSIMD)] = 3.0f;+ break;+ }+ }+ }+ } else {+ L1Cache = _cpu.getDataCacheSize(0);+ L2Cache = _cpu.getDataCacheSize(1);+ numthreads = numcores;+ }+ L3Cache = _cpu.getDataCacheSize(2);+#if FIXED_CACHE+ L2Cache = L2Cache >= FIXED_CACHE_SIZE ? FIXED_CACHE_SIZE : L2Cache;+ E_L2Cache = E_L2Cache >= FIXED_CACHE_SIZE ? FIXED_CACHE_SIZE : E_L2Cache;+#endif+ }++ static CpuDevice* getInstance() {+ static CpuDevice instance;+ return &instance;+ }++ int get_actual_usable_cores() {+#ifdef _WIN32+ return numcores;+#else+ cpu_set_t cpuset;+ // 初始化 CPU 集合+ CPU_ZERO(&cpuset);++ // 获取当前进程的亲和性掩码+ // 第一个参数为 0 表示当前进程+ if (sched_getaffinity(0, sizeof(cpu_set_t), &cpuset) == -1) {+ return numcores;+ }++ // 计算集合中包含的核心数量+ int count = CPU_COUNT(&cpuset);+ return count;+#endif+ }++ void print() {+ printf(+ "AVX:%d AVX2:%d AVX512F:%d AVX512BW:%d AVX_VNNI:%d AVX512_VNNI:%d AMX_INT8:%d AMX_BF16:%d AVX512_BF16:%d "+ "AVX512_FP16:%d\n",+ mHasAVX, mHasAVX2, mHasAVX512F, mHasAVX512BW, mHasAVX_VNNI, mHasAVX512_VNNI, mHasAMX_INT8, mHasAMX_BF16,+ mHasAVX512_BF16, mHasAVX512_FP16);+ }+#undef ADD_FLAG++ static bool isEcore() {+ Xbyak::util::Cpu cpu;+ uint32_t tmp[4];+ cpu.getCpuid(0x1A, tmp);+ int core_type = (tmp[0] >> 24) & ((1u << 8) - 1); // cpu.extractBit(a[0], 24, 31);+ switch (core_type) {+ case 32:+ // printf("Atom\n");+ return true; // E-core or LPE-core+ break;+ case 64:+ // printf("Core\n");+ return false; // P-core+ break;+ default:+ // printf("No hyper\n");+ return false;+ break;+ }+ return false;+ }++ int getCoreId(Xbyak::util::Cpu& cpu) {+ uint32_t tmp[4];+ cpu.getCpuidEx(0x1F, 1, tmp); // sub-leaf 1 is core domain+ // printf("!!!%x\t%x\t%x\t%x!!!\n", tmp[0], tmp[1], tmp[2], tmp[3]);+ if (tmp[0] != 0 && tmp[1] != 0)+ return tmp[3] >> 3; // tmp[3] is APIC+ else+ return tmp[3];+ }++ bool isEcore(Xbyak::util::Cpu& cpu) {+ uint32_t tmp[4];+ cpu.getCpuid(0x1A, tmp);+ int core_type = (tmp[0] >> 24) & ((1u << 7) - 1); // cpu.extractBit(a[0], 24, 31);+ switch (core_type) {+ case 32:+ // printf("Atom\n");+ return true; // E-core or LPE-core+ break;+ case 64:+ // printf("Core\n");+ return false; // P-core+ break;+ default:+ // printf("No hyper\n");+ return false;+ break;+ }+ return false;+ }+ static void core_bond(int core) {+#ifdef _WIN32+ SetThreadAffinityMask(GetCurrentThread(), 1LL << core);+#else+ cpu_set_t cpuset;+ CPU_ZERO(&cpuset);+ CPU_SET(core, &cpuset);+ int s = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);+ if (s != 0) printf("Bond Core ERROR:%d\n", core);+#endif+ }++ static void core_bond(std::thread& thread, int core) {+#ifdef _WIN32+ HANDLE handle = thread.native_handle();+ SetThreadAffinityMask(handle, 1LL << core);+#else+ cpu_set_t cpuset;+ CPU_ZERO(&cpuset);+ CPU_SET(core, &cpuset);+ pthread_t pt = thread.native_handle();+ int s = pthread_setaffinity_np(pt, sizeof(cpuset), &cpuset);+ if (s != 0) printf("Bond Core ERROR:%d\n", core);+#endif+ }++ bool isHybrid() { return mHybrid; }+ bool isClient() { return mClient; }++ protected:+ std::vector<CpuRuntimeV2> compute_infos_;+ uint32_t L2Cache = 0, L1Cache = 0, L3Cache = 0;+ bool mHybrid = false, mClient = false;+ bool mHasAVX2 = false, mHasAVX_VNNI = false, mHasAVX = false, mHasAVX512_VNNI = false, mHasAMX_INT8 = false,+ mHasAMX_BF16 = false, mHasAVX512F = false, mHasAVX512BW, mHasAVX512_BF16 = false, mHasAVX512_FP16 = false,+ mHasAMX_FP16 = false, mIntel = false;+ int numcores = 0;+ int numthreads = 0, num_total_threads_ = 0;+ std::vector<int> P_core, E_core, SMT_core;+ uint32_t E_L2Cache = 0, E_L1Cache = 0;+ float PE[int(BTLA_ISA::ISA_COUNT)] = {1.f};+};++#define GetCPUDevice() auto _cd = bestla::device::CpuDevice::getInstance();++/*+1. P coers+3. P cores + P hyper+5. P cores => numanodes+6. P cores => numanodes + hyper+7. P cores => sockets (numanodes + hyper)++client only:+2. P cores + E Cores+4. P cores + P hyper + E Cores+*/+class CpuRange {+ public:+ enum class CoreType : int { Undef = 0, E = 1, P = 2, Count };+ /*inline bool AVX2() { return mHasAVX2; }+ inline bool AVX_VNNI() { return mHasAVX_VNNI; }+ inline bool AVX512F() { return mHasAVX512F; }+ inline bool AVX512BW() { return mHasAVX512BW; }+ inline bool AVX512_VNNI() { return mHasAVX512_VNNI; }+ inline bool AMX_INT8() { return mHasAMX_INT8; }+ inline bool AMX_BF16() { return mHasAMX_BF16; }+ inline bool AVX512_BF16() { return mHasAVX512_BF16; }+ inline bool AVX512_FP16() { return mHasAVX512_FP16; }+ inline bool AMX_FP16() { return mHasAMX_FP16; }+ inline float* const getPE() { return PE; }+ inline int getPcoreNum() { return static_cast<int>(P_core.size()); }+ inline int getEcoreNum() { return static_cast<int>(E_core.size()); }+ inline int getSMTcoreNum() { return isHybrid() ? static_cast<int>(SMT_core.size()) : getCores(); }+ inline int* getPCores() { return P_core.data(); }+ inline int* getECores() { return E_core.data(); }+ inline int* getSMTCores() { return SMT_core.data(); }*/++ CpuRange() { // single thread+ cores_ = {0};+ update();+ }++ CpuRange(const std::vector<int>& _cores, bool _hyper = false) {+ cores_ = _cores;+ hyper_ = _hyper;+ update();+ }++ CpuRange(const std::vector<std::vector<int>>& _core_groups, bool _hyper = true) {+ hyper_ = _hyper;+ core_groups_ = _core_groups;+ for (size_t i = 0; i < _core_groups.size(); i++) {+ cores_.insert(cores_.end(), _core_groups[i].begin(), _core_groups[i].end());+ }+ update();+ }++ CpuRange(int pcores, int ecores, bool _hyper = false) {+ int p_num = _hyper ? pcores * 2 : pcores;+ int e_num = ecores;+ hyper_ = _hyper;+ Xbyak::util::Cpu cpu;+ auto n_core = cpu.getNumCores(Xbyak::util::SmtLevel) * cpu.getNumCores(Xbyak::util::CoreLevel);+ std::vector<int> all_cores(n_core);+ for (size_t i = 0; i < all_cores.size(); i++) {+ all_cores[i] = i;+ }+ cores_ = all_cores;+ update();+ std::vector<int> p_cores(p_num);+ for (size_t i = 0; i < p_num; i++) {+ p_cores[i] = _hyper ? i : i * 2;+ }+ core_groups_.emplace_back(p_cores);+ if (e_num) {+ std::vector<int> e_cores(e_num);+ auto e_start = core_type_count_[(int)CoreType::P];+ for (size_t i = 0; i < e_num; i++) {+ e_cores[i] = e_start + i;+ }+ core_groups_.emplace_back(e_cores);+ }+ }++ void update() {+ std::vector<std::thread> thdset(cores_.size());+ core_infos_.resize(cores_.size());+ core_types_.resize(cores_.size());+ core_ids_.resize(cores_.size());+ core_type_count_.resize((int)CoreType::Count, 0);+ for (size_t i = 0; i < thdset.size(); i++) {+ thdset[i] = std::thread(+ [&](int tidx) {+ core_bond(cores_[tidx]);+ Xbyak::util::Cpu cpu;+ core_infos_[tidx] = cpu;+ if (isEcore(cpu))+ core_types_[tidx] = CoreType::E;+ else+ core_types_[tidx] = CoreType::P;+ core_ids_[tidx] = getCoreId(cpu);+ },+ int(i));+ }+ for (size_t i = 0; i < thdset.size(); i++) {+ thdset[i].join();+ }+ for (size_t i = 0; i < core_ids_.size(); i++) {+ core_type_count_[(int)core_types_[i]] += 1;+ }+ }++#define ADD_FUNC(isa) \+ inline bool isa() { return core_infos_[0].has(core_infos_[0].t##isa); }+ inline int core_size() { return cores_.size(); }+ ADD_FUNC(AVX);+ ADD_FUNC(AVX2);+ ADD_FUNC(AVX512F);+ ADD_FUNC(AVX512BW);+ ADD_FUNC(AVX512_VNNI);+ ADD_FUNC(AVX_VNNI);+ ADD_FUNC(AMX_INT8);+ ADD_FUNC(AMX_BF16);+ ADD_FUNC(AMX_FP16);+ ADD_FUNC(AVX512_BF16);+ ADD_FUNC(AVX512_FP16);+ void print() {+ if (core_infos_.size()) {+ auto& info = core_infos_[0];+ printf(+ "AVX:%d AVX2:%d AVX512F:%d AVX512BW:%d AVX_VNNI:%d AVX512_VNNI:%d AMX_INT8:%d AMX_BF16:%d AVX512_BF16:%d "+ "AVX512_FP16:%d\n",+ AVX(), AVX2(), AVX512F(), AVX512BW(), AVX_VNNI(), AVX512_VNNI(), AMX_INT8(), AMX_BF16(), AVX512_BF16(),+ AVX512_FP16());+ }+ }+#undef ADD_FUNC++ static bool isEcore() {+ Xbyak::util::Cpu cpu;+ uint32_t tmp[4];+ cpu.getCpuid(0x1A, tmp);+ int core_type = (tmp[0] >> 24) & ((1u << 8) - 1); // cpu.extractBit(a[0], 24, 31);+ switch (core_type) {+ case 32:+ // printf("Atom\n");+ return true; // E-core or LPE-core+ break;+ case 64:+ // printf("Core\n");+ return false; // P-core+ break;+ default:+ // printf("No hyper\n");+ return false;+ break;+ }+ return false;+ }++ int getCoreId(Xbyak::util::Cpu& cpu) {+ uint32_t tmp[4];+ cpu.getCpuidEx(0x1F, 1, tmp); // sub-leaf 1 is core domain+ // printf("!!!%x\t%x\t%x\t%x!!!\n", tmp[0], tmp[1], tmp[2], tmp[3]);+ if (tmp[0] != 0 && tmp[1] != 0)+ return tmp[3] >> 3; // tmp[3] is APIC+ else+ return tmp[3];+ }++ bool isEcore(Xbyak::util::Cpu& cpu) {+ uint32_t tmp[4];+ cpu.getCpuid(0x1A, tmp);+ int core_type = (tmp[0] >> 24) & ((1u << 7) - 1); // cpu.extractBit(a[0], 24, 31);+ switch (core_type) {+ case 32:+ // printf("Atom\n");+ return true; // E-core or LPE-core+ break;+ case 64:+ // printf("Core\n");+ return false; // P-core+ break;+ default:+ // printf("No hyper\n");+ return false;+ break;+ }+ return false;+ }++ static void core_bond(int core) {+#ifdef _WIN32+ SetThreadAffinityMask(GetCurrentThread(), 1LL << core);+#else+ cpu_set_t cpuset;+ CPU_ZERO(&cpuset);+ CPU_SET(core, &cpuset);+ int s = sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);+ if (s != 0) printf("Bond Core ERROR:%d\n", core);+#endif+ }++ static void core_bond(std::thread& thread, int core) {+#ifdef _WIN32+ HANDLE handle = thread.native_handle();+ SetThreadAffinityMask(handle, 1LL << core);+#else+ cpu_set_t cpuset;+ CPU_ZERO(&cpuset);+ CPU_SET(core, &cpuset);+ pthread_t pt = thread.native_handle();+ int s = pthread_setaffinity_np(pt, sizeof(cpuset), &cpuset);+ if (s != 0) printf("Bond Core ERROR:%d\n", core);+#endif+ }++ std::vector<std::vector<int>> get_groups() const { return core_groups_; }++ protected:+ std::vector<std::vector<int>> core_groups_;+ std::vector<int> cores_;+ std::vector<Xbyak::util::Cpu> core_infos_;+ std::vector<CoreType> core_types_;+ std::vector<int> core_ids_;+ std::vector<CpuRuntimeV2> compute_infos_;+ std::vector<int> core_type_count_;+ bool hyper_ = false;+};++#define GetCPUDevice() auto _cd = bestla::device::CpuDevice::getInstance();++class CpuRuntime {+ public:+ CpuRuntime() = default;+ static CpuRuntime& getInstance(int thread) {+ static std::map<int, CpuRuntime> instances;+ if (instances.count(thread) == 0) instances[thread] = CpuRuntime(thread);+ return instances[thread];+ }++ inline float getPE(const BTLA_ISA isa) {+ // printf("GET:%d\t%f\n",int(isa), *cur_PE);+ return PE[int(isa)] * P_core_num / E_core_num;+ }++ inline void adjustPE(const BTLA_ISA isa, const float PE_) {+ // printf("Adjust:%d,%f\n",int(isa),PE_);+ PE[int(isa)] = PE[int(isa)] * PE_ * 0.7f + PE[int(isa)] * 0.3f;+ }++ size_t mL2Cache = 0, mL1Cache = 0, mL2Cache_P = 0, mL1Cache_P = 0, mL2Cache_E = 0, mL1Cache_E = 0;+ int P_core_num = 0, E_core_num = 0;+ bool mHybrid = false;++ private:+ CpuRuntime(int thread) {+ GetCPUDevice();+ mL2Cache = _cd->getL2CacheSize();+ mL1Cache = _cd->getL1CacheSize();+ maxThreads = _cd->getThreads();+ mHybrid = false;+ if (_cd->isClient() && thread > _cd->getPcoreNum()) {+ if (thread > _cd->getPcoreNum() + _cd->getEcoreNum()) {+ mL1Cache_P = mL1Cache / 2;+ mL2Cache_P = mL2Cache / 2;+ P_core_num = _cd->getPcoreNum();+ E_core_num = _cd->getEcoreNum();+ } else {+ mL1Cache_P = mL1Cache;+ mL2Cache_P = mL2Cache;+ P_core_num = static_cast<int>(_cd->getPcoreNum());+ E_core_num = thread - P_core_num;+ }+ if (_cd->isHybrid()) {+ mL1Cache_E = _cd->getL1CacheSize_E();+ mL2Cache_E = _cd->getL2CacheSize_E();+ mHybrid = true;+ memcpy(PE, _cd->getPE(), int(BTLA_ISA::ISA_COUNT) * sizeof(float));+ }+ }+ }+ float PE[int(BTLA_ISA::ISA_COUNT)] = {1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f};+ int maxThreads = 0;+};+} // namespace device+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_epilogue.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_epilogue.hnew file mode 100644index 000000000..8c4d30f0a--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_epilogue.h@@ -0,0 +1,250 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include <tuple>++#include "bestla.h"+#include "bestla_jit.h"+#include "bestla_utils.h"+#include "kernel_wrapper.h"++namespace bestla {+namespace epilogue {+namespace gemm {++struct ParamPcKBlockCompInt8Epilogue {+ void* scalesB;+ BTLA_DTYPE scaleBdtype;+ float* scalesA;+ // optional if A asym+ uint8_t* zpA = nullptr;+ const int32_t* reduceB = nullptr;+ // optional if B asym+ int8_t* zpB = nullptr;+ const int32_t* reduceA = nullptr;+ int K = 1;+};+template <class Fp32Epilogue>+class PcKBlockCompInt8Epilogue {+ public:+ using Fp32Param = typename Fp32Epilogue::Param;+ struct Param {+ ParamPcKBlockCompInt8Epilogue param1;+ Fp32Param param2;+ };+ using Fp32Epi = Fp32Epilogue;+ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(const int32_t* srcptr, const int cachestep, const int M_offset, const int N_offset,+ const int M, const int N, const Param& _param, void* tmpcache, size_t cachesize) {+ BTLA_CODE ret = BTLA_CODE::NotSupport;+ auto& param1 = _param.param1;+ auto zpa_ptr = param1.zpA ? param1.zpA + M_offset : nullptr;+ auto zpb_ptr = param1.zpB ? param1.zpB + N_offset : nullptr;+ if (zpa_ptr || zpb_ptr) {+ ret = kernel::wrapper::RemoveZeroPointBias::template forward<ISA_T>(+ const_cast<int32_t*>(srcptr), cachestep, M, N, zpa_ptr, zpb_ptr, 1, param1.K, param1.reduceA + M_offset,+ param1.reduceB + N_offset);+ assert(ret == BTLA_CODE::Success);+ }+ auto tmpfp32ptr = reinterpret_cast<float*>(const_cast<int32_t*>(srcptr));+ if (param1.scaleBdtype == BTLA_DTYPE::BF16) {+ auto sbptr = reinterpret_cast<utils::bf16*>(param1.scalesB) + N_offset;+ ret = kernel::wrapper::DequanS32Fp32::template forward<ISA_T>(srcptr, cachestep, tmpfp32ptr, cachestep, M, N,+ param1.scalesA + M_offset, 1, sbptr);+ } else if (param1.scaleBdtype == BTLA_DTYPE::F16) {+ auto sbptr = reinterpret_cast<utils::fp16*>(param1.scalesB) + N_offset;+ ret = kernel::wrapper::DequanS32Fp32::template forward<ISA_T>(srcptr, cachestep, tmpfp32ptr, cachestep, M, N,+ param1.scalesA + M_offset, 1, sbptr);+ } else if (param1.scaleBdtype == BTLA_DTYPE::F32) {+ auto sbptr = reinterpret_cast<float*>(param1.scalesB) + N_offset;+ ret = kernel::wrapper::DequanS32Fp32::template forward<ISA_T>(srcptr, cachestep, tmpfp32ptr, cachestep, M, N,+ param1.scalesA + M_offset, 1, sbptr);+ }+ assert(ret == BTLA_CODE::Success);+ ret = Fp32Epilogue::template forward<ISA_T>(tmpfp32ptr, cachestep, M_offset, N_offset, M, N, _param.param2,+ tmpcache, cachesize);+ return ret;+ }+};++template <typename DT>+struct ParamAccumulatorWriteBack {+ DT* C;+ int ldc;+ void* elt_const_v;+};++template <typename _SRC_T, typename _DST_T>+class AccumulatorWriteBack {+ public:+ using SType = _SRC_T;+ using DType = _DST_T;+ using Param = ParamAccumulatorWriteBack<DType>;+ using PcCompInt8Epi = bestla::epilogue::gemm::PcKBlockCompInt8Epilogue<AccumulatorWriteBack<_SRC_T, _DST_T>>;++ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(const _SRC_T* cacheptr, const int cachestep, const int M_offset, const int N_offset,+ const int M, const int N, const Param& _param, void* tmpcache, size_t cachesize) {+ auto COffset = M_offset * _param.ldc + N_offset;+ auto cptr = _param.C + COffset;+ if constexpr (std::is_same_v<_SRC_T, DType>) {+ if (cacheptr == cptr) {+ return BTLA_CODE::Success;+ }+ }++ return kernel::wrapper::Memcpy2D::template forward<ISA_T, SType, DType>(cacheptr, cptr, M, N, cachestep, _param.ldc,+ _param.elt_const_v);+ }+};++template <typename _SRC_T, typename _DST_T, BTLA_ELTWISEOP _OP>+class CustomAccumulatorWriteBackWithEltop {+ public:+ using PcCompInt8Epi =+ bestla::epilogue::gemm::PcKBlockCompInt8Epilogue<CustomAccumulatorWriteBackWithEltop<_SRC_T, _DST_T, _OP>>;+ using Param = ParamAccumulatorWriteBack<_DST_T>;+ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(const _SRC_T* cacheptr, const int cachestep, const int M_offset, const int N_offset,+ const int M, const int N, const Param& _param, void* tmpcache, size_t cachesize) {+ auto COffset = M_offset * _param.ldc + N_offset;+ auto cptr = _param.C + COffset;+ if constexpr (std::is_same<_SRC_T, float>::value && std::is_same<_DST_T, float>::value) {+ return kernel::wrapper::Memcpy2D::template forward1<ISA_T, float, float, _OP>(cacheptr, cptr, M, N, cachestep,+ _param.ldc, _param.elt_const_v);+ } else {+ assert(false);+ }+ }+};+using AccumulatorWriteBackFp32 = AccumulatorWriteBack<float, float>;+using AccumulatorWriteBackInt32 = AccumulatorWriteBack<int, int>;+using AccumulatorWriteBackBf16 = AccumulatorWriteBack<utils::bf16, utils::bf16>;+using AccumulatorWriteBackFp16 = AccumulatorWriteBack<utils::fp16, utils::fp16>;+using AccumulatorWriteBackBf16Fp32 = AccumulatorWriteBack<utils::bf16, float>;+using AccumulatorWriteBackFp16Fp32 = AccumulatorWriteBack<utils::fp16, float>;+using AccumulatorWriteBackFp32Bf16 = AccumulatorWriteBack<float, utils::bf16>;++using AccumulatorWriteBackWithGeluFp32 = CustomAccumulatorWriteBackWithEltop<float, float, BTLA_ELTWISEOP::GELU>;++using AccumulatorWriteBackWithSwishFp32 = CustomAccumulatorWriteBackWithEltop<float, float, BTLA_ELTWISEOP::SWISH>;++template <typename DT>+struct ParamAlphaBetaProcess {+ DT *C, *D;+ int ldc, ldd;+ float alpha, beta;+};+class AlphaBetaProcessFp32 {+ public:+ using Param = ParamAlphaBetaProcess<float>;++ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(const float* cacheptr, const int cachestep, const int M_offset, const int N_offset,+ const int M, const int N, const Param& _param, void* tmpcache, size_t cachesize) {+ auto DOffset = M_offset * _param.ldd + N_offset;+ auto COffset = M_offset * _param.ldc + N_offset;+ auto cptr = _param.C + COffset;+ auto dptr = _param.D + DOffset;+ return kernel::wrapper::AlphaBetaF32F32::template forward<ISA_T>(_param.alpha, cacheptr, cachestep, _param.beta,+ dptr, _param.ldd, cptr, _param.ldc, M, N);+ }+};++struct ParamDequantInt32ToFp32 {+ float* C;+ int ldc;+ int ldsa;+ float* scalesA;+ float* scalesB;+};+class DequantInt32ToFp32 {+ public:+ using Param = ParamDequantInt32ToFp32;+ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(const int32_t* cacheptr, const int cachestep, const int M_offset, const int N_offset,+ const int M, const int N, const Param& _param, void* tmpcache, size_t cachesize) {+ auto COffset = M_offset * _param.ldc + N_offset;+ auto cptr = _param.C + COffset;+ return kernel::wrapper::DequanS32Fp32::template forward<ISA_T>(cacheptr, cachestep, cptr, _param.ldc, M, N,+ _param.scalesA + M_offset * _param.ldsa, _param.ldsa,+ _param.scalesB + N_offset);+ }+};++struct ParamZpDequantInt32ToFp32 {+ // necessary+ float* C;+ int ldc;+ int ldsa;+ float* scalesA;+ float* scalesB;+ // optional if A asym+ uint8_t* zpA = nullptr;+ int32_t* reduceB = nullptr;+ // optional if B asym+ int8_t* zpB = nullptr;+ int32_t* reduceA = nullptr;+ int K = 1;+};+class ZpDequantInt32ToFp32 {+ public:+ using Param = ParamZpDequantInt32ToFp32;+ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(const int32_t* cacheptr, const int cachestep, const int M_offset, const int N_offset,+ const int M, const int N, const Param& _param, void* tmpcache, size_t cachesize) {+ auto COffset = M_offset * _param.ldc + N_offset;+ auto cptr = _param.C + COffset;+ auto zpa_ptr = _param.zpA ? _param.zpA + M_offset : nullptr;+ auto zpb_ptr = _param.zpB ? _param.zpB + N_offset : nullptr;+ auto ret = BTLA_CODE::Success;+ if (zpa_ptr || zpb_ptr) {+ ret = kernel::wrapper::RemoveZeroPointBias::template forward<ISA_T>(+ const_cast<int32_t*>(cacheptr), cachestep, M, N, zpa_ptr, zpb_ptr, _param.ldsa, _param.K,+ _param.reduceA + M_offset * _param.ldsa, _param.reduceB + N_offset);+ assert(ret == BTLA_CODE::Success);+ }+ ret = kernel::wrapper::DequanS32Fp32::template forward<ISA_T>(cacheptr, cachestep, cptr, _param.ldc, M, N,+ _param.scalesA + M_offset * _param.ldsa, _param.ldsa,+ _param.scalesB + N_offset);+ if (ret != BTLA_CODE::Success) {+ return ret;+ }+ return ret;+ }+};++struct ParamAlphaBetaProcessS32U8 {+ uint8_t* C;+ int ldc;+ float alpha;+ float scaleAcc, scaleC;+ int zpC;+};+class AlphaBetaProcessS32U8 {+ public:+ using Param = ParamAlphaBetaProcessS32U8;+ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(const int32_t* cacheptr, const int cachestep, const int M_offset, const int N_offset,+ const int M, const int N, const Param& _param, void* tmpcache, size_t cachesize) {+ auto COffset = M_offset * _param.ldc + N_offset;+ auto cptr = _param.C + COffset;+ return kernel::wrapper::QuanOutS32U32::template forward<ISA_T>(_param.alpha, cacheptr, cachestep, cptr, _param.ldc,+ M, N, _param.scaleAcc, _param.scaleC, _param.zpC);+ }+};++} // namespace gemm+} // namespace epilogue+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_gemm.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_gemm.hnew file mode 100644index 000000000..b773fa28c--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_gemm.h@@ -0,0 +1,3125 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include <array>++#include "bestla_utils.h"+#include "bestla_jit.h"++namespace bestla {+namespace gemm {+enum class CompType : uint16_t {+ // base type, too many bits if reuse BTLA_DTYPE+ tFP32 = 0,+ tBF16 = 1,+ tFP16 = 2,+ tS8 = 3,+ tU8 = 4,+ tS32 = 5,+ tS16 = 6,+ MASK_A = 0xf,+ SHIFT_A = 0,+ MASK_B = 0xf0,+ SHIFT_B = 4,+ MASK_C = 0xf00,+ SHIFT_C = 8,+ COMP_FP32 = (tFP32 << SHIFT_A) | (tFP32 << SHIFT_B) | (tFP32 << SHIFT_C),+ COMP_BF16_FP32 = (tBF16 << SHIFT_A) | (tBF16 << SHIFT_B) | (tFP32 << SHIFT_C),+ COMP_FP16_FP16 = (tFP16 << SHIFT_A) | (tFP16 << SHIFT_B) | (tFP16 << SHIFT_C),+ COMP_FP16_FP32 = (tFP16 << SHIFT_A) | (tFP16 << SHIFT_B) | (tFP32 << SHIFT_C),+ COMP_INT8_US_INT32 = (tU8 << SHIFT_A) | (tS8 << SHIFT_B) | (tS32 << SHIFT_C),+ COMP_INT8_UU_INT32 = (tU8 << SHIFT_A) | (tU8 << SHIFT_B) | (tS32 << SHIFT_C),+ COMP_INT8_SS_INT32 = (tS8 << SHIFT_A) | (tS8 << SHIFT_B) | (tS32 << SHIFT_C),+ COMP_INT8_SU_INT32 = (tS8 << SHIFT_A) | (tU8 << SHIFT_B) | (tS32 << SHIFT_C),+ COMP_INT16_SS_INT32 = (tS16 << SHIFT_A) | (tS16 << SHIFT_B) | (tS32 << SHIFT_C),+ COMP_INT8_US_FP32 = (tU8 << SHIFT_A) | (tS8 << SHIFT_B) | (tFP32 << SHIFT_C),+ COMP_INT8_UU_FP32 = (tU8 << SHIFT_A) | (tU8 << SHIFT_B) | (tFP32 << SHIFT_C),+ COMP_INT8_SS_FP32 = (tS8 << SHIFT_A) | (tS8 << SHIFT_B) | (tFP32 << SHIFT_C),+ COMP_INT8_SU_FP32 = (tS8 << SHIFT_A) | (tU8 << SHIFT_B) | (tFP32 << SHIFT_C),+};++class CompTypeHelper {+ public:+ static inline uint64_t get_mask_val(CompType raw, CompType mask, CompType shift) {+ return (static_cast<uint64_t>(raw) & static_cast<uint64_t>(mask)) >> static_cast<uint64_t>(shift);+ }++ static void parse_id(CompType id, uint64_t* vals) {+ vals[0] = get_mask_val(id, CompType::MASK_A, CompType::SHIFT_A);+ vals[1] = get_mask_val(id, CompType::MASK_B, CompType::SHIFT_B);+ vals[2] = get_mask_val(id, CompType::MASK_C, CompType::SHIFT_C);+ }++ static const char* to_str(CompType id) {+ static char tmp[128];+ uint64_t vals[3];+ parse_id(id, vals);+ sprintf(tmp, "A%d_B%d_C%d", static_cast<int>(vals[0]), static_cast<int>(vals[1]), static_cast<int>(vals[2]));+ return tmp;+ }++ static inline uint64_t get_B(CompType id) { return get_mask_val(id, CompType::MASK_B, CompType::SHIFT_B); }++ static inline BTLA_DTYPE to_dtype(CompType _id_type) {+ switch (_id_type) {+ case CompType::tBF16:+ return BTLA_DTYPE::BF16;+ case CompType::tFP16:+ return BTLA_DTYPE::F16;+ case CompType::tFP32:+ return BTLA_DTYPE::F32;+ case CompType::tS8:+ return BTLA_DTYPE::S8;+ case CompType::tU8:+ return BTLA_DTYPE::U8;+ default:+ return BTLA_DTYPE::EleBitsUndef;+ }+ }++ static inline bool is_integer(CompType id) {+ auto bt = get_B(id);+ bool flag = false;+ flag |= bt == static_cast<uint64_t>(CompType::tS8);+ flag |= bt == static_cast<uint64_t>(CompType::tU8);+ return flag;+ }+};++class CoreAttr {+ public:+ // INT64=LSB|**8bits:NTile**||**8bits:PackRow**||**16bits:CompType**||**8bits:ISA**||**24bits:reversed**|+ static uint64_t constexpr NTILE_MASK = 0xff, NTILE_SHIFT = 0, PACKROW_MASK = 0xff00, PACKROW_SHIFT = 8,+ COMP_MASK = 0xffff0000, COMP_SHIFT = 16, ISA_MASK = 0xff00000000, ISA_SHIFT = 32;++ static inline uint64_t get_mask_val(uint64_t raw, uint64_t mask, uint64_t shift) { return (raw & mask) >> shift; }++ static constexpr uint64_t make_core_id(int NTile, int PackRow, CompType CompType, BTLA_ISA ISA) {+ return (static_cast<uint64_t>(NTile) << NTILE_SHIFT) | (static_cast<uint64_t>(PackRow) << PACKROW_SHIFT) |+ (static_cast<uint64_t>(CompType) << COMP_SHIFT) | (static_cast<uint64_t>(ISA) << ISA_SHIFT);+ }+ static void parse_id(uint64_t id, uint64_t* vals) {+ vals[0] = get_mask_val(id, NTILE_MASK, NTILE_SHIFT);+ vals[1] = get_mask_val(id, PACKROW_MASK, PACKROW_SHIFT);+ vals[2] = get_mask_val(id, COMP_MASK, COMP_SHIFT);+ vals[3] = get_mask_val(id, ISA_MASK, ISA_SHIFT);+ }++ static const char* to_str(uint64_t id) {+ static char tmp[128];+ uint64_t vals[4];+ parse_id(id, vals);+ sprintf(tmp, "N%d_PACK%d_COMP%d_ISA%d", static_cast<int>(vals[0]), static_cast<int>(vals[1]),+ static_cast<int>(vals[2]), static_cast<int>(vals[3]));+ return tmp;+ }++ static inline int get_packrow(uint64_t id) { return static_cast<int>(get_mask_val(id, PACKROW_MASK, PACKROW_SHIFT)); }++ static inline size_t get_bsize(uint64_t id) {+ auto packrow = get_packrow(id);+ return size_t(4 / packrow);+ }++ static inline BTLA_ISA get_ISA(uint64_t id) { return static_cast<BTLA_ISA>(get_mask_val(id, ISA_MASK, ISA_SHIFT)); }++ static inline CompType get_comp(uint64_t id) {+ return static_cast<CompType>(get_mask_val(id, COMP_MASK, COMP_SHIFT));+ }+};++namespace code {++template <int _NTILE, int _MTILE = 0>+class Avx2N8P1 : protected bestla::xbyak::JitAvx2 {+ public:+ static int constexpr RegLen = 8, PackRow = 1;+ static_assert(_NTILE % RegLen == 0);+ static int constexpr NRegs = _NTILE / RegLen;+ static int constexpr MRegs = _MTILE == 0 ? (RegCount - 1) / NRegs : _MTILE;+ static_assert(NRegs * MRegs <= RegCount - 1);+ static int constexpr NTILE = RegLen * NRegs, MTILE = MRegs, KTILE = 1;+ static int constexpr KUNROLL = 2;+ static auto constexpr ISA = BTLA_ISA::AVX2;+ static auto constexpr COMPUTE = CompType::COMP_FP32;+ typedef float AType;+ typedef float BType;+ typedef float CType;++ struct params {+ AType* matA;+ int astride;+ BType* matB;+ int bstride;+ CType* matC;+ int cstride;+ int k;+ int n;+ int init;+ };+ typedef long long (*func_t)(params*);++ int CRegCount = 0, BRegCount = 0, ARegCount = 0, TmpRegCount = 0;+ int CReg = 0, BReg = 0, AReg = 0, TmpReg = 0;+ static int constexpr BKStepSize = KTILE * NTILE * sizeof(BType);+ static int constexpr AKStepSize = KTILE * sizeof(AType);++ void generate_code(int _mtile) {+ assign_regs();+ reset();+ generate_mtile(_mtile);+ ready();+ mKernel = getCode<func_t>();+ }+ func_t mKernel = nullptr;++ protected:+ Xbyak::Reg64 parambase;+ Xbyak::Reg64 reg_matAptr;+ Xbyak::Reg64 reg_matBptr;+ Xbyak::Reg64 reg_matCptr;+ Xbyak::Reg64 reg_ksize;+ Xbyak::Reg64 reg_nsize;+ Xbyak::Reg64 reg_cstride;+ Xbyak::Reg64 reg_astride;+ Xbyak::Reg64 reg_iterk;+ Xbyak::Reg64 reg_itern;+ Xbyak::Reg64 reg_tmp;+ Xbyak::Reg64 reg_tmp1;+ Xbyak::Reg64 reg_tmp2;+ Xbyak::Reg64 reg_ret = rax;+ Xbyak::Opmask msk_wr = k1;++ void assign_regs() {+ CRegCount = MRegs * NRegs;+ ARegCount = 1;+ BRegCount = RegCount - ARegCount - CRegCount;+ if (BRegCount < NRegs) {+ BRegCount = 0;+ ARegCount = BRegCount + 1;+ }+ if (BRegCount > NRegs) {+ BRegCount = NRegs;+ }+ CReg = 0;+ BReg = CReg + CRegCount;+ AReg = BReg + BRegCount;+ TmpReg = AReg + ARegCount;+ assert(TmpReg <= RegCount);+ TmpRegCount = RegCount - TmpReg;+ }++ void generate_mtile(int _mtile) {+ inLocalLabel(); // use local label for multiple instance+ Xbyak::util::StackFrame st(this, 1, 10, 16 * 10);+ parambase = st.p[0];+ reg_matAptr = st.t[0];+ reg_matBptr = st.t[1];+ reg_matCptr = st.t[0];+ reg_ksize = st.t[2];+ reg_astride = st.t[3];+ reg_cstride = st.t[3];+ reg_iterk = st.t[4];+ reg_tmp = st.t[5];+ reg_tmp1 = st.t[6];+ reg_tmp2 = st.t[7];+ reg_nsize = st.t[8];+ reg_itern = st.t[9];+ reg_ret = rax;++ vreg_push(rsp);++ load32(reg_ksize, ptr[parambase + OFFSET(k)]);+ load32(reg_nsize, ptr[parambase + OFFSET(n)]);+ xor_(reg_itern, reg_itern);+ L(".nloop");+ init_regs(_mtile);+ mov(reg_matAptr, ptr[parambase + OFFSET(matA)]);+ load32(reg_astride, ptr[parambase + OFFSET(astride)]);+ mov(reg_matBptr, ptr[parambase + OFFSET(matB)]);+ load32(reg_tmp, ptr[parambase + OFFSET(bstride)]);+ imul(reg_tmp, reg_itern);+ lea(reg_matBptr, ptr[reg_matBptr + reg_tmp]);+ xor_(reg_iterk, reg_iterk);+ generate_kloop(_mtile);+ write_back(_mtile);+ add(reg_itern, NTILE);+ cmp(reg_itern, reg_nsize);+ jb(".nloop");+ mov(reg_ret, 0);+ vreg_pop(rsp);++ outLocalLabel(); // end of local label+ }++ void generate_kloop(int _mtile) {+ inLocalLabel();+ mov(reg_tmp, reg_ksize);+ padto_le(reg_tmp, KUNROLL * KTILE);+ cmp(reg_tmp, 0);+ jz(".kloop", T_NEAR);+ L(".unkloop");+ generate_fma(_mtile, KUNROLL);+ add(reg_matAptr, KUNROLL * AKStepSize);+ add(reg_matBptr, KUNROLL * BKStepSize);+ add(reg_iterk, KUNROLL * KTILE);+ cmp(reg_iterk, reg_tmp); // k iteration variable+ jb(".unkloop");+ cmp(reg_tmp, reg_ksize);+ jge(".kend", T_NEAR);+ L(".kloop");+ generate_fma(_mtile, 1);+ add(reg_matAptr, 1 * AKStepSize);+ add(reg_matBptr, 1 * BKStepSize);+ add(reg_iterk, 1 * KTILE);+ cmp(reg_iterk, reg_ksize); // k iteration variable+ jb(".kloop");+ L(".kend");+ outLocalLabel();+ }++ void generate_fma(int _mtile, int _ktile) {+ for (int kk = 0; kk < _ktile; kk++) {+ lea(reg_tmp1, ptr[reg_matAptr + kk * AKStepSize]);+ if (BRegCount == NRegs) {+ for (int i = 0; i < NRegs; i++) {+ vmovups(vreg_t(BReg + i), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ for (int mm = 0; mm < _mtile; mm++) {+ vbroadcastss(vreg_t(AReg), ptr[reg_tmp1]);+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ vfmadd231ps(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg), vreg_t(BReg + i));+ }+ }+ } else if (BRegCount == 0) {+ for (int mm = 0; mm < _mtile; mm += ARegCount) {+ int mm_re = utils::remainsize(mm, _mtile, ARegCount);+ for (int imm = 0; imm < mm_re; imm++) {+ vbroadcastss(vreg_t(AReg + imm), ptr[reg_tmp1]);+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ vfmadd231ps(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg + imm),+ ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ }+ }+ } else {+ assert(0);+ }+ }+ }++ void init_regs(int _mtile) {+ inLocalLabel();+ load32(reg_tmp, ptr[parambase + OFFSET(init)]);+ cmp(reg_tmp, 0);+ je(".read", T_NEAR);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vxor(vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j));+ }+ }+ jmp(".end", T_NEAR);+ L(".read");+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(vreg_t(CReg + i * NRegs + j), ptr[reg_matCptr + j * VecBytes]);+ }+ add(reg_matCptr, reg_cstride);+ }+ L(".end");+ outLocalLabel();+ }++ void write_back(int _mtile) {+ inLocalLabel();+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(ptr[reg_matCptr + j * VecBytes], vreg_t(CReg + i * NRegs + j));+ }+ add(reg_matCptr, reg_cstride);+ }+ outLocalLabel();+ }+};++template <int _NTILE, int _MTILE = 0>+class Avx512fN16P1 : protected bestla::xbyak::JitAvx512f {+ public:+ static int constexpr RegLen = 16, PackRow = 1;+ static_assert(_NTILE % RegLen == 0);+ static int constexpr NRegs = _NTILE / RegLen;+ static int constexpr MRegs = _MTILE == 0 ? (RegCount - 1) / NRegs : _MTILE;+ static_assert(NRegs * MRegs <= RegCount - 1);+ static int constexpr NTILE = RegLen * NRegs, MTILE = MRegs, KTILE = 1;+ static int constexpr KUNROLL = 2;+ static auto constexpr ISA = BTLA_ISA::AVX512F;+ static auto constexpr COMPUTE = CompType::COMP_FP32;+ typedef float AType;+ typedef float BType;+ typedef float CType;++ struct params {+ AType* matA;+ int astride;+ BType* matB;+ int bstride;+ CType* matC;+ int cstride;+ int k;+ int n;+ int init;+ };+ typedef long long (*func_t)(params*);++ int CRegCount = 0, BRegCount = 0, ARegCount = 0, TmpRegCount = 0;+ int CReg = 0, BReg = 0, AReg = 0, TmpReg = 0;+ static int constexpr BKStepSize = KTILE * NTILE * sizeof(BType);+ static int constexpr AKStepSize = KTILE * sizeof(AType);++ void generate_code(int _mtile) {+ assign_regs();+ reset();+ generate_mtile(_mtile);+ ready();+ mKernel = getCode<func_t>();+ }+ func_t mKernel = nullptr;++ protected:+ Xbyak::Reg64 parambase;+ Xbyak::Reg64 reg_matAptr;+ Xbyak::Reg64 reg_matBptr;+ Xbyak::Reg64 reg_matCptr;+ Xbyak::Reg64 reg_ksize;+ Xbyak::Reg64 reg_nsize;+ Xbyak::Reg64 reg_cstride;+ Xbyak::Reg64 reg_astride;+ Xbyak::Reg64 reg_iterk;+ Xbyak::Reg64 reg_itern;+ Xbyak::Reg64 reg_tmp;+ Xbyak::Reg64 reg_tmp1;+ Xbyak::Reg64 reg_tmp2;+ Xbyak::Reg64 reg_ret = rax;+ Xbyak::Opmask msk_wr = k1;++ void assign_regs() {+ CRegCount = MRegs * NRegs;+ ARegCount = 1;+ BRegCount = RegCount - ARegCount - CRegCount;+ if (BRegCount < NRegs) {+ BRegCount = 0;+ ARegCount = BRegCount + 1;+ }+ if (BRegCount > NRegs) {+ BRegCount = NRegs;+ }+ CReg = 0;+ BReg = CReg + CRegCount;+ AReg = BReg + BRegCount;+ TmpReg = AReg + ARegCount;+ assert(TmpReg <= RegCount);+ TmpRegCount = RegCount - TmpReg;+ }++ void generate_mtile(int _mtile) {+ inLocalLabel(); // use local label for multiple instance+ Xbyak::util::StackFrame st(this, 1, 10, 16 * 10);+ parambase = st.p[0];+ reg_matAptr = st.t[0];+ reg_matBptr = st.t[1];+ reg_matCptr = st.t[0];+ reg_ksize = st.t[2];+ reg_astride = st.t[3];+ reg_cstride = st.t[3];+ reg_iterk = st.t[4];+ reg_tmp = st.t[5];+ reg_tmp1 = st.t[6];+ reg_tmp2 = st.t[7];+ reg_nsize = st.t[8];+ reg_itern = st.t[9];+ reg_ret = rax;++ vreg_push(rsp);++ load32(reg_ksize, ptr[parambase + OFFSET(k)]);+ load32(reg_nsize, ptr[parambase + OFFSET(n)]);+ xor_(reg_itern, reg_itern);+ L(".nloop");+ init_regs(_mtile);+ mov(reg_matAptr, ptr[parambase + OFFSET(matA)]);+ load32(reg_astride, ptr[parambase + OFFSET(astride)]);+ mov(reg_matBptr, ptr[parambase + OFFSET(matB)]);+ load32(reg_tmp, ptr[parambase + OFFSET(bstride)]);+ imul(reg_tmp, reg_itern);+ lea(reg_matBptr, ptr[reg_matBptr + reg_tmp]);+ xor_(reg_iterk, reg_iterk);+ generate_kloop(_mtile);+ write_back(_mtile);+ add(reg_itern, NTILE);+ cmp(reg_itern, reg_nsize);+ jb(".nloop");+ mov(reg_ret, 0);+ vreg_pop(rsp);++ outLocalLabel(); // end of local label+ }++ void generate_kloop(int _mtile) {+ inLocalLabel();+ mov(reg_tmp, reg_ksize);+ padto_le(reg_tmp, KUNROLL * KTILE);+ cmp(reg_tmp, 0);+ jz(".kloop", T_NEAR);+ L(".unkloop");+ generate_fma(_mtile, KUNROLL);+ add(reg_matAptr, KUNROLL * AKStepSize);+ add(reg_matBptr, KUNROLL * BKStepSize);+ add(reg_iterk, KUNROLL * KTILE);+ cmp(reg_iterk, reg_tmp); // k iteration variable+ jb(".unkloop");+ cmp(reg_tmp, reg_ksize);+ jge(".kend", T_NEAR);+ L(".kloop");+ generate_fma(_mtile, 1);+ add(reg_matAptr, 1 * AKStepSize);+ add(reg_matBptr, 1 * BKStepSize);+ add(reg_iterk, 1 * KTILE);+ cmp(reg_iterk, reg_ksize); // k iteration variable+ jb(".kloop");+ L(".kend");+ outLocalLabel();+ }++ void generate_fma(int _mtile, int _ktile) {+ for (int kk = 0; kk < _ktile; kk++) {+ lea(reg_tmp1, ptr[reg_matAptr + kk * AKStepSize]);+ if (BRegCount == NRegs) {+ for (int i = 0; i < NRegs; i++) {+ vmovups(vreg_t(BReg + i), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ for (int mm = 0; mm < _mtile; mm++) {+ vbroadcastss(vreg_t(AReg), ptr[reg_tmp1]);+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ vfmadd231ps(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg), vreg_t(BReg + i));+ }+ }+ } else if (BRegCount == 0) {+ for (int mm = 0; mm < _mtile; mm += ARegCount) {+ int mm_re = utils::remainsize(mm, _mtile, ARegCount);+ for (int imm = 0; imm < mm_re; imm++) {+ vbroadcastss(vreg_t(AReg + imm), ptr[reg_tmp1]);+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ vfmadd231ps(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg + imm),+ ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ }+ }+ } else {+ assert(0);+ }+ }+ }++ void init_regs(int _mtile) {+ inLocalLabel();+ load32(reg_tmp, ptr[parambase + OFFSET(init)]);+ cmp(reg_tmp, 0);+ je(".read", T_NEAR);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vxor(vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j));+ }+ }+ jmp(".end", T_NEAR);+ L(".read");+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(vreg_t(CReg + i * NRegs + j), ptr[reg_matCptr + j * VecBytes]);+ }+ add(reg_matCptr, reg_cstride);+ }+ L(".end");+ outLocalLabel();+ }++ void write_back(int _mtile) {+ inLocalLabel();+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(ptr[reg_matCptr + j * VecBytes], vreg_t(CReg + i * NRegs + j));+ }+ add(reg_matCptr, reg_cstride);+ }+ outLocalLabel();+ }+};++template <int _NTILE, int _MTILE = 0>+class Avx512fp16N32P1 : protected bestla::xbyak::JitAvx512_fp16 {+ public:+ static int constexpr RegLen = 32, PackRow = 1;+ static_assert(_NTILE % RegLen == 0);+ static int constexpr NRegs = _NTILE / RegLen;+ static int constexpr MRegs = _MTILE == 0 ? (RegCount - 1) / NRegs : _MTILE;+ static_assert(NRegs * MRegs <= RegCount - 1);+ static int constexpr NTILE = RegLen * NRegs, MTILE = MRegs, KTILE = 1;+ static int constexpr KUNROLL = 2;+ static auto constexpr ISA = BTLA_ISA::AVX512_FP16;+ static auto constexpr COMPUTE = CompType::COMP_FP16_FP16;+ typedef utils::fp16 AType;+ typedef utils::fp16 BType;+ typedef utils::fp16 CType;++ struct params {+ AType* matA;+ int astride;+ BType* matB;+ int bstride;+ CType* matC;+ int cstride;+ int k;+ int n;+ int init;+ };+ typedef long long (*func_t)(params*);++ int CRegCount = 0, BRegCount = 0, ARegCount = 0, TmpRegCount = 0;+ int CReg = 0, BReg = 0, AReg = 0, TmpReg = 0;+ static int constexpr BKStepSize = KTILE * NTILE * sizeof(BType);+ static int constexpr AKStepSize = KTILE * sizeof(AType);++ void generate_code(int _mtile) {+ assign_regs();+ reset();+ generate_mtile(_mtile);+ ready();+ mKernel = getCode<func_t>();+ }+ func_t mKernel = nullptr;++ protected:+ Xbyak::Reg64 parambase;+ Xbyak::Reg64 reg_matAptr;+ Xbyak::Reg64 reg_matBptr;+ Xbyak::Reg64 reg_matCptr;+ Xbyak::Reg64 reg_ksize;+ Xbyak::Reg64 reg_nsize;+ Xbyak::Reg64 reg_cstride;+ Xbyak::Reg64 reg_astride;+ Xbyak::Reg64 reg_iterk;+ Xbyak::Reg64 reg_itern;+ Xbyak::Reg64 reg_tmp;+ Xbyak::Reg64 reg_tmp1;+ Xbyak::Reg64 reg_tmp2;+ Xbyak::Reg64 reg_ret = rax;+ Xbyak::Opmask msk_wr = k1;++ void assign_regs() {+ CRegCount = MRegs * NRegs;+ ARegCount = 1;+ BRegCount = RegCount - ARegCount - CRegCount;+ if (BRegCount < NRegs) {+ BRegCount = 0;+ ARegCount = BRegCount + 1;+ }+ if (BRegCount > NRegs) {+ BRegCount = NRegs;+ }+ CReg = 0;+ BReg = CReg + CRegCount;+ AReg = BReg + BRegCount;+ TmpReg = AReg + ARegCount;+ assert(TmpReg <= RegCount);+ TmpRegCount = RegCount - TmpReg;+ }++ void generate_mtile(int _mtile) {+ inLocalLabel(); // use local label for multiple instance+ Xbyak::util::StackFrame st(this, 1, 10, 16 * 10);+ parambase = st.p[0];+ reg_matAptr = st.t[0];+ reg_matBptr = st.t[1];+ reg_matCptr = st.t[0];+ reg_ksize = st.t[2];+ reg_astride = st.t[3];+ reg_cstride = st.t[3];+ reg_iterk = st.t[4];+ reg_tmp = st.t[5];+ reg_tmp1 = st.t[6];+ reg_tmp2 = st.t[7];+ reg_nsize = st.t[8];+ reg_itern = st.t[9];+ reg_ret = rax;++ vreg_push(rsp);++ load32(reg_ksize, ptr[parambase + OFFSET(k)]);+ load32(reg_nsize, ptr[parambase + OFFSET(n)]);+ xor_(reg_itern, reg_itern);+ L(".nloop");+ init_regs(_mtile);+ mov(reg_matAptr, ptr[parambase + OFFSET(matA)]);+ load32(reg_astride, ptr[parambase + OFFSET(astride)]);+ mov(reg_matBptr, ptr[parambase + OFFSET(matB)]);+ load32(reg_tmp, ptr[parambase + OFFSET(bstride)]);+ imul(reg_tmp, reg_itern);+ lea(reg_matBptr, ptr[reg_matBptr + reg_tmp]);+ xor_(reg_iterk, reg_iterk);+ generate_kloop(_mtile);+ write_back(_mtile);+ add(reg_itern, NTILE);+ cmp(reg_itern, reg_nsize);+ jb(".nloop");+ mov(reg_ret, 0);+ vreg_pop(rsp);++ outLocalLabel(); // end of local label+ }++ void generate_kloop(int _mtile) {+ inLocalLabel();+ mov(reg_tmp, reg_ksize);+ padto_le(reg_tmp, KUNROLL * KTILE);+ cmp(reg_tmp, 0);+ jz(".kloop", T_NEAR);+ L(".unkloop");+ generate_fma(_mtile, KUNROLL);+ add(reg_matAptr, KUNROLL * AKStepSize);+ add(reg_matBptr, KUNROLL * BKStepSize);+ add(reg_iterk, KUNROLL * KTILE);+ cmp(reg_iterk, reg_tmp); // k iteration variable+ jb(".unkloop");+ cmp(reg_tmp, reg_ksize);+ jge(".kend", T_NEAR);+ L(".kloop");+ generate_fma(_mtile, 1);+ add(reg_matAptr, 1 * AKStepSize);+ add(reg_matBptr, 1 * BKStepSize);+ add(reg_iterk, 1 * KTILE);+ cmp(reg_iterk, reg_ksize); // k iteration variable+ jb(".kloop");+ L(".kend");+ outLocalLabel();+ }++ void generate_fma(int _mtile, int _ktile) {+ for (int kk = 0; kk < _ktile; kk++) {+ lea(reg_tmp1, ptr[reg_matAptr + kk * AKStepSize]);+ if (BRegCount == NRegs) {+ for (int i = 0; i < NRegs; i++) {+ vmovups(vreg_t(BReg + i), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ for (int mm = 0; mm < _mtile; mm++) {+ vpbroadcastw(vreg_t(AReg), ptr[reg_tmp1]);+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ vfmadd231ph(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg), vreg_t(BReg + i));+ }+ }+ } else if (BRegCount == 0) {+ for (int mm = 0; mm < _mtile; mm += ARegCount) {+ int mm_re = utils::remainsize(mm, _mtile, ARegCount);+ for (int imm = 0; imm < mm_re; imm++) {+ vpbroadcastw(vreg_t(AReg + imm), ptr[reg_tmp1]);+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ vfmadd231ph(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg + imm),+ ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ }+ }+ } else {+ assert(0);+ }+ }+ }++ void init_regs(int _mtile) {+ inLocalLabel();+ load32(reg_tmp, ptr[parambase + OFFSET(init)]);+ cmp(reg_tmp, 0);+ je(".read", T_NEAR);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vxor(vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j));+ }+ }+ jmp(".end", T_NEAR);+ L(".read");+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(vreg_t(CReg + i * NRegs + j), ptr[reg_matCptr + j * VecBytes]);+ }+ add(reg_matCptr, reg_cstride);+ }+ L(".end");+ outLocalLabel();+ }++ void write_back(int _mtile) {+ inLocalLabel();+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(ptr[reg_matCptr + j * VecBytes], vreg_t(CReg + i * NRegs + j));+ }+ add(reg_matCptr, reg_cstride);+ }+ outLocalLabel();+ }+};++template <int _NTILE, int _MTILE = 0>+class Avx512bf16N16P2 : protected bestla::xbyak::JitAvx512_bf16 {+ public:+ static int constexpr RegLen = 16, PackRow = 2;+ static_assert(_NTILE % RegLen == 0);+ static int constexpr NRegs = _NTILE / RegLen;+ static int constexpr MRegs = _MTILE == 0 ? (RegCount - 1) / NRegs : _MTILE;+ static_assert(NRegs * MRegs <= RegCount - 1);+ static int constexpr NTILE = RegLen * NRegs, MTILE = MRegs, KTILE = 2;+ static int constexpr KUNROLL = 2;+ static auto constexpr ISA = BTLA_ISA::AVX512_BF16;+ static auto constexpr COMPUTE = CompType::COMP_BF16_FP32;+ typedef utils::bf16 AType;+ typedef utils::bf16 BType;+ typedef float CType;++ struct params {+ AType* matA;+ int astride;+ BType* matB;+ int bstride;+ CType* matC;+ int cstride;+ int k;+ int n;+ int init;+ };+ typedef long long (*func_t)(params*);++ int CRegCount = 0, BRegCount = 0, ARegCount = 0, TmpRegCount = 0;+ int CReg = 0, BReg = 0, AReg = 0, TmpReg = 0;+ static int constexpr BKStepSize = KTILE * NTILE * sizeof(BType);+ static int constexpr AKStepSize = KTILE * sizeof(AType);++ void generate_code(int _mtile) {+ assign_regs();+ reset();+ generate_mtile(_mtile);+ ready();+ mKernel = getCode<func_t>();+ }+ func_t mKernel = nullptr;++ protected:+ Xbyak::Reg64 parambase;+ Xbyak::Reg64 reg_matAptr;+ Xbyak::Reg64 reg_matBptr;+ Xbyak::Reg64 reg_matCptr;+ Xbyak::Reg64 reg_ksize;+ Xbyak::Reg64 reg_nsize;+ Xbyak::Reg64 reg_cstride;+ Xbyak::Reg64 reg_astride;+ Xbyak::Reg64 reg_iterk;+ Xbyak::Reg64 reg_itern;+ Xbyak::Reg64 reg_tmp;+ Xbyak::Reg64 reg_tmp1;+ Xbyak::Reg64 reg_tmp2;+ Xbyak::Reg64 reg_ret = rax;+ Xbyak::Opmask msk_wr = k1;++ void assign_regs() {+ CRegCount = MRegs * NRegs;+ ARegCount = 1;+ BRegCount = RegCount - ARegCount - CRegCount;+ if (BRegCount < NRegs) {+ BRegCount = 0;+ ARegCount = BRegCount + 1;+ }+ if (BRegCount > NRegs) {+ BRegCount = NRegs;+ }+ CReg = 0;+ BReg = CReg + CRegCount;+ AReg = BReg + BRegCount;+ TmpReg = AReg + ARegCount;+ assert(TmpReg <= RegCount);+ TmpRegCount = RegCount - TmpReg;+ }++ void generate_mtile(int _mtile) {+ inLocalLabel(); // use local label for multiple instance+ Xbyak::util::StackFrame st(this, 1, 10, 16 * 10);+ parambase = st.p[0];+ reg_matAptr = st.t[0];+ reg_matBptr = st.t[1];+ reg_matCptr = st.t[0];+ reg_ksize = st.t[2];+ reg_astride = st.t[3];+ reg_cstride = st.t[3];+ reg_iterk = st.t[4];+ reg_tmp = st.t[5];+ reg_tmp1 = st.t[6];+ reg_tmp2 = st.t[7];+ reg_nsize = st.t[8];+ reg_itern = st.t[9];+ reg_ret = rax;++ vreg_push(rsp);++ load32(reg_ksize, ptr[parambase + OFFSET(k)]);+ load32(reg_nsize, ptr[parambase + OFFSET(n)]);+ xor_(reg_itern, reg_itern);+ L(".nloop");+ init_regs(_mtile);+ mov(reg_matAptr, ptr[parambase + OFFSET(matA)]);+ load32(reg_astride, ptr[parambase + OFFSET(astride)]);+ mov(reg_matBptr, ptr[parambase + OFFSET(matB)]);+ load32(reg_tmp, ptr[parambase + OFFSET(bstride)]);+ imul(reg_tmp, reg_itern);+ lea(reg_matBptr, ptr[reg_matBptr + reg_tmp]);+ xor_(reg_iterk, reg_iterk);+ generate_kloop(_mtile);+ write_back(_mtile);+ add(reg_itern, NTILE);+ cmp(reg_itern, reg_nsize);+ jb(".nloop");+ mov(reg_ret, 0);+ vreg_pop(rsp);++ outLocalLabel(); // end of local label+ }++ void generate_kloop(int _mtile) {+ inLocalLabel();+ mov(reg_tmp, reg_ksize);+ padto_le(reg_tmp, KUNROLL * KTILE);+ cmp(reg_tmp, 0);+ jz(".kloop", T_NEAR);+ L(".unkloop");+ generate_fma(_mtile, KUNROLL);+ add(reg_matAptr, KUNROLL * AKStepSize);+ add(reg_matBptr, KUNROLL * BKStepSize);+ add(reg_iterk, KUNROLL * KTILE);+ cmp(reg_iterk, reg_tmp); // k iteration variable+ jb(".unkloop");+ cmp(reg_tmp, reg_ksize);+ jge(".kend", T_NEAR);+ L(".kloop");+ generate_fma(_mtile, 1);+ add(reg_matAptr, 1 * AKStepSize);+ add(reg_matBptr, 1 * BKStepSize);+ add(reg_iterk, 1 * KTILE);+ cmp(reg_iterk, reg_ksize); // k iteration variable+ jb(".kloop");+ L(".kend");+ outLocalLabel();+ }++ void generate_fma(int _mtile, int _ktile) {+ for (int kk = 0; kk < _ktile; kk++) {+ lea(reg_tmp1, ptr[reg_matAptr + kk * AKStepSize]);+ if (BRegCount == NRegs) {+ for (int i = 0; i < NRegs; i++) {+ vmovups(vreg_t(BReg + i), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ for (int mm = 0; mm < _mtile; mm++) {+ vbroadcastss(vreg_t(AReg), ptr[reg_tmp1]);+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ vdpbf16ps(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg), vreg_t(BReg + i));+ }+ }+ } else if (BRegCount == 0) {+ for (int mm = 0; mm < _mtile; mm += ARegCount) {+ int mm_re = utils::remainsize(mm, _mtile, ARegCount);+ for (int imm = 0; imm < mm_re; imm++) {+ vbroadcastss(vreg_t(AReg + imm), ptr[reg_tmp1]);+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ vdpbf16ps(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg + imm),+ ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ }+ }+ } else {+ assert(0);+ }+ }+ }++ void init_regs(int _mtile) {+ inLocalLabel();+ load32(reg_tmp, ptr[parambase + OFFSET(init)]);+ cmp(reg_tmp, 0);+ je(".read", T_NEAR);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vxor(vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j));+ }+ }+ jmp(".end", T_NEAR);+ L(".read");+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(vreg_t(CReg + i * NRegs + j), ptr[reg_matCptr + j * VecBytes]);+ }+ add(reg_matCptr, reg_cstride);+ }+ L(".end");+ outLocalLabel();+ }++ void write_back(int _mtile) {+ inLocalLabel();+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(ptr[reg_matCptr + j * VecBytes], vreg_t(CReg + i * NRegs + j));+ }+ add(reg_matCptr, reg_cstride);+ }+ outLocalLabel();+ }+};++template <int _NTILE, int _MTILE = 0>+class Avx512vnniN16P4 : protected bestla::xbyak::JitAvx512vnni {+ public:+ static int constexpr RegLen = 16, PackRow = 4;+ static_assert(_NTILE % RegLen == 0);+ static int constexpr NRegs = _NTILE / RegLen;+ static int constexpr MRegs = _MTILE == 0 ? (RegCount - 1) / NRegs : _MTILE;+ static_assert(NRegs * MRegs <= RegCount - 1);+ static int constexpr NTILE = RegLen * NRegs, MTILE = MRegs, KTILE = 4;+ static int constexpr KUNROLL = 2;+ static auto constexpr ISA = BTLA_ISA::AVX512_VNNI;+ static auto constexpr COMPUTE = CompType::COMP_INT8_US_INT32;+ typedef uint8_t AType;+ typedef int8_t BType;+ typedef int32_t CType;+ struct params {+ AType* matA;+ int astride;+ BType* matB;+ int bstride;+ CType* matC;+ int cstride;+ int k;+ int n;+ int init;+ };+ typedef long long (*func_t)(params*);++ int CRegCount = 0, BRegCount = 0, ARegCount = 0, TmpRegCount = 0;+ int CReg = 0, BReg = 0, AReg = 0, TmpReg = 0;+ static int constexpr BKStepSize = KTILE * NTILE * sizeof(BType);+ static int constexpr AKStepSize = KTILE * sizeof(AType);++ void generate_code(int _mtile) {+ assign_regs();+ reset();+ generate_mtile(_mtile);+ ready();+ mKernel = getCode<func_t>();+ }+ func_t mKernel = nullptr;++ private:+ Xbyak::Reg64 parambase;+ Xbyak::Reg64 reg_matAptr;+ Xbyak::Reg64 reg_matBptr;+ Xbyak::Reg64 reg_matCptr;+ Xbyak::Reg64 reg_ksize;+ Xbyak::Reg64 reg_nsize;+ Xbyak::Reg64 reg_cstride;+ Xbyak::Reg64 reg_astride;+ Xbyak::Reg64 reg_iterk;+ Xbyak::Reg64 reg_itern;+ Xbyak::Reg64 reg_tmp;+ Xbyak::Reg64 reg_tmp1;+ Xbyak::Reg64 reg_tmp2;+ Xbyak::Reg64 reg_ret = rax;++ protected:+ void assign_regs() {+ CRegCount = MRegs * NRegs;+ ARegCount = 1;+ BRegCount = RegCount - ARegCount - CRegCount;+ if (BRegCount < NRegs) {+ BRegCount = 0;+ ARegCount = BRegCount + 1;+ }+ if (BRegCount > NRegs) {+ BRegCount = NRegs;+ }+ CReg = 0;+ BReg = CReg + CRegCount;+ AReg = BReg + BRegCount;+ TmpReg = AReg + ARegCount;+ assert(TmpReg <= RegCount);+ TmpRegCount = RegCount - TmpReg;+ }++ void generate_mtile(int _mtile) {+ inLocalLabel();+ Xbyak::util::StackFrame st(this, 1, 10, 16 * 10);+ parambase = st.p[0];+ reg_matAptr = st.t[0];+ reg_matBptr = st.t[1];+ reg_matCptr = st.t[0];+ reg_ksize = st.t[2];+ reg_astride = st.t[3];+ reg_cstride = st.t[3];+ reg_iterk = st.t[4];+ reg_tmp = st.t[5];+ reg_tmp1 = st.t[6];+ reg_tmp2 = st.t[7];+ reg_nsize = st.t[8];+ reg_itern = st.t[9];+ reg_ret = rax;++ vreg_push(rsp);++ load32(reg_ksize, ptr[parambase + OFFSET(k)]);+ load32(reg_nsize, ptr[parambase + OFFSET(n)]);+ xor_(reg_itern, reg_itern);+ L(".nloop");+ init_regs(_mtile);+ mov(reg_matAptr, ptr[parambase + OFFSET(matA)]);+ load32(reg_astride, ptr[parambase + OFFSET(astride)]);+ mov(reg_matBptr, ptr[parambase + OFFSET(matB)]);+ load32(reg_tmp, ptr[parambase + OFFSET(bstride)]);+ imul(reg_tmp, reg_itern);+ lea(reg_matBptr, ptr[reg_matBptr + reg_tmp]);+ xor_(reg_iterk, reg_iterk);+ generate_kloop(_mtile);+ write_back(_mtile);+ add(reg_itern, NTILE);+ cmp(reg_itern, reg_nsize);+ jb(".nloop");+ mov(reg_ret, 0);+ vreg_pop(rsp);++ outLocalLabel(); // end of local label+ }++ void generate_kloop(int _mtile) {+ inLocalLabel();+ mov(reg_tmp, reg_ksize);+ padto_le(reg_tmp, KUNROLL * KTILE);+ cmp(reg_tmp, 0);+ jz(".kloop", T_NEAR);+ L(".unkloop");+ generate_fma(_mtile, KUNROLL);+ add(reg_matAptr, KUNROLL * AKStepSize);+ add(reg_matBptr, KUNROLL * BKStepSize);+ add(reg_iterk, KUNROLL * KTILE);+ cmp(reg_iterk, reg_tmp); // k iteration variable+ jb(".unkloop");+ cmp(reg_tmp, reg_ksize);+ jge(".kend", T_NEAR);+ L(".kloop");+ generate_fma(_mtile, 1);+ add(reg_matAptr, 1 * AKStepSize);+ add(reg_matBptr, 1 * BKStepSize);+ add(reg_iterk, 1 * KTILE);+ cmp(reg_iterk, reg_ksize); // k iteration variable+ jb(".kloop");+ L(".kend");+ outLocalLabel();+ }++ void generate_fma(int _mtile, int _kunroll) {+ for (int kk = 0; kk < _kunroll; kk++) {+ lea(reg_tmp1, ptr[reg_matAptr + kk * AKStepSize]);+ if (BRegCount == NRegs) {+ for (int i = 0; i < NRegs; i++) {+ vmovups(vreg_t(BReg + i), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ for (int mm = 0; mm < _mtile; mm++) {+ vpbroadcastd(vreg_t(AReg), ptr[reg_tmp1]);+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg), vreg_t(BReg + i));+ }+ }+ } else if (BRegCount == 0) {+ for (int mm = 0; mm < _mtile; mm += ARegCount) {+ int mm_re = utils::remainsize(mm, _mtile, ARegCount);+ for (int imm = 0; imm < mm_re; imm++) {+ vpbroadcastd(vreg_t(AReg + imm), ptr[reg_tmp1]);+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg + imm),+ ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ }+ }+ } else {+ assert(0);+ }+ }+ }++ void init_regs(int _mtile) {+ inLocalLabel();+ load32(reg_tmp, ptr[parambase + OFFSET(init)]);+ cmp(reg_tmp, 0);+ je(".read", T_NEAR);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vxor(vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j));+ }+ }+ jmp(".end", T_NEAR);+ L(".read");+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(vreg_t(CReg + i * NRegs + j), ptr[reg_matCptr + j * VecBytes]);+ }+ add(reg_matCptr, reg_cstride);+ }+ L(".end");+ outLocalLabel();+ }++ void write_back(int _mtile) {+ inLocalLabel();+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(ptr[reg_matCptr + j * VecBytes], vreg_t(CReg + i * NRegs + j));+ }+ add(reg_matCptr, reg_cstride);+ }+ outLocalLabel();+ }+};++template <int _NTILE, int _MTILE = 0>+class Avx512bwN16P4 : protected bestla::xbyak::JitAvx512bw {+ public:+ static int constexpr RegLen = 16, PackRow = 4;+ static_assert(_NTILE % RegLen == 0);+ static int constexpr NRegs = _NTILE / RegLen;+ static int constexpr KeepRegs = 3;+ static int constexpr MRegs = _MTILE == 0 ? (RegCount - KeepRegs) / NRegs : _MTILE;+ static_assert(NRegs * MRegs <= RegCount - 1);+ static int constexpr NTILE = RegLen * NRegs, MTILE = MRegs, KTILE = 4;+ static int constexpr KUNROLL = 2;+ static auto constexpr ISA = BTLA_ISA::AVX512BW;+ static auto constexpr COMPUTE = CompType::COMP_INT8_US_INT32;+ typedef uint8_t AType;+ typedef int8_t BType;+ typedef int32_t CType;+ struct params {+ AType* matA;+ int astride;+ BType* matB;+ int bstride;+ CType* matC;+ int cstride;+ int k;+ int n;+ int init;+ const int16_t one = 1;+ };+ typedef long long (*func_t)(params*);++ int CRegCount = 0, BRegCount = 0, ARegCount = 0, TmpRegCount = 0;+ int CReg = 0, BReg = 0, AReg = 0, TmpReg = 0;+ static int constexpr BKStepSize = KTILE * NTILE * sizeof(BType);+ static int constexpr AKStepSize = KTILE * sizeof(AType);++ void generate_code(int _mtile) {+ assign_regs();+ reset();+ generate_mtile(_mtile);+ ready();+ mKernel = getCode<func_t>();+ }+ func_t mKernel = nullptr;++ private:+ Xbyak::Reg64 parambase;+ Xbyak::Reg64 reg_matAptr;+ Xbyak::Reg64 reg_matBptr;+ Xbyak::Reg64 reg_matCptr;+ Xbyak::Reg64 reg_ksize;+ Xbyak::Reg64 reg_nsize;+ Xbyak::Reg64 reg_cstride;+ Xbyak::Reg64 reg_astride;+ Xbyak::Reg64 reg_iterk;+ Xbyak::Reg64 reg_itern;+ Xbyak::Reg64 reg_tmp;+ Xbyak::Reg64 reg_tmp1;+ Xbyak::Reg64 reg_tmp2;+ Xbyak::Reg64 reg_ret = rax;++ protected:+ void assign_regs() {+ CRegCount = MRegs * NRegs;+ ARegCount = 1;+ TmpRegCount = 2;++ BRegCount = RegCount - ARegCount - CRegCount - TmpRegCount;+ if (BRegCount < NRegs) {+ BRegCount = 0;+ ARegCount = BRegCount + 1;+ }+ if (BRegCount > NRegs) {+ BRegCount = NRegs;+ }+ CReg = 0;+ BReg = CReg + CRegCount;+ AReg = BReg + BRegCount;+ TmpReg = AReg + ARegCount;+ assert(TmpReg + TmpRegCount <= RegCount);+ }++ void generate_mtile(int _mtile) {+ inLocalLabel();+ Xbyak::util::StackFrame st(this, 1, 10, 16 * 10);+ parambase = st.p[0];+ reg_matAptr = st.t[0];+ reg_matBptr = st.t[1];+ reg_matCptr = st.t[0];+ reg_ksize = st.t[2];+ reg_astride = st.t[3];+ reg_cstride = st.t[3];+ reg_iterk = st.t[4];+ reg_tmp = st.t[5];+ reg_tmp1 = st.t[6];+ reg_tmp2 = st.t[7];+ reg_nsize = st.t[8];+ reg_itern = st.t[9];+ reg_ret = rax;++ vreg_push(rsp);+ vpbroadcastw(vreg_t(TmpReg + 0), ptr[parambase + OFFSET(one)]);+ load32(reg_ksize, ptr[parambase + OFFSET(k)]);+ load32(reg_nsize, ptr[parambase + OFFSET(n)]);+ xor_(reg_itern, reg_itern);+ L(".nloop");+ init_regs(_mtile);+ mov(reg_matAptr, ptr[parambase + OFFSET(matA)]);+ load32(reg_astride, ptr[parambase + OFFSET(astride)]);+ mov(reg_matBptr, ptr[parambase + OFFSET(matB)]);+ load32(reg_tmp, ptr[parambase + OFFSET(bstride)]);+ imul(reg_tmp, reg_itern);+ lea(reg_matBptr, ptr[reg_matBptr + reg_tmp]);+ xor_(reg_iterk, reg_iterk);+ generate_kloop(_mtile);+ write_back(_mtile);+ add(reg_itern, NTILE);+ cmp(reg_itern, reg_nsize);+ jb(".nloop");+ mov(reg_ret, 0);+ vreg_pop(rsp);++ outLocalLabel(); // end of local label+ }++ void generate_kloop(int _mtile) {+ inLocalLabel();+ mov(reg_tmp, reg_ksize);+ padto_le(reg_tmp, KUNROLL * KTILE);+ cmp(reg_tmp, 0);+ jz(".kloop", T_NEAR);+ L(".unkloop");+ generate_fma(_mtile, KUNROLL);+ add(reg_matAptr, KUNROLL * AKStepSize);+ add(reg_matBptr, KUNROLL * BKStepSize);+ add(reg_iterk, KUNROLL * KTILE);+ cmp(reg_iterk, reg_tmp); // k iteration variable+ jb(".unkloop");+ cmp(reg_tmp, reg_ksize);+ jge(".kend", T_NEAR);+ L(".kloop");+ generate_fma(_mtile, 1);+ add(reg_matAptr, 1 * AKStepSize);+ add(reg_matBptr, 1 * BKStepSize);+ add(reg_iterk, 1 * KTILE);+ cmp(reg_iterk, reg_ksize); // k iteration variable+ jb(".kloop");+ L(".kend");+ outLocalLabel();+ }+ void generate_fma(int _mtile, int _kunroll) {+ for (int kk = 0; kk < _kunroll; kk++) {+ lea(reg_tmp1, ptr[reg_matAptr + kk * AKStepSize]);+ if (BRegCount == NRegs) {+ for (int i = 0; i < NRegs; i++) {+ vmovups(vreg_t(BReg + i), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ for (int mm = 0; mm < _mtile; mm++) {+ vpbroadcastd(vreg_t(AReg), ptr[reg_tmp1]);+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(TmpReg + 1), vreg_t(AReg), vreg_t(BReg + i),+ vreg_t(TmpReg + 0));+ }+ }+ } else if (BRegCount == 0) {+ for (int mm = 0; mm < _mtile; mm += ARegCount) {+ int mm_re = utils::remainsize(mm, _mtile, ARegCount);+ for (int imm = 0; imm < mm_re; imm++) {+ vpbroadcastd(vreg_t(AReg + imm), ptr[reg_tmp1]);+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(TmpReg + 1), vreg_t(AReg + imm),+ ptr[reg_matBptr + kk * BKStepSize + i * VecBytes], vreg_t(TmpReg + 0));+ }+ }+ }+ } else {+ assert(0);+ }+ }+ }++ void init_regs(int _mtile) {+ inLocalLabel();+ load32(reg_tmp, ptr[parambase + OFFSET(init)]);+ cmp(reg_tmp, 0);+ je(".read", T_NEAR);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vxor(vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j));+ }+ }+ jmp(".end", T_NEAR);+ L(".read");+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(vreg_t(CReg + i * NRegs + j), ptr[reg_matCptr + j * VecBytes]);+ }+ add(reg_matCptr, reg_cstride);+ }+ L(".end");+ outLocalLabel();+ }++ void write_back(int _mtile) {+ inLocalLabel();+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(ptr[reg_matCptr + j * VecBytes], vreg_t(CReg + i * NRegs + j));+ }+ add(reg_matCptr, reg_cstride);+ }+ outLocalLabel();+ }+};++template <typename AT, int _NTILE, int _MTILE = 0>+class AvxvnniN8P4 : protected bestla::xbyak::JitAvxvnni {+ public:+ static int constexpr RegLen = 8, PackRow = 4;+ static_assert(_NTILE % RegLen == 0);+ static int constexpr NRegs = _NTILE / RegLen;+ static int constexpr KeepRegs = std::is_same_v<AT, uint8_t> ? 1 : 3;+ static int constexpr MRegs = _MTILE == 0 ? (RegCount - KeepRegs) / NRegs : _MTILE;+ static_assert(NRegs * MRegs <= RegCount - KeepRegs);+ static int constexpr NTILE = RegLen * NRegs, MTILE = MRegs, KTILE = 4;+ static int constexpr KUNROLL = 2;+ static auto constexpr ISA = BTLA_ISA::AVX_VNNI;+ static auto constexpr COMPUTE =+ std::is_same_v<AT, uint8_t> ? CompType::COMP_INT8_US_INT32 : CompType::COMP_INT8_SS_INT32;+ using AType = AT;+ typedef int8_t BType;+ typedef int32_t CType;+ struct params {+ AType* matA;+ int astride;+ BType* matB;+ int bstride;+ CType* matC;+ int cstride;+ int k;+ int n;+ int init;+ };+ typedef long long (*func_t)(params*);++ int CRegCount = 0, BRegCount = 0, ARegCount = 0, TmpRegCount = 0;+ int CReg = 0, BReg = 0, AReg = 0, TmpReg = 0;+ static int constexpr BKStepSize = KTILE * NTILE * sizeof(BType);+ static int constexpr AKStepSize = KTILE * sizeof(AType);++ void generate_code(int _mtile) {+ assign_regs();+ reset();+ generate_mtile(_mtile);+ ready();+ mKernel = getCode<func_t>();+ }+ func_t mKernel = nullptr;++ private:+ Xbyak::Reg64 parambase;+ Xbyak::Reg64 reg_matAptr;+ Xbyak::Reg64 reg_matBptr;+ Xbyak::Reg64 reg_matCptr;+ Xbyak::Reg64 reg_ksize;+ Xbyak::Reg64 reg_nsize;+ Xbyak::Reg64 reg_cstride;+ Xbyak::Reg64 reg_astride;+ Xbyak::Reg64 reg_iterk;+ Xbyak::Reg64 reg_itern;+ Xbyak::Reg64 reg_tmp;+ Xbyak::Reg64 reg_tmp1;+ Xbyak::Reg64 reg_tmp2;+ Xbyak::Reg64 reg_ret = rax;+ Xbyak::Opmask msk_wr = k1;++ protected:+ void assign_regs() {+ CRegCount = MRegs * NRegs;+ ARegCount = 1;+ if (std::is_same_v<AT, int8_t>) {+ TmpRegCount = 2;+ }+ BRegCount = RegCount - ARegCount - CRegCount - TmpRegCount;+ if (BRegCount < NRegs) {+ BRegCount = 0;+ ARegCount = BRegCount + 1;+ }+ if (BRegCount > NRegs) {+ BRegCount = NRegs;+ }+ CReg = 0;+ BReg = CReg + CRegCount;+ AReg = BReg + BRegCount;+ TmpReg = AReg + ARegCount;+ assert(TmpReg + TmpRegCount <= RegCount);+ }++ void generate_mtile(int _mtile) {+ inLocalLabel();+ Xbyak::util::StackFrame st(this, 1, 10, 16 * 10);+ parambase = st.p[0];+ reg_matAptr = st.t[0];+ reg_matBptr = st.t[1];+ reg_matCptr = st.t[0];+ reg_ksize = st.t[2];+ reg_astride = st.t[3];+ reg_cstride = st.t[3];+ reg_iterk = st.t[4];+ reg_tmp = st.t[5];+ reg_tmp1 = st.t[6];+ reg_tmp2 = st.t[7];+ reg_nsize = st.t[8];+ reg_itern = st.t[9];+ reg_ret = rax;++ vreg_push(rsp);++ load32(reg_ksize, ptr[parambase + OFFSET(k)]);+ load32(reg_nsize, ptr[parambase + OFFSET(n)]);+ xor_(reg_itern, reg_itern);+ L(".nloop");+ init_regs(_mtile);+ mov(reg_matAptr, ptr[parambase + OFFSET(matA)]);+ load32(reg_astride, ptr[parambase + OFFSET(astride)]);+ mov(reg_matBptr, ptr[parambase + OFFSET(matB)]);+ load32(reg_tmp, ptr[parambase + OFFSET(bstride)]);+ imul(reg_tmp, reg_itern);+ lea(reg_matBptr, ptr[reg_matBptr + reg_tmp]);+ xor_(reg_iterk, reg_iterk);+ generate_kloop(_mtile);+ write_back(_mtile);+ add(reg_itern, NTILE);+ cmp(reg_itern, reg_nsize);+ jb(".nloop");+ mov(reg_ret, 0);+ vreg_pop(rsp);++ outLocalLabel(); // end of local label+ }++ void generate_kloop(int _mtile) {+ inLocalLabel();+ mov(reg_tmp, reg_ksize);+ padto_le(reg_tmp, KUNROLL * KTILE);+ cmp(reg_tmp, 0);+ jz(".kloop", T_NEAR);+ L(".unkloop");+ generate_fma(_mtile, KUNROLL);+ add(reg_matAptr, KUNROLL * AKStepSize);+ add(reg_matBptr, KUNROLL * BKStepSize);+ add(reg_iterk, KUNROLL * KTILE);+ cmp(reg_iterk, reg_tmp); // k iteration variable+ jb(".unkloop");+ cmp(reg_tmp, reg_ksize);+ jge(".kend", T_NEAR);+ L(".kloop");+ generate_fma(_mtile, 1);+ add(reg_matAptr, 1 * AKStepSize);+ add(reg_matBptr, 1 * BKStepSize);+ add(reg_iterk, 1 * KTILE);+ cmp(reg_iterk, reg_ksize); // k iteration variable+ jb(".kloop");+ L(".kend");+ outLocalLabel();+ }++ void generate_fma(int _mtile, int _kunroll) {+ for (int kk = 0; kk < _kunroll; kk++) {+ lea(reg_tmp1, ptr[reg_matAptr + kk * AKStepSize]);+ if (BRegCount == NRegs) {+ for (int i = 0; i < NRegs; i++) {+ vmovups(vreg_t(BReg + i), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ for (int mm = 0; mm < _mtile; mm++) {+ vpbroadcastd(vreg_t(AReg), ptr[reg_tmp1]);+ if constexpr (std::is_same_v<AType, int8_t>) {+ vpsignb(vreg_t(TmpReg + 1), vreg_t(AReg), vreg_t(AReg));+ }+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ if constexpr (std::is_same_v<AType, uint8_t>) {+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg), vreg_t(BReg + i));+ } else {+ vpsignb(vreg_t(TmpReg), vreg_t(BReg + i), vreg_t(AReg));+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(TmpReg + 1), vreg_t(TmpReg));+ }+ }+ }+ } else if (BRegCount == 0) {+ for (int mm = 0; mm < _mtile; mm += ARegCount) {+ int mm_re = utils::remainsize(mm, _mtile, ARegCount);+ for (int imm = 0; imm < mm_re; imm++) {+ vpbroadcastd(vreg_t(AReg + imm), ptr[reg_tmp1]);+ if constexpr (std::is_same_v<AType, int8_t>) {+ vpsignb(vreg_t(TmpReg + 1), vreg_t(AReg + imm), vreg_t(AReg + imm));+ }+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ if constexpr (std::is_same_v<AType, uint8_t>) {+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg + imm),+ ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ } else {+ vmovups(vreg_t(TmpReg), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ vpsignb(vreg_t(TmpReg), vreg_t(TmpReg), vreg_t(AReg + imm));+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(TmpReg + 1), vreg_t(TmpReg));+ }+ }+ }+ }+ } else {+ assert(0);+ }+ }+ }++ void init_regs(int _mtile) {+ inLocalLabel();+ load32(reg_tmp, ptr[parambase + OFFSET(init)]);+ cmp(reg_tmp, 0);+ je(".read", T_NEAR);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vxor(vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j));+ }+ }+ jmp(".end", T_NEAR);+ L(".read");+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(vreg_t(CReg + i * NRegs + j), ptr[reg_matCptr + j * VecBytes]);+ }+ add(reg_matCptr, reg_cstride);+ }+ L(".end");+ outLocalLabel();+ }++ void write_back(int _mtile) {+ inLocalLabel();+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(ptr[reg_matCptr + j * VecBytes], vreg_t(CReg + i * NRegs + j));+ }+ add(reg_matCptr, reg_cstride);+ }+ outLocalLabel();+ }+};++template <int N, int M>+using AvxvnniN8P4U8 = AvxvnniN8P4<uint8_t, N, M>;++template <int N, int M>+using AvxvnniN8P4S8 = AvxvnniN8P4<int8_t, N, M>;++template <typename AT, int _NTILE, int _MTILE = 0>+class Avx2vnniN8P4 : protected bestla::xbyak::JitAvx2 {+ public:+ static int constexpr RegLen = 8, PackRow = 4;+ static_assert(_NTILE % RegLen == 0);+ static int constexpr NRegs = _NTILE / RegLen;+ static int constexpr KeepRegs = std::is_same_v<AT, uint8_t> ? 3 : 5;+ static int constexpr MRegs = _MTILE == 0 ? (RegCount - KeepRegs) / NRegs : _MTILE;+ static_assert(NRegs * MRegs <= RegCount - KeepRegs);+ static int constexpr NTILE = RegLen * NRegs, MTILE = MRegs, KTILE = 4;+ static int constexpr KUNROLL = 2;+ static auto constexpr ISA = BTLA_ISA::AVX2;+ static auto constexpr COMPUTE =+ std::is_same_v<AT, uint8_t> ? CompType::COMP_INT8_US_INT32 : CompType::COMP_INT8_SS_INT32;+ using AType = AT;+ typedef int8_t BType;+ typedef int32_t CType;+ struct params {+ AType* matA;+ int astride;+ BType* matB;+ int bstride;+ CType* matC;+ int cstride;+ int k;+ int n;+ int init;+ const int16_t one = 1;+ };+ typedef long long (*func_t)(params*);++ int CRegCount = 0, BRegCount = 0, ARegCount = 0, TmpRegCount = 0;+ int CReg = 0, BReg = 0, AReg = 0, TmpReg = 0;+ static int constexpr BKStepSize = KTILE * NTILE * sizeof(BType);+ static int constexpr AKStepSize = KTILE * sizeof(AType);++ void generate_code(int _mtile) {+ assign_regs();+ reset();+ generate_mtile(_mtile);+ ready();+ mKernel = getCode<func_t>();+ }+ func_t mKernel = nullptr;++ private:+ Xbyak::Reg64 parambase;+ Xbyak::Reg64 reg_matAptr;+ Xbyak::Reg64 reg_matBptr;+ Xbyak::Reg64 reg_matCptr;+ Xbyak::Reg64 reg_ksize;+ Xbyak::Reg64 reg_nsize;+ Xbyak::Reg64 reg_cstride;+ Xbyak::Reg64 reg_astride;+ Xbyak::Reg64 reg_iterk;+ Xbyak::Reg64 reg_itern;+ Xbyak::Reg64 reg_tmp;+ Xbyak::Reg64 reg_tmp1;+ Xbyak::Reg64 reg_tmp2;+ Xbyak::Reg64 reg_ret = rax;+ Xbyak::Opmask msk_wr = k1;++ protected:+ void assign_regs() {+ CRegCount = MRegs * NRegs;+ ARegCount = 1;+ if (std::is_same_v<AT, int8_t>) {+ TmpRegCount = 4;+ } else {+ TmpRegCount = 2;+ }+ BRegCount = RegCount - ARegCount - CRegCount - TmpRegCount;+ if (BRegCount < NRegs) {+ BRegCount = 0;+ ARegCount = BRegCount + 1;+ }+ if (BRegCount > NRegs) {+ BRegCount = NRegs;+ }+ CReg = 0;+ BReg = CReg + CRegCount;+ AReg = BReg + BRegCount;+ TmpReg = AReg + ARegCount;+ assert(TmpReg + TmpRegCount <= RegCount);+ }++ void generate_mtile(int _mtile) {+ inLocalLabel();+ Xbyak::util::StackFrame st(this, 1, 10, 16 * 10);+ parambase = st.p[0];+ reg_matAptr = st.t[0];+ reg_matBptr = st.t[1];+ reg_matCptr = st.t[0];+ reg_ksize = st.t[2];+ reg_astride = st.t[3];+ reg_cstride = st.t[3];+ reg_iterk = st.t[4];+ reg_tmp = st.t[5];+ reg_tmp1 = st.t[6];+ reg_tmp2 = st.t[7];+ reg_nsize = st.t[8];+ reg_itern = st.t[9];+ reg_ret = rax;++ vreg_push(rsp);+ vpbroadcastw(vreg_t(TmpReg + 0), ptr[parambase + OFFSET(one)]);+ load32(reg_ksize, ptr[parambase + OFFSET(k)]);+ load32(reg_nsize, ptr[parambase + OFFSET(n)]);+ xor_(reg_itern, reg_itern);+ L(".nloop");+ init_regs(_mtile);+ mov(reg_matAptr, ptr[parambase + OFFSET(matA)]);+ load32(reg_astride, ptr[parambase + OFFSET(astride)]);+ mov(reg_matBptr, ptr[parambase + OFFSET(matB)]);+ load32(reg_tmp, ptr[parambase + OFFSET(bstride)]);+ imul(reg_tmp, reg_itern);+ lea(reg_matBptr, ptr[reg_matBptr + reg_tmp]);+ xor_(reg_iterk, reg_iterk);+ generate_kloop(_mtile);+ write_back(_mtile);+ add(reg_itern, NTILE);+ cmp(reg_itern, reg_nsize);+ jb(".nloop");+ mov(reg_ret, 0);+ vreg_pop(rsp);++ outLocalLabel(); // end of local label+ }++ void generate_kloop(int _mtile) {+ inLocalLabel();+ mov(reg_tmp, reg_ksize);+ padto_le(reg_tmp, KUNROLL * KTILE);+ cmp(reg_tmp, 0);+ jz(".kloop", T_NEAR);+ L(".unkloop");+ generate_fma(_mtile, KUNROLL);+ add(reg_matAptr, KUNROLL * AKStepSize);+ add(reg_matBptr, KUNROLL * BKStepSize);+ add(reg_iterk, KUNROLL * KTILE);+ cmp(reg_iterk, reg_tmp); // k iteration variable+ jb(".unkloop");+ cmp(reg_tmp, reg_ksize);+ jge(".kend", T_NEAR);+ L(".kloop");+ generate_fma(_mtile, 1);+ add(reg_matAptr, 1 * AKStepSize);+ add(reg_matBptr, 1 * BKStepSize);+ add(reg_iterk, 1 * KTILE);+ cmp(reg_iterk, reg_ksize); // k iteration variable+ jb(".kloop");+ L(".kend");+ outLocalLabel();+ }++ void generate_fma(int _mtile, int _kunroll) {+ for (int kk = 0; kk < _kunroll; kk++) {+ lea(reg_tmp1, ptr[reg_matAptr + kk * AKStepSize]);+ if (BRegCount == NRegs) {+ for (int i = 0; i < NRegs; i++) {+ vmovups(vreg_t(BReg + i), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ for (int mm = 0; mm < _mtile; mm++) {+ vpbroadcastd(vreg_t(AReg), ptr[reg_tmp1]);+ if constexpr (std::is_same_v<AType, int8_t>) {+ vpsignb(vreg_t(TmpReg + 2), vreg_t(AReg), vreg_t(AReg));+ }+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ if constexpr (std::is_same_v<AType, uint8_t>) {+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(TmpReg + 1), vreg_t(AReg), vreg_t(BReg + i),+ vreg_t(TmpReg + 0));+ } else {+ vpsignb(vreg_t(TmpReg + 3), vreg_t(BReg + i), vreg_t(AReg));+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(TmpReg + 1), vreg_t(TmpReg + 2), vreg_t(TmpReg + 3),+ vreg_t(TmpReg + 0));+ }+ }+ }+ } else if (BRegCount == 0) {+ for (int mm = 0; mm < _mtile; mm += ARegCount) {+ int mm_re = utils::remainsize(mm, _mtile, ARegCount);+ for (int imm = 0; imm < mm_re; imm++) {+ vpbroadcastd(vreg_t(AReg + imm), ptr[reg_tmp1]);+ if constexpr (std::is_same_v<AType, int8_t>) {+ vpsignb(vreg_t(TmpReg + 2), vreg_t(AReg + imm), vreg_t(AReg + imm));+ }+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ if constexpr (std::is_same_v<AType, uint8_t>) {+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(TmpReg + 1), vreg_t(AReg + imm),+ ptr[reg_matBptr + kk * BKStepSize + i * VecBytes], vreg_t(TmpReg + 0));+ } else {+ vmovups(vreg_t(TmpReg + 3), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ vpsignb(vreg_t(TmpReg + 3), vreg_t(TmpReg + 3), vreg_t(AReg + imm));+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(TmpReg + 1), vreg_t(TmpReg + 2), vreg_t(TmpReg + 3),+ vreg_t(TmpReg + 0));+ }+ }+ }+ }+ } else {+ assert(0);+ }+ }+ }++ void init_regs(int _mtile) {+ inLocalLabel();+ load32(reg_tmp, ptr[parambase + OFFSET(init)]);+ cmp(reg_tmp, 0);+ je(".read", T_NEAR);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vxor(vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j));+ }+ }+ jmp(".end", T_NEAR);+ L(".read");+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(vreg_t(CReg + i * NRegs + j), ptr[reg_matCptr + j * VecBytes]);+ }+ add(reg_matCptr, reg_cstride);+ }+ L(".end");+ outLocalLabel();+ }++ void write_back(int _mtile) {+ inLocalLabel();+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(ptr[reg_matCptr + j * VecBytes], vreg_t(CReg + i * NRegs + j));+ }+ add(reg_matCptr, reg_cstride);+ }+ outLocalLabel();+ }+};++template <int N, int M>+using Avx2vnniN8P4U8 = Avx2vnniN8P4<uint8_t, N, M>;++template <int N, int M>+using Avx2vnniN8P4S8 = Avx2vnniN8P4<int8_t, N, M>;++template <int _NTILE, int _MTILE = 0>+class Amxbf16N16P2 : protected bestla::xbyak::JitAmxbf16 {+ public:+ static int constexpr RegLen = 16, PackRow = 2;+ static_assert(_NTILE % RegLen == 0);+ static_assert(_MTILE % RegLen == 0);+ static int constexpr NRegs = _NTILE / RegLen;+ static int constexpr MRegs = _MTILE == 0 ? 1 : _MTILE / RegLen;+ static_assert(NRegs * MRegs + 2 <= TileCount);+ static int constexpr NTILE = RegLen * NRegs, MTILE = MRegs * RegLen, KTILE = 32;+ static int constexpr KUNROLL = 2;+ static auto constexpr ISA = BTLA_ISA::AMX_BF16;+ static auto constexpr COMPUTE = CompType::COMP_BF16_FP32;+ typedef utils::bf16 AType;+ typedef utils::bf16 BType;+ typedef float CType;++ struct params {+ AType* matA;+ int astride;+ BType* matB;+ int bstride;+ CType* matC;+ int cstride;+ int k;+ int n;+ int init;+ void* workspace;+ };+ typedef long long (*func_t)(params*);++ int TmpRegCount = RegCount;+ int TmpReg = 0;+ int CTileCount = 0, ATileCount = 0, BTileCount = 0;+ int CTile = 0, ATile = 0, BTile = 0;+ static int constexpr BKStepSize = KTILE * NTILE * sizeof(BType);+ static int constexpr AKStepSize = KTILE * sizeof(AType);++ void generate_code(int _mtile) {+ assign_regs();+ reset();+ generate_mtile(_mtile);+ ready();+ mKernel = getCode<func_t>();+ }+ func_t mKernel = nullptr;++ protected:+ Xbyak::Reg64 parambase;+ Xbyak::Reg64 reg_matAptr;+ Xbyak::Reg64 reg_matBptr;+ Xbyak::Reg64 reg_matCptr;+ Xbyak::Reg64 reg_ksize;+ Xbyak::Reg64 reg_nsize;+ Xbyak::Reg64 reg_cstride;+ Xbyak::Reg64 reg_astride;+ Xbyak::Reg64 reg_iterk;+ Xbyak::Reg64 reg_itern;+ Xbyak::Reg64 reg_tmp;+ Xbyak::Reg64 reg_tmp1;+ Xbyak::Reg64 reg_tmp2;+ Xbyak::Reg64 reg_tmp3;+ Xbyak::Reg64 reg_ret = rax;++ void assign_regs() {+ CTileCount = NRegs * MRegs;+ auto tile_re = TileCount - CTileCount;+ if (tile_re - 1 >= NRegs) {+ BTileCount = NRegs;+ ATileCount = tile_re - BTileCount;+ } else if (tile_re - 1 >= MRegs) {+ ATileCount = MRegs;+ BTileCount = tile_re - ATileCount;+ } else {+ ATileCount = 1;+ BTileCount = tile_re - ATileCount;+ }+ CTile = 0;+ ATile = CTile + CTileCount;+ BTile = ATile + ATileCount;+ }++ void generate_mtile(int _mtile) {+ inLocalLabel(); // use local label for multiple instance+ Xbyak::util::StackFrame st(this, 1, 11, 16 * 10);+ parambase = st.p[0];+ reg_matAptr = st.t[0];+ reg_matBptr = st.t[1];+ reg_matCptr = st.t[0];+ reg_ksize = st.t[2];+ reg_astride = st.t[3];+ reg_cstride = st.t[3];+ reg_iterk = st.t[4];+ reg_tmp = st.t[5];+ reg_tmp1 = st.t[6];+ reg_tmp2 = st.t[7];+ reg_tmp3 = st.t[10];+ reg_nsize = st.t[8];+ reg_itern = st.t[9];+ reg_ret = rax;++ vreg_push(rsp);++ load32(reg_ksize, ptr[parambase + OFFSET(k)]);+ load32(reg_nsize, ptr[parambase + OFFSET(n)]);+ xor_(reg_itern, reg_itern);+ L(".nloop");+ init_regs(_mtile);+ mov(reg_matAptr, ptr[parambase + OFFSET(matA)]);+ load32(reg_astride, ptr[parambase + OFFSET(astride)]);+ mov(reg_matBptr, ptr[parambase + OFFSET(matB)]);+ load32(reg_tmp, ptr[parambase + OFFSET(bstride)]);+ imul(reg_tmp, reg_itern);+ lea(reg_matBptr, ptr[reg_matBptr + reg_tmp]);+ xor_(reg_iterk, reg_iterk);+ generate_kloop(_mtile);+ write_back(_mtile);+ add(reg_itern, NTILE);+ cmp(reg_itern, reg_nsize);+ jb(".nloop");+ mov(reg_ret, 0);+ vreg_pop(rsp);++ outLocalLabel(); // end of local label+ }++ void generate_kloop(int _mtile) {+ inLocalLabel();+ mov(reg_tmp, reg_ksize);+ padto_le(reg_tmp, KUNROLL * KTILE);+ cmp(reg_tmp, 0);+ jz(".kloop", T_NEAR);+ L(".unkloop");+ generate_fma(_mtile, KUNROLL);+ add(reg_matAptr, KUNROLL * AKStepSize);+ add(reg_matBptr, KUNROLL * BKStepSize);+ add(reg_iterk, KUNROLL * KTILE);+ cmp(reg_iterk, reg_tmp); // k iteration variable+ jb(".unkloop");+ cmp(reg_tmp, reg_ksize);+ jge(".kend", T_NEAR);+ L(".kloop");+ generate_fma(_mtile, 1);+ add(reg_matAptr, 1 * AKStepSize);+ add(reg_matBptr, 1 * BKStepSize);+ add(reg_iterk, 1 * KTILE);+ cmp(reg_iterk, reg_ksize); // k iteration variable+ jb(".kloop");+ L(".kend");+ outLocalLabel();+ }++ void generate_fma(int _mtile, int kunrll) {+ auto& reg_Bstride = reg_tmp1;+ mov(reg_Bstride, NTILE * 4);+ int mtiles = _mtile / RegLen;+ for (int kk = 0; kk < kunrll; kk++) {+ auto reg_Atmp = reg_tmp2;+ if (mtiles == 1) {+ reg_Atmp = reg_matAptr;+ } else {+ mov(reg_Atmp, reg_matAptr);+ }+ if (BTileCount == NRegs) {+ for (int i = 0; i < NRegs; i++) {+ tileloaddt1(Xbyak::Tmm(BTile + i), ptr[reg_matBptr + reg_Bstride + kk * BKStepSize + i * 64]);+ }+ for (int mm = 0; mm < mtiles; mm++) {+ tileloadd(Xbyak::Tmm(ATile), ptr[reg_Atmp + reg_astride + kk * AKStepSize]);+ for (int i = 0; i < NRegs; i++) {+ tdpbf16ps(Xbyak::Tmm(CTile + mm * NRegs + i), Xbyak::Tmm(ATile), Xbyak::Tmm(BTile + i));+ }+ if (mm != mtiles - 1) {+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ }+ }+ } else {+ if (ATileCount == mtiles) {+ for (int mm = 0; mm < mtiles; mm++) {+ tileloadd(Xbyak::Tmm(ATile + mm), ptr[reg_Atmp + reg_astride + kk * AKStepSize]);+ if (mm != mtiles - 1) {+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ }+ }+ for (int i = 0; i < NRegs; i++) {+ tileloaddt1(Xbyak::Tmm(BTile), ptr[reg_matBptr + reg_Bstride + kk * BKStepSize + i * 64]);+ for (int mm = 0; mm < mtiles; mm++) {+ tdpbf16ps(Xbyak::Tmm(CTile + mm * NRegs + i), Xbyak::Tmm(ATile + mm), Xbyak::Tmm(BTile));+ }+ }+ } else {+ for (int mm = 0; mm < mtiles; mm++) {+ tileloadd(Xbyak::Tmm(ATile), ptr[reg_Atmp + reg_astride + kk * AKStepSize]);+ for (int i = 0; i < NRegs; i++) {+ tileloaddt1(Xbyak::Tmm(BTile), ptr[reg_matBptr + reg_Bstride + kk * BKStepSize + i * 64]);+ tdpbf16ps(Xbyak::Tmm(CTile + mm * NRegs + i), Xbyak::Tmm(ATile), Xbyak::Tmm(BTile));+ }+ if (mm != mtiles - 1) {+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ }+ }+ }+ }+ }+ }++ void init_regs(int _mtile) {+ inLocalLabel();+ load32(reg_tmp, ptr[parambase + OFFSET(init)]);+ cmp(reg_tmp, 0);+ je(".read", T_NEAR);+ for (int i = 0; i < CTileCount; i++) {+ tilezero(Xbyak::Tmm(CTile + i));+ }+ jmp(".end", T_NEAR);+ L(".read");+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ int mtnum = _mtile / 16;+ for (int mm = 0; mm < mtnum; mm++) {+ for (int i = 0; i < NRegs; i++) {+ tileloaddt1(Xbyak::Tmm(CTile + mm * NRegs + i), ptr[reg_matCptr + reg_cstride + i * 64]);+ }+ if (mm != mtnum - 1) {+ lea(reg_matCptr, ptr[reg_matCptr + 8 * reg_cstride]);+ lea(reg_matCptr, ptr[reg_matCptr + 8 * reg_cstride]);+ }+ }+ L(".end");+ outLocalLabel();+ }++ void write_back(int _mtile) {+ inLocalLabel();+ mov(reg_tmp, dword[parambase + OFFSET(workspace)]);+ mov(reg_tmp1, NTILE * 4);+ for (int mm = 0; mm < MRegs; mm++) {+ for (int i = 0; i < NRegs; i++) {+ tilestored(ptr[reg_tmp + reg_tmp1 + i * 64 + mm * 16 * NTILE * 4], Xbyak::Tmm(CTile + mm * NRegs + i));+ }+ }+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ int zunroll = TmpRegCount / NRegs;+ for (int i = 0; i < _mtile; i += zunroll) {+ int m_re = utils::remainsize(i, _mtile, zunroll);+ for (int im = 0; im < m_re; im++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(vreg_t(TmpReg + im * NRegs + j), ptr[reg_tmp + j * 64 + (i + im) * NTILE * 4]);+ vmovups(ptr[reg_matCptr + j * VecBytes], vreg_t(TmpReg + im * NRegs + j));+ }+ add(reg_matCptr, reg_cstride);+ }+ }+ outLocalLabel();+ }+};++template <int _NTILE, int _MTILE = 0>+class Amxfp16N16P2 : protected bestla::xbyak::JitAmxbf16 {+ public:+ static int constexpr RegLen = 16, PackRow = 2;+ static_assert(_NTILE % RegLen == 0);+ static_assert(_MTILE % RegLen == 0);+ static int constexpr NRegs = _NTILE / RegLen;+ static int constexpr MRegs = _MTILE == 0 ? 1 : _MTILE / RegLen;+ static_assert(NRegs * MRegs + 2 <= TileCount);+ static int constexpr NTILE = RegLen * NRegs, MTILE = MRegs * RegLen, KTILE = 32;+ static int constexpr KUNROLL = 2;+ static auto constexpr ISA = BTLA_ISA::AMX_FP16;+ static auto constexpr COMPUTE = CompType::COMP_FP16_FP32;+ typedef utils::fp16 AType;+ typedef utils::fp16 BType;+ typedef float CType;++ struct params {+ AType* matA;+ int astride;+ BType* matB;+ int bstride;+ CType* matC;+ int cstride;+ int k;+ int n;+ int init;+ void* workspace;+ };+ typedef long long (*func_t)(params*);++ int TmpRegCount = RegCount;+ int TmpReg = 0;+ int CTileCount = 0, ATileCount = 0, BTileCount = 0;+ int CTile = 0, ATile = 0, BTile = 0;+ static int constexpr BKStepSize = KTILE * NTILE * sizeof(BType);+ static int constexpr AKStepSize = KTILE * sizeof(AType);++ void generate_code(int _mtile) {+ assign_regs();+ reset();+ generate_mtile(_mtile);+ ready();+ mKernel = getCode<func_t>();+ }+ func_t mKernel = nullptr;++ protected:+ Xbyak::Reg64 parambase;+ Xbyak::Reg64 reg_matAptr;+ Xbyak::Reg64 reg_matBptr;+ Xbyak::Reg64 reg_matCptr;+ Xbyak::Reg64 reg_ksize;+ Xbyak::Reg64 reg_nsize;+ Xbyak::Reg64 reg_cstride;+ Xbyak::Reg64 reg_astride;+ Xbyak::Reg64 reg_iterk;+ Xbyak::Reg64 reg_itern;+ Xbyak::Reg64 reg_tmp;+ Xbyak::Reg64 reg_tmp1;+ Xbyak::Reg64 reg_tmp2;+ Xbyak::Reg64 reg_tmp3;+ Xbyak::Reg64 reg_ret = rax;++ void assign_regs() {+ CTileCount = NRegs * MRegs;+ auto tile_re = TileCount - CTileCount;+ if (tile_re - 1 >= NRegs) {+ BTileCount = NRegs;+ ATileCount = tile_re - BTileCount;+ } else if (tile_re - 1 >= MRegs) {+ ATileCount = MRegs;+ BTileCount = tile_re - ATileCount;+ } else {+ ATileCount = 1;+ BTileCount = tile_re - ATileCount;+ }+ CTile = 0;+ ATile = CTile + CTileCount;+ BTile = ATile + ATileCount;+ }++ void generate_mtile(int _mtile) {+ inLocalLabel(); // use local label for multiple instance+ Xbyak::util::StackFrame st(this, 1, 11, 16 * 10);+ parambase = st.p[0];+ reg_matAptr = st.t[0];+ reg_matBptr = st.t[1];+ reg_matCptr = st.t[0];+ reg_ksize = st.t[2];+ reg_astride = st.t[3];+ reg_cstride = st.t[3];+ reg_iterk = st.t[4];+ reg_tmp = st.t[5];+ reg_tmp1 = st.t[6];+ reg_tmp2 = st.t[7];+ reg_tmp3 = st.t[10];+ reg_nsize = st.t[8];+ reg_itern = st.t[9];+ reg_ret = rax;++ vreg_push(rsp);++ load32(reg_ksize, ptr[parambase + OFFSET(k)]);+ load32(reg_nsize, ptr[parambase + OFFSET(n)]);+ xor_(reg_itern, reg_itern);+ L(".nloop");+ init_regs(_mtile);+ mov(reg_matAptr, ptr[parambase + OFFSET(matA)]);+ load32(reg_astride, ptr[parambase + OFFSET(astride)]);+ mov(reg_matBptr, ptr[parambase + OFFSET(matB)]);+ load32(reg_tmp, ptr[parambase + OFFSET(bstride)]);+ imul(reg_tmp, reg_itern);+ lea(reg_matBptr, ptr[reg_matBptr + reg_tmp]);+ xor_(reg_iterk, reg_iterk);+ generate_kloop(_mtile);+ write_back(_mtile);+ add(reg_itern, NTILE);+ cmp(reg_itern, reg_nsize);+ jb(".nloop");+ mov(reg_ret, 0);+ vreg_pop(rsp);++ outLocalLabel(); // end of local label+ }++ void generate_kloop(int _mtile) {+ inLocalLabel();+ mov(reg_tmp, reg_ksize);+ padto_le(reg_tmp, KUNROLL * KTILE);+ cmp(reg_tmp, 0);+ jz(".kloop", T_NEAR);+ L(".unkloop");+ generate_fma(_mtile, KUNROLL);+ add(reg_matAptr, KUNROLL * AKStepSize);+ add(reg_matBptr, KUNROLL * BKStepSize);+ add(reg_iterk, KUNROLL * KTILE);+ cmp(reg_iterk, reg_tmp); // k iteration variable+ jb(".unkloop");+ cmp(reg_tmp, reg_ksize);+ jge(".kend", T_NEAR);+ L(".kloop");+ generate_fma(_mtile, 1);+ add(reg_matAptr, 1 * AKStepSize);+ add(reg_matBptr, 1 * BKStepSize);+ add(reg_iterk, 1 * KTILE);+ cmp(reg_iterk, reg_ksize); // k iteration variable+ jb(".kloop");+ L(".kend");+ outLocalLabel();+ }++ void generate_fma(int _mtile, int kunrll) {+ auto& reg_Bstride = reg_tmp1;+ mov(reg_Bstride, NTILE * 4);+ int mtiles = _mtile / RegLen;+ for (int kk = 0; kk < kunrll; kk++) {+ auto reg_Atmp = reg_tmp2;+ if (mtiles == 1) {+ reg_Atmp = reg_matAptr;+ } else {+ mov(reg_Atmp, reg_matAptr);+ }+ if (BTileCount == NRegs) {+ for (int i = 0; i < NRegs; i++) {+ tileloaddt1(Xbyak::Tmm(BTile + i), ptr[reg_matBptr + reg_Bstride + kk * BKStepSize + i * 64]);+ }+ for (int mm = 0; mm < mtiles; mm++) {+ tileloadd(Xbyak::Tmm(ATile), ptr[reg_Atmp + reg_astride + kk * AKStepSize]);+ for (int i = 0; i < NRegs; i++) {+ tdpfp16ps(Xbyak::Tmm(CTile + mm * NRegs + i), Xbyak::Tmm(ATile), Xbyak::Tmm(BTile + i));+ }+ if (mm != mtiles - 1) {+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ }+ }+ } else {+ if (ATileCount == mtiles) {+ for (int mm = 0; mm < mtiles; mm++) {+ tileloadd(Xbyak::Tmm(ATile + mm), ptr[reg_Atmp + reg_astride + kk * AKStepSize]);+ if (mm != mtiles - 1) {+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ }+ }+ for (int i = 0; i < NRegs; i++) {+ tileloaddt1(Xbyak::Tmm(BTile), ptr[reg_matBptr + reg_Bstride + kk * BKStepSize + i * 64]);+ for (int mm = 0; mm < mtiles; mm++) {+ tdpfp16ps(Xbyak::Tmm(CTile + mm * NRegs + i), Xbyak::Tmm(ATile + mm), Xbyak::Tmm(BTile));+ }+ }+ } else {+ for (int mm = 0; mm < mtiles; mm++) {+ tileloadd(Xbyak::Tmm(ATile), ptr[reg_Atmp + reg_astride + kk * AKStepSize]);+ for (int i = 0; i < NRegs; i++) {+ tileloaddt1(Xbyak::Tmm(BTile), ptr[reg_matBptr + reg_Bstride + kk * BKStepSize + i * 64]);+ tdpfp16ps(Xbyak::Tmm(CTile + mm * NRegs + i), Xbyak::Tmm(ATile), Xbyak::Tmm(BTile));+ }+ if (mm != mtiles - 1) {+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ }+ }+ }+ }+ }+ }++ void init_regs(int _mtile) {+ inLocalLabel();+ load32(reg_tmp, ptr[parambase + OFFSET(init)]);+ cmp(reg_tmp, 0);+ je(".read", T_NEAR);+ for (int i = 0; i < CTileCount; i++) {+ tilezero(Xbyak::Tmm(CTile + i));+ }+ jmp(".end", T_NEAR);+ L(".read");+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ int mtnum = _mtile / 16;+ for (int mm = 0; mm < mtnum; mm++) {+ for (int i = 0; i < NRegs; i++) {+ tileloaddt1(Xbyak::Tmm(CTile + mm * NRegs + i), ptr[reg_matCptr + reg_cstride + i * 64]);+ }+ if (mm != mtnum - 1) {+ lea(reg_matCptr, ptr[reg_matCptr + 8 * reg_cstride]);+ lea(reg_matCptr, ptr[reg_matCptr + 8 * reg_cstride]);+ }+ }+ L(".end");+ outLocalLabel();+ }++ void write_back(int _mtile) {+ inLocalLabel();+ mov(reg_tmp, dword[parambase + OFFSET(workspace)]);+ mov(reg_tmp1, NTILE * 4);+ for (int mm = 0; mm < MRegs; mm++) {+ for (int i = 0; i < NRegs; i++) {+ tilestored(ptr[reg_tmp + reg_tmp1 + i * 64 + mm * 16 * NTILE * 4], Xbyak::Tmm(CTile + mm * NRegs + i));+ }+ }+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ int zunroll = TmpRegCount / NRegs;+ for (int i = 0; i < _mtile; i += zunroll) {+ int m_re = utils::remainsize(i, _mtile, zunroll);+ for (int im = 0; im < m_re; im++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(vreg_t(TmpReg + im * NRegs + j), ptr[reg_tmp + j * 64 + (i + im) * NTILE * 4]);+ vmovups(ptr[reg_matCptr + j * VecBytes], vreg_t(TmpReg + im * NRegs + j));+ }+ add(reg_matCptr, reg_cstride);+ }+ }+ outLocalLabel();+ }+};++template <typename AT, typename BT, int _NTILE, int _MTILE = 0>+class Amxint8N16P4 : protected bestla::xbyak::JitAmxint8 {+ public:+ static int constexpr RegLen = 16, PackRow = 4;+ static_assert(_NTILE % RegLen == 0);+ static_assert(_MTILE % RegLen == 0);+ static int constexpr NRegs = _NTILE / RegLen;+ static int constexpr MRegs = _MTILE == 0 ? 1 : _MTILE / RegLen;+ static_assert(NRegs * MRegs + 2 <= TileCount);+ static int constexpr NTILE = RegLen * NRegs, MTILE = MRegs * RegLen, KTILE = 64;+ static int constexpr KUNROLL = 2;+ static auto constexpr ISA = BTLA_ISA::AMX_INT8;+ static auto constexpr COMPUTE =+ (std::is_same_v<AT, int8_t>+ ? std::is_same_v<BT, int8_t> ? CompType::COMP_INT8_SS_INT32 : CompType::COMP_INT8_SU_INT32+ : std::is_same_v<BT, int8_t> ? CompType::COMP_INT8_US_INT32+ : CompType::COMP_INT8_UU_INT32);+ using AType = AT;+ using BType = BT;+ typedef int32_t CType;++ struct params {+ AType* matA;+ int astride;+ BType* matB;+ int bstride;+ CType* matC;+ int cstride;+ int k;+ int n;+ int init;+ void* workspace;+ };+ typedef long long (*func_t)(params*);++ int TmpRegCount = RegCount;+ int TmpReg = 0;+ int CTileCount = 0, ATileCount = 0, BTileCount = 0;+ int CTile = 0, ATile = 0, BTile = 0;+ static int constexpr BKStepSize = KTILE * NTILE * sizeof(BType);+ static int constexpr AKStepSize = KTILE * sizeof(AType);++ void generate_code(int _mtile) {+ assign_regs();+ reset();+ generate_mtile(_mtile);+ ready();+ mKernel = getCode<func_t>();+ }+ func_t mKernel = nullptr;++ protected:+ Xbyak::Reg64 parambase;+ Xbyak::Reg64 reg_matAptr;+ Xbyak::Reg64 reg_matBptr;+ Xbyak::Reg64 reg_matCptr;+ Xbyak::Reg64 reg_ksize;+ Xbyak::Reg64 reg_nsize;+ Xbyak::Reg64 reg_cstride;+ Xbyak::Reg64 reg_astride;+ Xbyak::Reg64 reg_iterk;+ Xbyak::Reg64 reg_itern;+ Xbyak::Reg64 reg_tmp;+ Xbyak::Reg64 reg_tmp1;+ Xbyak::Reg64 reg_tmp2;+ Xbyak::Reg64 reg_tmp3;+ Xbyak::Reg64 reg_ret = rax;++ void assign_regs() {+ CTileCount = NRegs * MRegs;+ auto tile_re = TileCount - CTileCount;+ if (tile_re - 1 >= NRegs) {+ BTileCount = NRegs;+ ATileCount = tile_re - BTileCount;+ } else if (tile_re - 1 >= MRegs) {+ ATileCount = MRegs;+ BTileCount = tile_re - ATileCount;+ } else {+ ATileCount = 1;+ BTileCount = tile_re - ATileCount;+ }+ CTile = 0;+ ATile = CTile + CTileCount;+ BTile = ATile + ATileCount;+ }++ void generate_mtile(int _mtile) {+ inLocalLabel(); // use local label for multiple instance+ Xbyak::util::StackFrame st(this, 1, 11, 16 * 10);+ parambase = st.p[0];+ reg_matAptr = st.t[0];+ reg_matBptr = st.t[1];+ reg_matCptr = st.t[0];+ reg_ksize = st.t[2];+ reg_astride = st.t[3];+ reg_cstride = st.t[3];+ reg_iterk = st.t[4];+ reg_tmp = st.t[5];+ reg_tmp1 = st.t[6];+ reg_tmp2 = st.t[7];+ reg_tmp3 = st.t[10];+ reg_nsize = st.t[8];+ reg_itern = st.t[9];+ reg_ret = rax;++ vreg_push(rsp);++ load32(reg_ksize, ptr[parambase + OFFSET(k)]);+ load32(reg_nsize, ptr[parambase + OFFSET(n)]);+ xor_(reg_itern, reg_itern);+ L(".nloop");+ init_regs(_mtile);+ mov(reg_matAptr, ptr[parambase + OFFSET(matA)]);+ load32(reg_astride, ptr[parambase + OFFSET(astride)]);+ mov(reg_matBptr, ptr[parambase + OFFSET(matB)]);+ load32(reg_tmp, ptr[parambase + OFFSET(bstride)]);+ imul(reg_tmp, reg_itern);+ lea(reg_matBptr, ptr[reg_matBptr + reg_tmp]);+ xor_(reg_iterk, reg_iterk);+ generate_kloop(_mtile);+ write_back(_mtile);+ add(reg_itern, NTILE);+ cmp(reg_itern, reg_nsize);+ jb(".nloop");+ mov(reg_ret, 0);+ vreg_pop(rsp);++ outLocalLabel(); // end of local label+ }++ void generate_kloop(int _mtile) {+ inLocalLabel();+ mov(reg_tmp, reg_ksize);+ padto_le(reg_tmp, KUNROLL * KTILE);+ cmp(reg_tmp, 0);+ jz(".kloop", T_NEAR);+ L(".unkloop");+ generate_fma(_mtile, KUNROLL);+ add(reg_matAptr, KUNROLL * AKStepSize);+ add(reg_matBptr, KUNROLL * BKStepSize);+ add(reg_iterk, KUNROLL * KTILE);+ cmp(reg_iterk, reg_tmp); // k iteration variable+ jb(".unkloop");+ cmp(reg_tmp, reg_ksize);+ jge(".kend", T_NEAR);+ L(".kloop");+ generate_fma(_mtile, 1);+ add(reg_matAptr, 1 * AKStepSize);+ add(reg_matBptr, 1 * BKStepSize);+ add(reg_iterk, 1 * KTILE);+ cmp(reg_iterk, reg_ksize); // k iteration variable+ jb(".kloop");+ L(".kend");+ outLocalLabel();+ }++ void generate_fma(int _mtile, int kunrll) {+ auto& reg_Bstride = reg_tmp1;+ mov(reg_Bstride, NTILE * 4);+ int mtiles = _mtile / RegLen;++ for (int kk = 0; kk < kunrll; kk++) {+ auto reg_Atmp = reg_tmp2;+ if (mtiles == 1) {+ reg_Atmp = reg_matAptr;+ } else {+ mov(reg_Atmp, reg_matAptr);+ }+ if (BTileCount == NRegs) {+ for (int i = 0; i < NRegs; i++) {+ tileloaddt1(Xbyak::Tmm(BTile + i), ptr[reg_matBptr + reg_Bstride + kk * BKStepSize + i * 64]);+ }+ for (int mm = 0; mm < mtiles; mm++) {+ tileloadd(Xbyak::Tmm(ATile), ptr[reg_Atmp + reg_astride + kk * AKStepSize]);+ for (int i = 0; i < NRegs; i++) {+ _tdpb<AT, BT>(Xbyak::Tmm(CTile + mm * NRegs + i), Xbyak::Tmm(ATile), Xbyak::Tmm(BTile + i));+ }+ if (mm != mtiles - 1) {+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ }+ }+ } else {+ if (ATileCount == mtiles) {+ for (int mm = 0; mm < mtiles; mm++) {+ tileloadd(Xbyak::Tmm(ATile + mm), ptr[reg_Atmp + reg_astride + kk * AKStepSize]);+ if (mm != mtiles - 1) {+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ }+ }+ for (int i = 0; i < NRegs; i++) {+ tileloaddt1(Xbyak::Tmm(BTile), ptr[reg_matBptr + reg_Bstride + kk * BKStepSize + i * 64]);+ for (int mm = 0; mm < mtiles; mm++) {+ _tdpb<AT, BT>(Xbyak::Tmm(CTile + mm * NRegs + i), Xbyak::Tmm(ATile + mm), Xbyak::Tmm(BTile));+ }+ }+ } else {+ for (int mm = 0; mm < mtiles; mm++) {+ tileloadd(Xbyak::Tmm(ATile), ptr[reg_Atmp + reg_astride + kk * AKStepSize]);+ for (int i = 0; i < NRegs; i++) {+ tileloaddt1(Xbyak::Tmm(BTile), ptr[reg_matBptr + reg_Bstride + kk * BKStepSize + i * 64]);+ _tdpb<AT, BT>(Xbyak::Tmm(CTile + mm * NRegs + i), Xbyak::Tmm(ATile), Xbyak::Tmm(BTile));+ }+ if (mm != mtiles - 1) {+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ }+ }+ }+ }+ }+ }++ void init_regs(int _mtile) {+ inLocalLabel();+ load32(reg_tmp, ptr[parambase + OFFSET(init)]);+ cmp(reg_tmp, 0);+ je(".read", T_NEAR);+ for (int i = 0; i < CTileCount; i++) {+ tilezero(Xbyak::Tmm(CTile + i));+ }+ jmp(".end", T_NEAR);+ L(".read");+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ int mtnum = _mtile / 16;+ for (int mm = 0; mm < mtnum; mm++) {+ for (int i = 0; i < NRegs; i++) {+ tileloaddt1(Xbyak::Tmm(CTile + mm * NRegs + i), ptr[reg_matCptr + reg_cstride + i * 64]);+ }+ if (mm != mtnum - 1) {+ lea(reg_matCptr, ptr[reg_matCptr + 8 * reg_cstride]);+ lea(reg_matCptr, ptr[reg_matCptr + 8 * reg_cstride]);+ }+ }+ L(".end");+ outLocalLabel();+ }++ void write_back(int _mtile) {+ inLocalLabel();+ mov(reg_tmp, dword[parambase + OFFSET(workspace)]);+ mov(reg_tmp1, NTILE * 4);+ for (int mm = 0; mm < MRegs; mm++) {+ for (int i = 0; i < NRegs; i++) {+ tilestored(ptr[reg_tmp + reg_tmp1 + i * 64 + mm * 16 * NTILE * 4], Xbyak::Tmm(CTile + mm * NRegs + i));+ }+ }+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ int zunroll = TmpRegCount / NRegs;+ for (int i = 0; i < _mtile; i += zunroll) {+ int m_re = utils::remainsize(i, _mtile, zunroll);+ for (int im = 0; im < m_re; im++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(vreg_t(TmpReg + im * NRegs + j), ptr[reg_tmp + j * 64 + (i + im) * NTILE * 4]);+ vmovups(ptr[reg_matCptr + j * VecBytes], vreg_t(TmpReg + im * NRegs + j));+ }+ add(reg_matCptr, reg_cstride);+ }+ }+ outLocalLabel();+ }+};+template <int N, int M>+using Amxint8N16P4US = Amxint8N16P4<uint8_t, int8_t, N, M>;++template <int N, int M>+using Amxint8N16P4SS = Amxint8N16P4<int8_t, int8_t, N, M>;++class AmxConfigure : protected xbyak::JitAmxtile {+ public:+ typedef long long (*func_t)(tileconfig_t*);++ static void configure(int TILE_M, int TILE_N, int TILE_K, int elesize, int ANum, int BNum, int CNum) {+ static AmxConfigure code;+ tileconfig_t cfg;+ std::memset(&cfg, 0, sizeof(cfg));+ configure_tiles(cfg, TILE_M, TILE_N, TILE_K, elesize, ANum, BNum, CNum);+ code.mKernel(&cfg);+ }++ protected:+ AmxConfigure() {+ generate_config(this);+ mKernel = getCode<func_t>();+ }++ func_t mKernel = nullptr;+};+} // namespace code+template <template <int, int> class CodeT, int _NTILE, int _MTILE = 0>+class CoreCodeBase {+ public:+ using Code = CodeT<_NTILE, _MTILE>;+ using AType = typename Code::AType;+ using BType = typename Code::BType;+ using CType = typename Code::CType;+ static auto constexpr NTILE = Code::NTILE;+ static auto constexpr MTILE = Code::MTILE;+ static auto constexpr KTILE = Code::KTILE;+ static auto constexpr PACK_ROW = Code::PackRow;+ static auto constexpr COMP = Code::COMPUTE;+ static int constexpr PREFERRED_N = NTILE * 4;+ static auto constexpr ISA = Code::ISA;+ static auto constexpr ID = CoreAttr::make_core_id(NTILE, PACK_ROW, COMP, ISA);+ static void configure(int _M, int _N, int _K) { (void)(0); }++ static CoreCodeBase<CodeT, _NTILE, _MTILE>* getInstance() {+ static CoreCodeBase<CodeT, _NTILE, _MTILE> instance;+ return &instance;+ }++ std::array<Code, Code::MTILE> mCodes;++ protected:+ CoreCodeBase() {+ for (int i = 0; i < mCodes.size(); i++) {+ mCodes[i].generate_code(i + 1);+ }+ }+};++template <template <int, int> class CodeT, int _NTILE, int _MTILE = 0>+class CoreCodeBaseAMX {+ public:+ using Code = CodeT<_NTILE, _MTILE>;+ using AType = typename Code::AType;+ using BType = typename Code::BType;+ using CType = typename Code::CType;+ static auto constexpr NTILE = Code::NTILE;+ static auto constexpr MTILE = Code::MTILE;+ static auto constexpr KTILE = Code::KTILE;+ static auto constexpr PACK_ROW = Code::PackRow;+ static auto constexpr COMP = Code::COMPUTE;+ static int constexpr PREFERRED_N = NTILE * 4;+ static auto constexpr ISA = Code::ISA;+ static auto constexpr ID = CoreAttr::make_core_id(_NTILE, PACK_ROW, COMP, ISA);+ Xbyak::CodeGenerator cfgcode;++ static CoreCodeBaseAMX<CodeT, _NTILE, _MTILE>* getInstance() {+ static CoreCodeBaseAMX<CodeT, _NTILE, _MTILE> instance;+ return &instance;+ }++ std::array<Code, Code::MRegs> mCodes;++ protected:+ CoreCodeBaseAMX() {+ for (int i = 0; i < mCodes.size(); i++) {+ mCodes[i].generate_code((i + 1) * 16);+ }+ }+};++template <int _NTILE, int _MTILE = 0>+class SCoreRowNAvx2 : public CoreCodeBase<code::Avx2N8P1, _NTILE, _MTILE> {+ public:+ using Base = CoreCodeBase<code::Avx2N8P1, _NTILE, _MTILE>;+ using Code = typename Base::Code;+ static void forward(float* matA, float* matB, float* matC, int _m, int _n, int _k, int _astride, int _bstride,+ int _cstride, int kpos, void* tmpcache, size_t cachesize) {+ auto param = typename Code::params{matA, _astride, matB, _bstride, matC, _cstride, _k, _n, kpos == 0 ? 1 : 0};+ if (_m <= Code::MTILE) {+ Base::getInstance()->mCodes[_m - 1].mKernel(¶m);+ } else {+ assert(0);+ }+ }+};++template <int _NTILE, int _MTILE = 0>+class SCoreRowNAvx512f : public CoreCodeBase<code::Avx512fN16P1, _NTILE, _MTILE> {+ public:+ using Base = CoreCodeBase<code::Avx512fN16P1, _NTILE, _MTILE>;+ using Code = typename Base::Code;+ static void forward(float* matA, float* matB, float* matC, int _m, int _n, int _k, int _astride, int _bstride,+ int _cstride, int kpos, void* tmpcache, size_t cachesize) {+ auto param = typename Code::params{matA, _astride, matB, _bstride, matC, _cstride, _k, _n, kpos == 0 ? 1 : 0};+ if (_m <= Code::MTILE) {+ Base::getInstance()->mCodes[_m - 1].mKernel(¶m);+ } else {+ assert(0);+ }+ }+};++template <int _NTILE, int _MTILE = 0>+class HCoreRowNAvx512fp16 : public CoreCodeBase<code::Avx512fp16N32P1, _NTILE, _MTILE> {+ public:+ using Base = CoreCodeBase<code::Avx512fp16N32P1, _NTILE, _MTILE>;+ using Code = typename Base::Code;+ static void forward(utils::fp16* matA, utils::fp16* matB, utils::fp16* matC, int _m, int _n, int _k, int _astride,+ int _bstride, int _cstride, int kpos, void* tmpcache, size_t cachesize) {+ auto param = typename Code::params{matA, _astride, matB, _bstride, matC, _cstride, _k, _n, kpos == 0 ? 1 : 0};+ if (_m <= Code::MTILE) {+ Base::getInstance()->mCodes[_m - 1].mKernel(¶m);+ } else {+ assert(0);+ }+ }+};++template <int _NTILE, int _MTILE = 0>+class HCoreRowNAvx512bf16 : public CoreCodeBase<code::Avx512bf16N16P2, _NTILE, _MTILE> {+ public:+ using Base = CoreCodeBase<code::Avx512bf16N16P2, _NTILE, _MTILE>;+ using Code = typename Base::Code;+ static void forward(utils::bf16* matA, utils::bf16* matB, float* matC, int _m, int _n, int _k, int _astride,+ int _bstride, int _cstride, int kpos, void* tmpcache, size_t cachesize) {+ auto param = typename Code::params{matA, _astride, matB, _bstride, matC, _cstride, _k, _n, kpos == 0 ? 1 : 0};+ if (_m <= Code::MTILE) {+ Base::getInstance()->mCodes[_m - 1].mKernel(¶m);+ } else {+ assert(0);+ }+ }+};++template <int _NTILE, int _MTILE = 0>+class HCoreRowNAmxbf16 : public CoreCodeBaseAMX<code::Amxbf16N16P2, _NTILE, _MTILE> {+ public:+ using Base = CoreCodeBaseAMX<code::Amxbf16N16P2, _NTILE, _MTILE>;+ using Code = typename Base::Code;+ using AType = typename Code::AType;+ using BType = typename Code::BType;+ using CType = typename Code::CType;++ static void configure(int _M, int _N, int _K) {+ code::AmxConfigure::configure(_M < 16 ? _M : 16, 16, Code::KTILE, sizeof(BType),+ Base::getInstance()->mCodes[0].ATileCount, Base::getInstance()->mCodes[0].BTileCount,+ Base::getInstance()->mCodes[0].CTileCount);+ }++ static void forward(AType* matA, BType* matB, CType* matC, int _m, int _n, int _k, int _astride, int _bstride,+ int _cstride, int kpos, void* tmpcache, size_t cachesize) {+ auto param =+ typename Code::params{matA, _astride, matB, _bstride, matC, _cstride, _k, _n, kpos == 0 ? 1 : 0, tmpcache};+ if (_m <= Code::MTILE) {+ int idx = utils::updiv(_m, 16) - 1;+ Base::getInstance()->mCodes[idx].mKernel(¶m);+ } else {+ assert(0);+ }+ }+};++template <int _NTILE, int _MTILE = 0>+class HCoreRowNAmxfp16 : public CoreCodeBaseAMX<code::Amxfp16N16P2, _NTILE, _MTILE> {+ public:+ using Base = CoreCodeBaseAMX<code::Amxfp16N16P2, _NTILE, _MTILE>;+ using Code = typename Base::Code;+ using AType = typename Code::AType;+ using BType = typename Code::BType;+ using CType = typename Code::CType;++ static void configure(int _M, int _N, int _K) {+ code::AmxConfigure::configure(_M < 16 ? _M : 16, 16, Code::KTILE, sizeof(BType),+ Base::getInstance()->mCodes[0].ATileCount, Base::getInstance()->mCodes[0].BTileCount,+ Base::getInstance()->mCodes[0].CTileCount);+ }++ static void forward(AType* matA, BType* matB, CType* matC, int _m, int _n, int _k, int _astride, int _bstride,+ int _cstride, int kpos, void* tmpcache, size_t cachesize) {+ auto param =+ typename Code::params{matA, _astride, matB, _bstride, matC, _cstride, _k, _n, kpos == 0 ? 1 : 0, tmpcache};+ if (_m <= Code::MTILE) {+ int idx = utils::updiv(_m, 16) - 1;+ Base::getInstance()->mCodes[idx].mKernel(¶m);+ } else {+ assert(0);+ }+ }+};++template <int _NTILE, int _MTILE = 0>+class ICoreRowNAvx512vnni : public CoreCodeBase<code::Avx512vnniN16P4, _NTILE, _MTILE> {+ public:+ using Base = CoreCodeBase<code::Avx512vnniN16P4, _NTILE, _MTILE>;+ using Code = typename Base::Code;+ static void forward(uint8_t* matA, int8_t* matB, int32_t* matC, int _m, int _n, int _k, int _astride, int _bstride,+ int _cstride, int kpos, void* tmpcache, size_t cachesize) {+ auto param = typename Code::params{matA, _astride, matB, _bstride, matC, _cstride, _k, _n, kpos == 0 ? 1 : 0};+ if (_m <= Code::MTILE) {+ Base::getInstance()->mCodes[_m - 1].mKernel(¶m);+ } else {+ assert(0);+ }+ }+};++template <int _NTILE, int _MTILE = 0>+class ICoreRowNAvxvnni : public CoreCodeBase<code::AvxvnniN8P4U8, _NTILE, _MTILE> {+ public:+ using Base = CoreCodeBase<code::AvxvnniN8P4U8, _NTILE, _MTILE>;+ using Code = typename Base::Code;++ static void forward(uint8_t* matA, int8_t* matB, int32_t* matC, int _m, int _n, int _k, int _astride, int _bstride,+ int _cstride, int kpos, void* tmpcache, size_t cachesize) {+ auto param = typename Code::params{matA, _astride, matB, _bstride, matC, _cstride, _k, _n, kpos == 0 ? 1 : 0};+ if (_m <= Code::MTILE) {+ Base::getInstance()->mCodes[_m - 1].mKernel(¶m);+ } else {+ assert(0);+ }+ }+};++template <int _NTILE, int _MTILE = 0>+class ICoreRowNAvxvnniSS : public CoreCodeBase<code::AvxvnniN8P4S8, _NTILE, _MTILE> {+ public:+ using Base = CoreCodeBase<code::AvxvnniN8P4S8, _NTILE, _MTILE>;+ using Code = typename CoreCodeBase<code::AvxvnniN8P4S8, _NTILE, _MTILE>::Code;++ static void forward(int8_t* matA, int8_t* matB, int32_t* matC, int _m, int _n, int _k, int _astride, int _bstride,+ int _cstride, int kpos, void* tmpcache, size_t cachesize) {+ auto param = typename Code::params{matA, _astride, matB, _bstride, matC, _cstride, _k, _n, kpos == 0 ? 1 : 0};+ if (_m <= Code::MTILE) {+ Base::getInstance()->mCodes[_m - 1].mKernel(¶m);+ } else {+ assert(0);+ }+ }+};++template <int _NTILE, int _MTILE = 0>+class ICoreRowNAvx2vnni : public CoreCodeBase<code::Avx2vnniN8P4U8, _NTILE, _MTILE> {+ public:+ using Base = CoreCodeBase<code::Avx2vnniN8P4U8, _NTILE, _MTILE>;+ using Code = typename CoreCodeBase<code::Avx2vnniN8P4U8, _NTILE, _MTILE>::Code;++ static void forward(uint8_t* matA, int8_t* matB, int32_t* matC, int _m, int _n, int _k, int _astride, int _bstride,+ int _cstride, int kpos, void* tmpcache, size_t cachesize) {+ auto param = typename Code::params{matA, _astride, matB, _bstride, matC, _cstride, _k, _n, kpos == 0 ? 1 : 0};+ if (_m <= Code::MTILE) {+ Base::getInstance()->mCodes[_m - 1].mKernel(¶m);+ } else {+ assert(0);+ }+ }+};++template <int _NTILE, int _MTILE = 0>+class ICoreRowNAvx2vnniSS : public CoreCodeBase<code::Avx2vnniN8P4S8, _NTILE, _MTILE> {+ public:+ using Base = CoreCodeBase<code::Avx2vnniN8P4S8, _NTILE, _MTILE>;+ using Code = typename CoreCodeBase<code::Avx2vnniN8P4S8, _NTILE, _MTILE>::Code;++ static void forward(int8_t* matA, int8_t* matB, int32_t* matC, int _m, int _n, int _k, int _astride, int _bstride,+ int _cstride, int kpos, void* tmpcache, size_t cachesize) {+ auto param = typename Code::params{matA, _astride, matB, _bstride, matC, _cstride, _k, _n, kpos == 0 ? 1 : 0};+ if (_m <= Code::MTILE) {+ Base::getInstance()->mCodes[_m - 1].mKernel(¶m);+ } else {+ assert(0);+ }+ }+};++template <int _NTILE, int _MTILE = 0>+class ICoreRowNAvx512bw : public CoreCodeBase<code::Avx512bwN16P4, _NTILE, _MTILE> {+ public:+ using Base = CoreCodeBase<code::Avx512bwN16P4, _NTILE, _MTILE>;+ using Code = typename CoreCodeBase<code::Avx512bwN16P4, _NTILE, _MTILE>::Code;++ static void forward(uint8_t* matA, int8_t* matB, int32_t* matC, int _m, int _n, int _k, int _astride, int _bstride,+ int _cstride, int kpos, void* tmpcache, size_t cachesize) {+ auto param = typename Code::params{matA, _astride, matB, _bstride, matC, _cstride, _k, _n, kpos == 0 ? 1 : 0};+ if (_m <= Code::MTILE) {+ Base::getInstance()->mCodes[_m - 1].mKernel(¶m);+ } else {+ assert(0);+ }+ }+};++template <int _NTILE, int _MTILE = 0>+class ICoreRowNAmxint8 : public CoreCodeBaseAMX<code::Amxint8N16P4US, _NTILE, _MTILE> {+ public:+ using Base = CoreCodeBaseAMX<code::Amxint8N16P4US, _NTILE, _MTILE>;+ using Code = typename CoreCodeBaseAMX<code::Amxint8N16P4US, _NTILE, _MTILE>::Code;+ using AType = typename Code::AType;+ using BType = typename Code::BType;+ using CType = typename Code::CType;+ static void configure(int _M, int _N, int _K) {+ code::AmxConfigure::configure(_M < 16 ? _M : 16, 16, Code::KTILE, sizeof(BType),+ Base::getInstance()->mCodes[0].ATileCount, Base::getInstance()->mCodes[0].BTileCount,+ Base::getInstance()->mCodes[0].CTileCount);+ }++ static void forward(uint8_t* matA, int8_t* matB, int32_t* matC, int _m, int _n, int _k, int _astride, int _bstride,+ int _cstride, int kpos, void* tmpcache, size_t cachesize) {+ auto param =+ typename Code::params{matA, _astride, matB, _bstride, matC, _cstride, _k, _n, kpos == 0 ? 1 : 0, tmpcache};+ if (_m <= Code::MTILE) {+ int idx = utils::updiv(_m, 16) - 1;+ Base::getInstance()->mCodes[idx].mKernel(¶m);+ } else {+ assert(0);+ }+ }+};++template <int _NTILE, int _MTILE = 0>+class ICoreRowNAmxint8SS : public CoreCodeBaseAMX<code::Amxint8N16P4SS, _NTILE, _MTILE> {+ public:+ using Base = CoreCodeBaseAMX<code::Amxint8N16P4SS, _NTILE, _MTILE>;+ using Code = typename CoreCodeBaseAMX<code::Amxint8N16P4SS, _NTILE, _MTILE>::Code;+ using AType = typename Code::AType;+ using BType = typename Code::BType;+ using CType = typename Code::CType;+ static void configure(int _M, int _N, int _K) {+ code::AmxConfigure::configure(_M < 16 ? _M : 16, 16, Code::KTILE, sizeof(BType),+ Base::getInstance()->mCodes[0].ATileCount, Base::getInstance()->mCodes[0].BTileCount,+ Base::getInstance()->mCodes[0].CTileCount);+ }++ static void forward(int8_t* matA, int8_t* matB, int32_t* matC, int _m, int _n, int _k, int _astride, int _bstride,+ int _cstride, int kpos, void* tmpcache, size_t cachesize) {+ auto param =+ typename Code::params{matA, _astride, matB, _bstride, matC, _cstride, _k, _n, kpos == 0 ? 1 : 0, tmpcache};+ if (_m <= Code::MTILE) {+ int idx = utils::updiv(_m, 16) - 1;+ Base::getInstance()->mCodes[idx].mKernel(¶m);+ } else {+ assert(0);+ }+ }+};++} // namespace gemm+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_gemm_kblock.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_gemm_kblock.hnew file mode 100644index 000000000..0d5462fd0--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_gemm_kblock.h@@ -0,0 +1,799 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include <array>++#include "bestla_gemm.h"++namespace bestla {+namespace gemm {+namespace code {+namespace kblock {+struct alignas(64) param_i8_i8_f32 {+ void* matA;+ int astride;+ void* matB;+ int bstride;+ float* matC;+ int cstride;+ void* zpA;+ float* scaleA;+ int ldsa;+ float* scaleB;+ int* reduceB;+ int ldsb;+ int k;+ int n;+ int kblock;+ int init;+ void* workspace;+ const uint16_t one = 1;+};++typedef long long (*func_i8_i8_f32_t)(param_i8_i8_f32*);++template <BTLA_ISA ISA_T, typename AT, int _NTILE, int _MTILE = 0>+class IGemmP4I8S8F32 : public bestla::xbyak::JitCode {+ public:+ using Info = xbyak::JitInfo<ISA_T>;+ using vreg_t = typename xbyak::JitInfo<ISA_T>::vreg_t;+ using AType = AT;+ typedef int8_t BType;+ typedef float CType;+ static auto constexpr ISA = ISA_T;+ static auto constexpr COMPUTE =+ std::is_same_v<AT, uint8_t> ? CompType::COMP_INT8_US_FP32 : CompType::COMP_INT8_SS_FP32;+ static int constexpr KUNROLL = 2;+ static int constexpr RegLen = Info::VecLen32, PACK_ROW = 4;+ static int constexpr VecBytes = Info::VecBytes;+ static_assert(_NTILE % RegLen == 0);+ static_assert(!(Info::HasTile && _MTILE == 0)); // No auto resize for AMX tiles+ static int constexpr NRegs = _NTILE / RegLen;++ static int constexpr TmpRegCount = ISA == BTLA_ISA::AMX_INT8 ? 0+ : ISA == BTLA_ISA::AVX512BW ? 2+ : ISA == BTLA_ISA::AVX512_VNNI ? 1+ : ISA == BTLA_ISA::AVX_VNNI ? std::is_same_v<AT, uint8_t> ? 1 : 2+ : ISA == BTLA_ISA::AVX2 ? std::is_same_v<AT, uint8_t> ? 2 : 4+ : 0;+ static int constexpr ARegCount = 1;+ static int constexpr ABMinReg = ARegCount; // B count can be 0+ static int constexpr ReserveReg = ABMinReg + TmpRegCount;+ static int constexpr MRegs = _MTILE == 0 ? (Info::RegCount - ReserveReg) / (NRegs * 2) : _MTILE;+ static int constexpr CRegCount = NRegs * MRegs;+ static int constexpr BRegCount = Info::RegCount - CRegCount * 2 - ARegCount - TmpRegCount >= NRegs ? NRegs : 0;+ static int constexpr CReg = 0, CF32Reg = CReg + CRegCount, AReg = CF32Reg + CRegCount, BReg = AReg + ARegCount,+ TmpReg = BReg + BRegCount;+ static_assert(!(!Info::HasTile && TmpReg + TmpRegCount > Info::RegCount));+ static int constexpr NTILE = RegLen * NRegs, MTILE = MRegs, KTILE = Info::HasTile ? 64 : 4;+ static int constexpr BKStepSize = KTILE * NTILE * sizeof(BType);+ static int constexpr AKStepSize = KTILE * sizeof(AType);+ static int constexpr NTiles = NRegs, MTiles = MTILE / 16;+ static_assert(!(Info::HasTile && MTILE % 16 != 0));+ static int constexpr CTileCount = NTiles * MTiles;+ static int constexpr BTileCount = Info::TileCount - CTileCount - 1 >= NTiles ? NTiles : 1;+ static int constexpr ATileCount = Info::TileCount - CTileCount - BTileCount >= MTiles ? MTiles : 1;+ static_assert(!(Info::HasTile && ATileCount + BTileCount + CTileCount > Info::TileCount));+ static auto constexpr ID = CoreAttr::make_core_id(NTILE, PACK_ROW, COMPUTE, ISA);+ static int constexpr PREFERRED_N = NTILE * 4;+ static int constexpr CTile = 0, ATile = CTile + CTileCount, BTile = ATile + ATileCount;+ static int constexpr MStepPerKernel = Info::HasTile ? 16 : 1;++ using params = param_i8_i8_f32;+ func_i8_i8_f32_t mKernel = nullptr;++ void generate_code(int _mtile) {+ reset();+ bestla::xbyak::JitCode::ISA = ISA;+ generate_mtile(_mtile);+ ready();+ mKernel = getCode<func_i8_i8_f32_t>();+ }++ protected:+ Xbyak::Reg64 parambase;+ Xbyak::Reg64 reg_matAptr;+ Xbyak::Reg64 reg_matBptr;+ Xbyak::Reg64 reg_matCptr;+ Xbyak::Reg64 reg_ksize;+ Xbyak::Reg64 reg_nsize;+ Xbyak::Reg64 reg_cstride;+ Xbyak::Reg64 reg_astride;+ Xbyak::Reg64 reg_iterk;+ Xbyak::Reg64 reg_iterkb;+ Xbyak::Reg64 reg_itern;+ Xbyak::Reg64 reg_tmp;+ Xbyak::Reg64 reg_tmp1;+ Xbyak::Reg64 reg_tmp2;+ Xbyak::Reg64 reg_tmp3;+ Xbyak::Reg64 reg_tmp4;+ Xbyak::Reg64 reg_ret = rax;++ void generate_mtile(int _mtile) {+ inLocalLabel(); // use local label for multiple instance+ Xbyak::util::StackFrame st(this, 1, 13, 16 * 10);+ parambase = st.p[0];+ reg_matAptr = st.t[0];+ reg_matBptr = st.t[1];+ reg_matCptr = st.t[0];+ reg_ksize = st.t[2];+ reg_astride = st.t[3];+ reg_cstride = st.t[3];+ reg_iterk = st.t[4];+ reg_iterkb = st.t[12];+ reg_tmp = st.t[5];+ reg_tmp1 = st.t[6];+ reg_tmp2 = st.t[7];+ reg_tmp3 = st.t[10];+ reg_tmp4 = st.t[11];+ reg_nsize = st.t[8];+ reg_itern = st.t[9];+ reg_ret = rax;++ vreg_push(rsp);++ load32(reg_ksize, ptr[parambase + OFFSET(k)]);+ load32(reg_nsize, ptr[parambase + OFFSET(n)]);+ xor_(reg_itern, reg_itern);+ load_constant();+ L(".nloop");+ init_accumulator(_mtile);+ mov(reg_matAptr, ptr[parambase + OFFSET(matA)]);+ load32(reg_astride, ptr[parambase + OFFSET(astride)]);+ mov(reg_matBptr, ptr[parambase + OFFSET(matB)]);+ load32(reg_tmp, ptr[parambase + OFFSET(bstride)]);+ imul(reg_tmp, reg_itern);+ lea(reg_matBptr, ptr[reg_matBptr + reg_tmp]);+ xor_(reg_iterk, reg_iterk);+ generate_kloop(_mtile);+ write_back(_mtile);+ add(reg_itern, NTILE);+ cmp(reg_itern, reg_nsize);+ jb(".nloop");+ mov(reg_ret, 0);+ vreg_pop(rsp);++ outLocalLabel(); // end of local label+ }++ void load_constant() {+ if constexpr (ISA == BTLA_ISA::AVX2) {+ if constexpr (TmpRegCount == 4) {+ vpbroadcastw(vreg_t(TmpReg + 3), ptr[parambase + OFFSET(one)]);+ } else {+ vpbroadcastw(vreg_t(TmpReg + 1), ptr[parambase + OFFSET(one)]);+ }+ } else if constexpr (ISA == BTLA_ISA::AVX512BW) {+ vpbroadcastw(vreg_t(TmpReg + 1), ptr[parambase + OFFSET(one)]);+ }+ }++ void zero_int_acc(int _mtile) {+ if constexpr (ISA != BTLA_ISA::AMX_INT8) {+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vxor(vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j));+ }+ }+ } else {+ int mtiles = _mtile / MStepPerKernel;+ static_assert(VecBytes == 64);+ for (int mm = 0; mm < mtiles; mm++) {+ for (int i = 0; i < NRegs; i++) {+ tilezero(Xbyak::Tmm(CTile + mm * NRegs + i));+ }+ }+ }+ }++ void generate_kloop(int _mtile) {+ inLocalLabel();+ xor_(reg_iterkb, reg_iterkb);+ L(".kloop");+ zero_int_acc(_mtile);+ xor_(reg_tmp2, reg_tmp2);+ load32(reg_tmp3, ptr[parambase + OFFSET(kblock)]);+ mov(reg_tmp, reg_tmp3);+ padto_le(reg_tmp, KUNROLL * KTILE);+ cmp(reg_tmp, 0);+ jz(".kbloop", T_NEAR);+ L(".unkbloop");+ generate_mma(_mtile, KUNROLL, reg_tmp1, reg_tmp4);+ add(reg_matAptr, KUNROLL * AKStepSize);+ add(reg_matBptr, KUNROLL * BKStepSize);+ add(reg_tmp2, KUNROLL * KTILE);+ cmp(reg_tmp2, reg_tmp);+ jb(".unkbloop");+ cmp(reg_tmp, reg_tmp3);+ jge(".kend", T_NEAR);+ L(".kbloop");+ generate_mma(_mtile, 1, reg_tmp1, reg_tmp4);+ add(reg_matAptr, 1 * AKStepSize);+ add(reg_matBptr, 1 * BKStepSize);+ add(reg_tmp2, 1 * KTILE);+ cmp(reg_tmp2, reg_tmp3);+ jb(".kbloop");+ L(".kend");+ add(reg_iterk, reg_tmp2);+ generate_zp_correction(_mtile);+ generate_f32_accumulate(_mtile);+ inc(reg_iterkb);+ cmp(reg_iterk, reg_ksize); // k iteration variable+ jb(".kloop");+ outLocalLabel();+ }++ void generate_mma(int _mtile, int _kunroll, Xbyak::Reg64& tmp, Xbyak::Reg64& tmp1) {+ if constexpr (ISA == BTLA_ISA::AVX_VNNI) {+ generate_mma_avxvnni(_mtile, _kunroll, tmp, tmp1);+ } else if constexpr (ISA == BTLA_ISA::AVX2) {+ generate_mma_avx2vnni(_mtile, _kunroll, tmp, tmp1);+ } else if constexpr (ISA == BTLA_ISA::AVX512_VNNI) {+ generate_mma_avx512vnni(_mtile, _kunroll, tmp, tmp1);+ } else if constexpr (ISA == BTLA_ISA::AVX512BW) {+ generate_mma_avx512bw(_mtile, _kunroll, tmp, tmp1);+ } else if constexpr (ISA == BTLA_ISA::AMX_INT8) {+ generate_mma_amxint8(_mtile, _kunroll, tmp, tmp1);+ }+ }++ void init_accumulator(int _mtile) {+ if constexpr (ISA == BTLA_ISA::AMX_INT8) {+ mov(reg_tmp3, ptr[parambase + OFFSET(workspace)]);+ for (int j = 0; j < NRegs; j++) {+ vxor(vreg_t(AccReg + j), vreg_t(AccReg + j), vreg_t(AccReg + j));+ }+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(ptr[reg_tmp3 + j * VecBytes + i * 4 * NTILE + WorkspaceFp32Offset], vreg_t(AccReg + j));+ }+ }+ } else {+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vxor(vreg_t(CF32Reg + i * NRegs + j), vreg_t(CF32Reg + i * NRegs + j), vreg_t(CF32Reg + i * NRegs + j));+ }+ }+ }+ }++ void generate_f32_accumulate(int _mtile) {+ if constexpr (ISA == BTLA_ISA::AMX_INT8) {+ return;+ }+ load32(reg_tmp, ptr[parambase + OFFSET(ldsb)]);+ imul(reg_tmp, reg_iterkb);+ mov(reg_tmp2, ptr[parambase + OFFSET(scaleB)]);+ lea(reg_tmp2, ptr[reg_tmp2 + reg_tmp * sizeof(float)]);+ lea(reg_tmp2, ptr[reg_tmp2 + reg_itern * sizeof(float)]);++ mov(reg_tmp, ptr[parambase + OFFSET(scaleA)]);+ lea(reg_tmp, ptr[reg_tmp + reg_iterkb * sizeof(float)]);+ load32(reg_tmp1, ptr[parambase + OFFSET(ldsa)]);+ if (BRegCount == NRegs) {+ for (int i = 0; i < NRegs; i++) {+ vmovups(vreg_t(BReg + i), ptr[reg_tmp2 + i * VecBytes]);+ }+ }+ for (int mm = 0; mm < _mtile; mm++) {+ vbroadcastss(vreg_t(TmpReg), ptr[reg_tmp]);+ lea(reg_tmp, ptr[reg_tmp + reg_tmp1 * sizeof(float)]);+ for (int i = 0; i < NRegs; i++) {+ vcvtdq2ps(vreg_t(CReg + mm * NRegs + i), vreg_t(CReg + mm * NRegs + i));+ if (BRegCount == NRegs) {+ vmulps(vreg_t(AReg), vreg_t(TmpReg), vreg_t(BReg + i));+ } else {+ vmulps(vreg_t(AReg), vreg_t(TmpReg), ptr[reg_tmp2 + i * VecBytes]);+ }+ vmulps(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg));+ vaddps(vreg_t(CF32Reg + mm * NRegs + i), vreg_t(CReg + mm * NRegs + i));+ }+ }+ }++ void generate_zp_correction(int _mtile) {+ if constexpr (ISA == BTLA_ISA::AMX_INT8) {+ return generate_zp_correction_amx(_mtile);+ }+ inLocalLabel();+ mov(reg_tmp, ptr[parambase + OFFSET(zpA)]);+ cmp(reg_tmp, 0);+ je(".NOZP", T_NEAR);+ lea(reg_tmp, ptr[reg_tmp + reg_iterkb * sizeof(AType)]);+ auto& reg_zpA = reg_tmp;++ load32(reg_tmp1, ptr[parambase + OFFSET(ldsb)]);+ imul(reg_tmp1, reg_iterkb);+ mov(reg_tmp2, ptr[parambase + OFFSET(reduceB)]);+ lea(reg_tmp2, ptr[reg_tmp2 + reg_tmp1 * sizeof(int32_t)]);+ lea(reg_tmp2, ptr[reg_tmp2 + reg_itern * sizeof(int32_t)]);+ auto& reg_redB = reg_tmp2;++ load32(reg_tmp3, ptr[parambase + OFFSET(ldsa)]);+ auto& reg_ldsa = reg_tmp3;+ if (BRegCount == NRegs) {+ for (int i = 0; i < NRegs; i++) {+ vmovups(vreg_t(BReg + i), ptr[reg_redB + i * VecBytes]);+ }+ }+ for (int i = 0; i < _mtile; i++) {+ vpbroadcastb(Xbyak::Xmm(AReg), ptr[reg_zpA]);+ vpmovzxbd(vreg_t(AReg), Xbyak::Xmm(AReg));+ for (int j = 0; j < NRegs; j++) {+ if (BRegCount == NRegs) {+ vpmulld(vreg_t(TmpReg), vreg_t(AReg), vreg_t(BReg + j));+ } else {+ vpmulld(vreg_t(TmpReg), vreg_t(AReg), ptr[reg_redB + j * VecBytes]);+ }+ vpsubd(vreg_t(CReg + i * NRegs + j), vreg_t(CReg + i * NRegs + j), vreg_t(TmpReg));+ }+ lea(reg_zpA, ptr[reg_zpA + reg_ldsa * sizeof(AType)]);+ }+ L(".NOZP");+ outLocalLabel();+ }++ void write_back(int _mtile) {+ if (ISA == BTLA_ISA::AMX_INT8) {+ return write_back_amx(_mtile);+ }+ inLocalLabel();+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);++ load32(reg_tmp, ptr[parambase + OFFSET(init)]);+ cmp(reg_tmp, 0);+ je(".add_back", T_NEAR);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(ptr[reg_matCptr + j * VecBytes], vreg_t(CF32Reg + i * NRegs + j));+ }+ add(reg_matCptr, reg_cstride);+ }+ jmp(".end", T_NEAR);+ L(".add_back");+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vaddps(vreg_t(CF32Reg + i * NRegs + j), vreg_t(CF32Reg + i * NRegs + j), ptr[reg_matCptr + j * VecBytes]);+ vmovups(ptr[reg_matCptr + j * VecBytes], vreg_t(CF32Reg + i * NRegs + j));+ }+ add(reg_matCptr, reg_cstride);+ }+ L(".end");+ outLocalLabel();+ }++ private:+ void generate_mma_avxvnni(int _mtile, int kunrll, Xbyak::Reg64& tmp, Xbyak::Reg64& tmp1) {+ for (int kk = 0; kk < kunrll; kk++) {+ lea(tmp, ptr[reg_matAptr + kk * AKStepSize]);+ if (BRegCount == 0) {+ for (int mm = 0; mm < _mtile; mm++) {+ vpbroadcastd(vreg_t(AReg), ptr[reg_tmp1]);+ if constexpr (std::is_same_v<AType, int8_t>) {+ vpsignb(vreg_t(TmpReg + 1), vreg_t(AReg), vreg_t(AReg));+ }+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ if constexpr (std::is_same_v<AType, uint8_t>) {+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg),+ ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ } else {+ vmovups(vreg_t(TmpReg), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ vpsignb(vreg_t(TmpReg), vreg_t(TmpReg), vreg_t(AReg));+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(TmpReg + 1), vreg_t(TmpReg));+ }+ }+ }+ } else {+ for (int i = 0; i < NRegs; i++) {+ vmovups(vreg_t(BReg + i), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ for (int mm = 0; mm < _mtile; mm++) {+ vpbroadcastd(vreg_t(AReg), ptr[reg_tmp1]);+ if constexpr (std::is_same_v<AType, int8_t>) {+ vpsignb(vreg_t(TmpReg + 1), vreg_t(AReg), vreg_t(AReg));+ }+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ if constexpr (std::is_same_v<AType, uint8_t>) {+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg), vreg_t(BReg + i));+ } else {+ vpsignb(vreg_t(TmpReg), vreg_t(BReg + i), vreg_t(AReg));+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(TmpReg + 1), vreg_t(TmpReg));+ }+ }+ }+ }+ }+ }++ void generate_mma_avx2vnni(int _mtile, int kunrll, Xbyak::Reg64& tmp, Xbyak::Reg64& tmp1) {+ for (int kk = 0; kk < kunrll; kk++) {+ lea(tmp, ptr[reg_matAptr + kk * AKStepSize]);+ if (BRegCount == 0) {+ for (int mm = 0; mm < _mtile; mm++) {+ vpbroadcastd(vreg_t(AReg), ptr[reg_tmp1]);+ if constexpr (std::is_same_v<AType, int8_t>) {+ vpsignb(vreg_t(TmpReg + 2), vreg_t(AReg), vreg_t(AReg));+ }+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ if constexpr (std::is_same_v<AType, uint8_t>) {+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes],+ vreg_t(TmpReg), vreg_t(TmpReg + 1));+ } else {+ vmovups(vreg_t(TmpReg), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ vpsignb(vreg_t(TmpReg), vreg_t(TmpReg), vreg_t(AReg));+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(TmpReg + 2), vreg_t(TmpReg), vreg_t(TmpReg + 1),+ vreg_t(TmpReg + 3));+ }+ }+ }+ } else {+ for (int i = 0; i < NRegs; i++) {+ vmovups(vreg_t(BReg + i), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ for (int mm = 0; mm < _mtile; mm++) {+ vpbroadcastd(vreg_t(AReg), ptr[reg_tmp1]);+ if constexpr (std::is_same_v<AType, int8_t>) {+ vpsignb(vreg_t(TmpReg + 2), vreg_t(AReg), vreg_t(AReg));+ }+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ if constexpr (std::is_same_v<AType, uint8_t>) {+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg), vreg_t(BReg + i), vreg_t(TmpReg + 0),+ vreg_t(TmpReg + 1));+ } else {+ vpsignb(vreg_t(TmpReg + 0), vreg_t(BReg + i), vreg_t(AReg));+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(TmpReg + 2), vreg_t(TmpReg + 0), vreg_t(TmpReg + 1),+ vreg_t(TmpReg + 3));+ }+ }+ }+ }+ }+ }++ void generate_mma_avx512vnni(int _mtile, int kunrll, Xbyak::Reg64& tmp, Xbyak::Reg64& tmp1) {+ for (int kk = 0; kk < kunrll; kk++) {+ lea(tmp, ptr[reg_matAptr + kk * AKStepSize]);+ if (BRegCount == NRegs) {+ for (int i = 0; i < NRegs; i++) {+ vmovups(vreg_t(BReg + i), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ }+ for (int mm = 0; mm < _mtile; mm++) {+ vpbroadcastd(vreg_t(AReg), ptr[reg_tmp1]);+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ if (BRegCount == NRegs) {+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg), vreg_t(BReg + i));+ } else {+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ }+ }+ }+ }++ void generate_mma_avx512bw(int _mtile, int kunrll, Xbyak::Reg64& tmp, Xbyak::Reg64& tmp1) {+ static_assert(std::is_same_v<AType, uint8_t>);+ for (int kk = 0; kk < kunrll; kk++) {+ lea(tmp, ptr[reg_matAptr + kk * AKStepSize]);+ if (BRegCount == NRegs) {+ for (int i = 0; i < NRegs; i++) {+ vmovups(vreg_t(BReg + i), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes]);+ }+ }+ for (int mm = 0; mm < _mtile; mm++) {+ vpbroadcastd(vreg_t(AReg), ptr[reg_tmp1]);+ add(reg_tmp1, reg_astride);+ for (int i = 0; i < NRegs; i++) {+ if (BRegCount == NRegs) {+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg), vreg_t(BReg + i), vreg_t(TmpReg + 0),+ vreg_t(TmpReg + 1));+ } else {+ vpdpbusds_(vreg_t(CReg + mm * NRegs + i), vreg_t(AReg), ptr[reg_matBptr + kk * BKStepSize + i * VecBytes],+ vreg_t(TmpReg + 0), vreg_t(TmpReg + 1));+ }+ }+ }+ }+ }++ static int constexpr WorkspaceFp32Offset = MTILE * NTILE * 4;+ static int constexpr AccReg = 0;+ static int constexpr RedBReg = AccReg + NRegs;+ static int constexpr ScaBReg = RedBReg + NRegs;+ static int constexpr ZpAReg = ScaBReg + NRegs;+ static int constexpr ScaAReg = ZpAReg + 1;+ static int constexpr TmpCReg = ScaAReg + 1;+ void generate_mma_amxint8(int _mtile, int kunrll, Xbyak::Reg64& tmpreg, Xbyak::Reg64& tmpreg2) {+ auto& reg_Bstride = tmpreg2;+ mov(reg_Bstride, NTILE * 4);+ int mtiles = _mtile / MStepPerKernel;++ for (int kk = 0; kk < kunrll; kk++) {+ auto reg_Atmp = tmpreg;+ if (mtiles == 1) {+ reg_Atmp = reg_matAptr;+ } else {+ mov(reg_Atmp, reg_matAptr);+ }+ if (BTileCount == NTiles) {+ for (int i = 0; i < NTiles; i++) {+ tileloaddt1(Xbyak::Tmm(BTile + i), ptr[reg_matBptr + reg_Bstride + kk * BKStepSize + i * 64]);+ }+ for (int mm = 0; mm < mtiles; mm++) {+ tileloadd(Xbyak::Tmm(ATile), ptr[reg_Atmp + reg_astride + kk * AKStepSize]);+ for (int i = 0; i < NTiles; i++) {+ _tdpb<AType, BType>(Xbyak::Tmm(CTile + mm * NTiles + i), Xbyak::Tmm(ATile), Xbyak::Tmm(BTile + i));+ }+ if (mm != mtiles - 1) {+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ }+ }+ } else {+ if (ATileCount == mtiles) {+ for (int mm = 0; mm < mtiles; mm++) {+ tileloadd(Xbyak::Tmm(ATile + mm), ptr[reg_Atmp + reg_astride + kk * AKStepSize]);+ if (mm != mtiles - 1) {+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ }+ }+ for (int i = 0; i < NTiles; i++) {+ tileloaddt1(Xbyak::Tmm(BTile), ptr[reg_matBptr + reg_Bstride + kk * BKStepSize + i * 64]);+ for (int mm = 0; mm < mtiles; mm++) {+ _tdpb<AType, BType>(Xbyak::Tmm(CTile + mm * NTiles + i), Xbyak::Tmm(ATile + mm), Xbyak::Tmm(BTile));+ }+ }+ } else {+ for (int mm = 0; mm < mtiles; mm++) {+ tileloadd(Xbyak::Tmm(ATile), ptr[reg_Atmp + reg_astride + kk * AKStepSize]);+ for (int i = 0; i < NTiles; i++) {+ tileloaddt1(Xbyak::Tmm(BTile), ptr[reg_matBptr + reg_Bstride + kk * BKStepSize + i * 64]);+ _tdpb<AType, BType>(Xbyak::Tmm(CTile + mm * NTiles + i), Xbyak::Tmm(ATile), Xbyak::Tmm(BTile));+ }+ if (mm != mtiles - 1) {+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ lea(reg_Atmp, ptr[reg_Atmp + 8 * reg_astride]);+ }+ }+ }+ }+ }+ }++ void generate_zp_correction_amx(int _mtile) {+ int mtiles = _mtile / MStepPerKernel;+ static_assert(VecBytes == 64);+ inLocalLabel();+ mov(reg_tmp3, ptr[parambase + OFFSET(workspace)]);+ mov(reg_tmp1, NTILE * 4);+ for (int mm = 0; mm < mtiles; mm++) {+ for (int i = 0; i < NRegs; i++) {+ tilestored(ptr[reg_tmp3 + reg_tmp1 + i * VecBytes + mm * 16 * NTILE * 4], Xbyak::Tmm(CTile + mm * NRegs + i));+ }+ }++ load32(reg_tmp1, ptr[parambase + OFFSET(ldsb)]);+ imul(reg_tmp1, reg_iterkb);+ mov(reg_tmp2, ptr[parambase + OFFSET(scaleB)]);+ lea(reg_tmp2, ptr[reg_tmp2 + reg_tmp1 * sizeof(float)]);+ lea(reg_tmp2, ptr[reg_tmp2 + reg_itern * sizeof(float)]);+ for (int i = 0; i < NRegs; i++) {+ vmovups(vreg_t(ScaBReg + i), ptr[reg_tmp2 + i * VecBytes]);+ }++ mov(reg_tmp, ptr[parambase + OFFSET(zpA)]);+ cmp(reg_tmp, 0);+ je(".NOZP", T_NEAR);++ lea(reg_tmp, ptr[reg_tmp + reg_iterkb * sizeof(AType)]);+ auto& reg_zpA = reg_tmp;++ mov(reg_tmp2, ptr[parambase + OFFSET(reduceB)]);+ lea(reg_tmp2, ptr[reg_tmp2 + reg_tmp1 * sizeof(int32_t)]);+ lea(reg_tmp2, ptr[reg_tmp2 + reg_itern * sizeof(int32_t)]);+ for (int i = 0; i < NRegs; i++) {+ vmovups(vreg_t(RedBReg + i), ptr[reg_tmp2 + i * VecBytes]);+ }++ load32(reg_tmp1, ptr[parambase + OFFSET(ldsa)]);+ auto& reg_ldsa = reg_tmp1;+ auto& reg_sca = reg_tmp2;+ mov(reg_sca, ptr[parambase + OFFSET(scaleA)]);+ lea(reg_sca, ptr[reg_sca + reg_iterkb * sizeof(float)]);++ for (int i = 0; i < _mtile; i++) {+ vpbroadcastb(Xbyak::Xmm(ZpAReg), ptr[reg_zpA]);+ vpmovzxbd(vreg_t(ZpAReg), Xbyak::Xmm(ZpAReg));+ vbroadcastss(vreg_t(ScaAReg), ptr[reg_sca]);+ for (int j = 0; j < NRegs; j++) {+ vmovups(vreg_t(AccReg + j), ptr[reg_tmp3 + j * VecBytes + i * 4 * NTILE]);+ vpmulld(vreg_t(TmpCReg), vreg_t(ZpAReg), vreg_t(RedBReg + j));+ vpsubd(vreg_t(AccReg + j), vreg_t(AccReg + j), vreg_t(TmpCReg));+ vmulps(vreg_t(TmpCReg), vreg_t(ScaAReg), vreg_t(ScaBReg + j));+ vcvtdq2ps(vreg_t(AccReg + j), vreg_t(AccReg + j));+ vmulps(vreg_t(AccReg + j), vreg_t(TmpCReg));+ vaddps(vreg_t(AccReg + j), vreg_t(AccReg + j),+ ptr[reg_tmp3 + j * VecBytes + i * 4 * NTILE + WorkspaceFp32Offset]);+ vmovups(ptr[reg_tmp3 + j * VecBytes + i * 4 * NTILE + WorkspaceFp32Offset], vreg_t(AccReg + j));+ }+ lea(reg_zpA, ptr[reg_zpA + reg_ldsa * sizeof(AType)]);+ lea(reg_sca, ptr[reg_sca + reg_ldsa * sizeof(float)]);+ }+ jmp(".END", T_NEAR);+ L(".NOZP");+ load32(reg_ldsa, ptr[parambase + OFFSET(ldsa)]);+ mov(reg_sca, ptr[parambase + OFFSET(scaleA)]);+ lea(reg_sca, ptr[reg_sca + reg_iterkb * sizeof(float)]);++ for (int i = 0; i < _mtile; i++) {+ vbroadcastss(vreg_t(ScaAReg), ptr[reg_sca]);+ for (int j = 0; j < NRegs; j++) {+ vmovups(vreg_t(AccReg + j), ptr[reg_tmp3 + j * VecBytes + i * 4 * NTILE]);+ vmulps(vreg_t(TmpCReg), vreg_t(ScaAReg), vreg_t(ScaBReg + j));+ vcvtdq2ps(vreg_t(AccReg + j), vreg_t(AccReg + j));+ vmulps(vreg_t(AccReg + j), vreg_t(TmpCReg));+ vaddps(vreg_t(AccReg + j), vreg_t(AccReg + j),+ ptr[reg_tmp3 + j * VecBytes + i * 4 * NTILE + WorkspaceFp32Offset]);+ vmovups(ptr[reg_tmp3 + j * VecBytes + i * 4 * NTILE + WorkspaceFp32Offset], vreg_t(AccReg + j));+ }+ lea(reg_sca, ptr[reg_sca + reg_ldsa * sizeof(float)]);+ }+ L(".END");+ outLocalLabel();+ }++ void write_back_amx(int _mtile) {+ inLocalLabel();+ mov(reg_tmp3, ptr[parambase + OFFSET(workspace)]);+ mov(reg_matCptr, ptr[parambase + OFFSET(matC)]);+ lea(reg_matCptr, ptr[reg_matCptr + reg_itern * sizeof(CType)]);+ load32(reg_cstride, ptr[parambase + OFFSET(cstride)]);++ load32(reg_tmp, ptr[parambase + OFFSET(init)]);+ cmp(reg_tmp, 0);+ je(".add_back", T_NEAR);+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(vreg_t(AccReg + j), ptr[reg_tmp3 + j * VecBytes + i * 4 * NTILE + WorkspaceFp32Offset]);+ vmovups(ptr[reg_matCptr + j * VecBytes], vreg_t(AccReg + j));+ }+ add(reg_matCptr, reg_cstride);+ }+ jmp(".end", T_NEAR);+ L(".add_back");+ for (int i = 0; i < _mtile; i++) {+ for (int j = 0; j < NRegs; j++) {+ vmovups(vreg_t(AccReg + j), ptr[reg_tmp3 + j * VecBytes + i * 4 * NTILE + WorkspaceFp32Offset]);+ vaddps(vreg_t(AccReg + j), vreg_t(AccReg + j), ptr[reg_matCptr + j * VecBytes]);+ vmovups(ptr[reg_matCptr + j * VecBytes], vreg_t(AccReg + j));+ }+ add(reg_matCptr, reg_cstride);+ }+ L(".end");+ outLocalLabel();+ }+};++template <int N, int M>+using AvxvnniUS = IGemmP4I8S8F32<BTLA_ISA::AVX_VNNI, uint8_t, N, M>;+template <int N, int M>+using AvxvnniSS = IGemmP4I8S8F32<BTLA_ISA::AVX_VNNI, int8_t, N, M>;+template <int N, int M>+using Avx2vnniUS = IGemmP4I8S8F32<BTLA_ISA::AVX2, uint8_t, N, M>;+template <int N, int M>+using Avx2vnniSS = IGemmP4I8S8F32<BTLA_ISA::AVX2, int8_t, N, M>;++template <int N, int M>+using Avx512vnniUS = IGemmP4I8S8F32<BTLA_ISA::AVX512_VNNI, uint8_t, N, M>;+template <int N, int M>+using Avx512bwUS = IGemmP4I8S8F32<BTLA_ISA::AVX512BW, uint8_t, N, M>;++template <int N, int M>+using Amxint8US = IGemmP4I8S8F32<BTLA_ISA::AMX_INT8, uint8_t, N, M>;+template <int N, int M>+using Amxint8SS = IGemmP4I8S8F32<BTLA_ISA::AMX_INT8, int8_t, N, M>;+} // namespace kblock+} // namespace code++template <template <int, int> class CodeT, int _NTILE, int _MTILE = 0>+class KblockCoreInterface {+ public:+ using Code = CodeT<_NTILE, _MTILE>;+ using AType = typename Code::AType;+ using BType = typename Code::BType;+ using CType = typename Code::CType;+ static auto constexpr NTILE = Code::NTILE;+ static auto constexpr MTILE = Code::MTILE;+ static auto constexpr KTILE = Code::KTILE;+ static auto constexpr PACK_ROW = Code::PACK_ROW;+ static auto constexpr COMP = Code::COMPUTE;+ static int constexpr PREFERRED_N = Code::PREFERRED_N;+ static auto constexpr ISA = Code::ISA;+ static auto constexpr ID = Code::ID;+ static auto constexpr MStep = Code::MStepPerKernel;+ static void configure(int _M, int _N, int _K) {+ if (ISA >= BTLA_ISA::AMX_BF16) {+ gemm::code::AmxConfigure::configure(_M < 16 ? _M : 16, 16, Code::KTILE, sizeof(BType), Code::ATileCount,+ Code::BTileCount, Code::CTileCount);++ } else {+ (void)(_M);+ (void)(_N);+ (void)(_K);+ }+ }++ static void forward(AType* matA, BType* matB, CType* matC, AType* zpA, float* scaleA, int _ldsa, float* scaleB,+ int32_t* reduceB, int _ldsb, int _m, int _n, int _k, int _kblock, int _astride, int _bstride,+ int _cstride, int kpos, void* tmpcache, size_t cachesize) {+ auto param = typename Code::params{matA, _astride, matB, _bstride, matC, _cstride, zpA, scaleA,+ _ldsa, scaleB, reduceB, _ldsb, _k, _n, _kblock, kpos == 0 ? 1 : 0,+ tmpcache};+ if (_m <= Code::MTILE) {+ int code_idx = utils::updiv(_m, MStep);+ getInstance()->mCodes[code_idx - 1].mKernel(¶m);+ } else {+ assert(0);+ }+ }++ static KblockCoreInterface<CodeT, _NTILE, _MTILE>* getInstance() {+ static KblockCoreInterface<CodeT, _NTILE, _MTILE> instance;+ return &instance;+ }++ std::array<Code, Code::MTILE / MStep> mCodes;++ protected:+ KblockCoreInterface() {+ for (int i = 0; i < mCodes.size(); i++) {+ mCodes[i].generate_code((i + 1) * MStep);+ }+ }+};++template <int _NTILE, int _MTILE = 0>+using KblockGemmAvxvnni = KblockCoreInterface<code::kblock::AvxvnniUS, _NTILE, _MTILE>;++template <int _NTILE, int _MTILE = 0>+using KblockGemmAvxvnniSS = KblockCoreInterface<code::kblock::AvxvnniSS, _NTILE, _MTILE>;++template <int _NTILE, int _MTILE = 0>+using KblockGemmAvx2vnni = KblockCoreInterface<code::kblock::Avx2vnniUS, _NTILE, _MTILE>;++template <int _NTILE, int _MTILE = 0>+using KblockGemmAvx2vnniSS = KblockCoreInterface<code::kblock::Avx2vnniSS, _NTILE, _MTILE>;++template <int _NTILE, int _MTILE = 0>+using KblockGemmAvx512vnni = KblockCoreInterface<code::kblock::Avx512vnniUS, _NTILE, _MTILE>;++template <int _NTILE, int _MTILE = 0>+using KblockGemmAvx512bw = KblockCoreInterface<code::kblock::Avx512bwUS, _NTILE, _MTILE>;++template <int _NTILE, int _MTILE = 0>+using KblockGemmAmxint8 = KblockCoreInterface<code::kblock::Amxint8US, _NTILE, _MTILE>;++template <int _NTILE, int _MTILE = 0>+using KblockGemmAmxint8SS = KblockCoreInterface<code::kblock::Amxint8SS, _NTILE, _MTILE>;++} // namespace gemm+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_jit.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_jit.hnew file mode 100644index 000000000..75144efb6--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_jit.h@@ -0,0 +1,480 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include <stdint.h>++#include <cstddef>+#include <type_traits>+#include "xbyak/xbyak.h"+#include "xbyak/xbyak_util.h"++#define OFFSET(field) offsetof(params, field)++namespace bestla {++namespace xbyak {+class JitBase : protected Xbyak::CodeGenerator {+ protected:+ JitBase(size_t size = 16 * 1024) : CodeGenerator(size) {}++ void load32(const Xbyak::Reg64& reg, const Xbyak::Address& addr) {+ xor_(reg, reg);+ mov(reg.cvt32(), addr);+ }++ void vreg_push(const Xbyak::Reg64& baseaddr) {+#ifdef _WIN32+ for (int i = 0; i < 10; i++) {+ movaps(xword[baseaddr + i * 16], Xbyak::Xmm(6 + i));+ }+#endif+ }++ void vreg_pop(const Xbyak::Reg64& baseaddr) {+#ifdef _WIN32+ for (int i = 0; i < 10; i++) {+ movaps(Xbyak::Xmm(6 + i), xword[baseaddr + i * 16]);+ }+#endif+ }++ void padto_le(const Xbyak::Reg64& _src, int padding) {+ // _src=_src/padding*padding+ if (padding == 1) {+ return;+ }+ for (int i = 1; i < 16; i++) {+ if ((1 << i) == padding) {+ shr(_src, i);+ shl(_src, i);+ return;+ }+ }+ assert(0);+ }++ void generate_Nbitsmask(const Xbyak::Opmask& _msk, const Xbyak::Reg64& _pos, const Xbyak::Address& _total,+ const Xbyak::Reg64& _tmp, const Xbyak::Reg64& _tmp1, int N) {+ inLocalLabel();+ lea(_tmp, _total);+ sub(_tmp, _pos);+ cmp(_tmp, N);+ jb(".maskflag");+ cmp(_tmp, 0);+ jl(".zeroflag");+ uint64_t allmask = (static_cast<uint64_t>(1) << N) - 1;+ if (N == 64) {+ allmask = static_cast<uint64_t>(-1);+ }+ mov(_tmp, allmask);+ kmovq(_msk, _tmp);+ jmp(".maskend");+ L(".maskflag");+ mov(_tmp1, 1);+ shlx(_tmp1, _tmp1, _tmp);+ sub(_tmp1, 1);+ kmovq(_msk, _tmp1);+ jmp(".maskend");+ L(".zeroflag");+ mov(_tmp1, 0);+ kmovq(_msk, _tmp1);+ L(".maskend");+ outLocalLabel();+ }+ void generate_Nbitsmask(const Xbyak::Opmask& _msk, const Xbyak::Reg64& _pos, const Xbyak::Reg64& _total,+ const Xbyak::Reg64& _tmp, const Xbyak::Reg64& _tmp1, int N) {+ generate_Nbitsmask(_msk, _pos, ptr[_total], _tmp, _tmp1, N);+ }+};++class JitAvx : protected JitBase {+ protected:+ static auto constexpr ISA = BTLA_ISA::AVX;+ static int constexpr VBits = 256;+ static int constexpr VecBytes = VBits / 8;+ static int constexpr RegCount = 16;+ typedef Xbyak::Ymm vreg_t;+};++class JitAvx2 : protected JitAvx {+ protected:+ static auto constexpr ISA = BTLA_ISA::AVX2;+ static int constexpr VBits = 256;+ typedef Xbyak::Ymm vreg_t;+ void vxor(const vreg_t& x1, const vreg_t& x2, const Xbyak::Operand& op) { vpxor(x1, x2, op); }++ void loadbf16_f32(const Xbyak::Ymm& dst, const Xbyak::Address& addr) {+ vpmovzxwd(dst, addr);+ vpslld(dst, dst, 16);+ }++ void vpdpbusds_(const Xbyak::Xmm& sum4, const Xbyak::Xmm& sum2, const Xbyak::Xmm& x, const Xbyak::Operand& op,+ const Xbyak::Xmm& ones) {+ vpmaddubsw(sum2, x, op);+ vpmaddwd(sum2, sum2, ones);+ vpaddd(sum4, sum4, sum2);+ }+};++class JitAvx512f : protected JitAvx2 {+ protected:+ static auto constexpr ISA = BTLA_ISA::AVX512F;+ static int constexpr VBits = 512;+ static int constexpr VecBytes = VBits / 8;+ static int constexpr RegCount = 32;+ typedef Xbyak::Zmm vreg_t;++ void vxor(const vreg_t& x1, const vreg_t& x2, const Xbyak::Operand& op) { vpxorq(x1, x2, op); }++ void interleave_2rows_4regs(Xbyak::Zmm* src_2regs, Xbyak::Zmm* tmp_2reg) {+ vpunpcklwd(tmp_2reg[0], src_2regs[0], src_2regs[1]);+ vpunpckhwd(tmp_2reg[1], src_2regs[0], src_2regs[1]);+ vshuff32x4(src_2regs[0], tmp_2reg[0], tmp_2reg[1], 0 | (1 << 2) | (0 << 4) | (1 << 6));+ vshuff32x4(src_2regs[0], src_2regs[0], src_2regs[0], 0 | (2 << 2) | (1 << 4) | (3 << 6));+ vshuff32x4(src_2regs[1], tmp_2reg[0], tmp_2reg[1], 2 | (3 << 2) | (2 << 4) | (3 << 6));+ vshuff32x4(src_2regs[1], src_2regs[1], src_2regs[1], 0 | (2 << 2) | (1 << 4) | (3 << 6));+ }++ void transpose16x16_4B(Xbyak::Zmm* src, Xbyak::Zmm* tmp, const int N = 16) {+ for (int i = 0; i < 8; ++i) {+ vpunpckldq(tmp[2 * i + 0], src[2 * i], src[2 * i + 1]);+ vpunpckhdq(tmp[2 * i + 1], src[2 * i], src[2 * i + 1]);+ }++ for (int i = 0; i < 4; ++i) {+ vpunpcklqdq(src[4 * i + 0], tmp[4 * i + 0], tmp[4 * i + 2]);+ vpunpckhqdq(src[4 * i + 1], tmp[4 * i + 0], tmp[4 * i + 2]);+ vpunpcklqdq(src[4 * i + 2], tmp[4 * i + 1], tmp[4 * i + 3]);+ vpunpckhqdq(src[4 * i + 3], tmp[4 * i + 1], tmp[4 * i + 3]);+ }++ for (int i = 0; i < 2; ++i) {+ vshufi32x4(tmp[8 * i + 0], src[8 * i + 0], src[8 * i + 4], 0x88);+ vshufi32x4(tmp[8 * i + 1], src[8 * i + 1], src[8 * i + 5], 0x88);+ vshufi32x4(tmp[8 * i + 2], src[8 * i + 2], src[8 * i + 6], 0x88);+ vshufi32x4(tmp[8 * i + 3], src[8 * i + 3], src[8 * i + 7], 0x88);+ vshufi32x4(tmp[8 * i + 4], src[8 * i + 0], src[8 * i + 4], 0xdd);+ vshufi32x4(tmp[8 * i + 5], src[8 * i + 1], src[8 * i + 5], 0xdd);+ vshufi32x4(tmp[8 * i + 6], src[8 * i + 2], src[8 * i + 6], 0xdd);+ vshufi32x4(tmp[8 * i + 7], src[8 * i + 3], src[8 * i + 7], 0xdd);+ }++ // last step and move out+ for (int i = 0; i < N; ++i) {+ vshufi32x4(src[i], tmp[i % 8], tmp[8 + i % 8], i < 8 ? 0x88 : 0xdd);+ }+ }++ void interleave_4rows_6regs(Xbyak::Zmm* src_4regs, Xbyak::Zmm* tmp_regs, const Xbyak::Opmask* masks) {+ vpunpcklbw(tmp_regs[0], src_4regs[0], src_4regs[1]);+ vpunpckhbw(tmp_regs[1], src_4regs[0], src_4regs[1]);+ vpunpcklbw(tmp_regs[2], src_4regs[2], src_4regs[3]);+ vpunpckhbw(tmp_regs[3], src_4regs[2], src_4regs[3]);++ vpunpcklwd(tmp_regs[4], tmp_regs[0], tmp_regs[2]);+ vpunpckhwd(tmp_regs[5], tmp_regs[0], tmp_regs[2]);+ vpunpcklwd(tmp_regs[0], tmp_regs[1], tmp_regs[3]);+ vpunpckhwd(tmp_regs[2], tmp_regs[1], tmp_regs[3]);+ vshuff32x4(tmp_regs[1], tmp_regs[4], tmp_regs[0], (4 << 4) | 4);+ vshuff32x4(tmp_regs[3], tmp_regs[5], tmp_regs[2], (4 << 4) | 4);+ vmovups(src_4regs[0], tmp_regs[1]);+ vshuff32x4(src_4regs[0] | masks[0], tmp_regs[3], tmp_regs[3], 0 | (0 << 2) | (0 << 4) | (2 << 6));+ vmovups(src_4regs[1], tmp_regs[3]);+ vshuff32x4(src_4regs[1] | masks[1], tmp_regs[1], tmp_regs[1], 1 | (0 << 2) | (3 << 4) | (0 << 6));+ vshuff32x4(tmp_regs[1], tmp_regs[4], tmp_regs[0], (14 << 4) | 14);+ vshuff32x4(tmp_regs[3], tmp_regs[5], tmp_regs[2], (14 << 4) | 14);+ vmovups(src_4regs[2], tmp_regs[1]);+ vshuff32x4(src_4regs[2] | masks[0], tmp_regs[3], tmp_regs[3], 0 | (0 << 2) | (0 << 4) | (2 << 6));+ vmovups(src_4regs[3], tmp_regs[3]);+ vshuff32x4(src_4regs[3] | masks[1], tmp_regs[1], tmp_regs[1], 1 | (0 << 2) | (3 << 4) | (0 << 6));+ }++ void cvt_fp32_bf16(const Xbyak::Ymm& _bf16, const Xbyak::Zmm& _fp32) {+ vpsrld(_fp32, _fp32, 16);+ vpmovdw(_bf16, _fp32);+ }++ void loadbf16_f32(const Xbyak::Zmm& dst, const Xbyak::Address& addr) {+ vpmovzxwd(dst, addr);+ vpslld(dst, dst, 16);+ }++ void broadcastbf16_f32(const Xbyak::Zmm& dst, const Xbyak::Reg64& tmp, const Xbyak::Address& addr) {+ mov(tmp.cvt16(), addr);+ shl(tmp.cvt32(), 16);+ vpbroadcastd(dst, tmp.cvt32());+ }++ void store_fp32_bf16(const Xbyak::Zmm& _fp32, const Xbyak::Address& _add) {+ auto bf16 = Xbyak::Ymm(_fp32.getIdx());+ cvt_fp32_bf16(bf16, _fp32);+ vmovups(_add, bf16);+ }+};++class JitAvx512bw : protected JitAvx512f {+ static auto constexpr ISA = BTLA_ISA::AVX512BW;++ protected:+ void vpdpbusds_(const Xbyak::Xmm& sum4, const Xbyak::Xmm& sum2, const Xbyak::Xmm& x, const Xbyak::Operand& op,+ const Xbyak::Xmm& ones) {+ vpmaddubsw(sum2, x, op);+ vpmaddwd(sum2, sum2, ones);+ vpaddd(sum4, sum4, sum2);+ }+};++class JitAvx512_bf16 : protected JitAvx512f {+ public:+ static auto constexpr ISA = BTLA_ISA::AVX512_BF16;+};++class JitAvx512_fp16 : protected JitAvx512f {+ public:+ static auto constexpr ISA = BTLA_ISA::AVX512_FP16;+};++class JitAvx512vnni : protected JitAvx512f {+ public:+ static auto constexpr ISA = BTLA_ISA::AVX512_VNNI;++ protected:+ void vpdpbusds_(const Xbyak::Xmm& x1, const Xbyak::Xmm& x2, const Xbyak::Operand& op) {+ vpdpbusds(x1, x2, op, Xbyak::EvexEncoding);+ }+};++class JitAvxvnni : protected JitAvx2 {+ public:+ static auto constexpr ISA = BTLA_ISA::AVX_VNNI;++ protected:+ void vpdpbusds_(const Xbyak::Xmm& x1, const Xbyak::Xmm& x2, const Xbyak::Operand& op) {+ vpdpbusds(x1, x2, op, Xbyak::VexEncoding);+ }+};++class JitAmxtile : protected JitAvx512f {+ public:+ static auto constexpr ISA = BTLA_ISA::AMX_BF16;++ struct alignas(64) tileconfig_t {+ uint8_t palette_id;+ uint8_t reserved[15];+ uint16_t colb[16];+ uint8_t rows[16];+ };+ static int constexpr TileCount = 8;++ typedef long long (*configure_t)(void*);++ static void generate_config(Xbyak::CodeGenerator* g) {+ Xbyak::util::StackFrame st(g, 1, 0, 0);+ auto& parambase = st.p[0];+ g->ldtilecfg(g->ptr[parambase]);+ }++ static void configure_tiles(tileconfig_t& tc, int TILE_M, int TILE_N, int TILE_K, int elesize, int ANum, int BNum,+ int CNum) {+ // Filling tile configure structure. Could be done offline.+ tc.palette_id = 1;+ // Configure C tiles+ int t = 0;+ for (; t < CNum; ++t) {+ tc.rows[t] = static_cast<uint8_t>(TILE_M);+ tc.colb[t] = static_cast<uint16_t>(TILE_N * 4);+ }+ // Configure A tiles+ for (; t < CNum + ANum; ++t) {+ tc.rows[t] = static_cast<uint8_t>(TILE_M);+ tc.colb[t] = static_cast<uint16_t>(TILE_K * elesize);+ }+ // Configure B tile. B effectively has 64 rows and 16 columns.+ int kpack = 4 / elesize;+ for (; t < CNum + ANum + BNum; ++t) {+ tc.rows[t] = static_cast<uint8_t>(TILE_K / kpack);+ tc.colb[t] = static_cast<uint16_t>(TILE_N * 4);+ }+ }+};++class JitAmxbf16 : protected JitAmxtile {+ public:+ static auto constexpr ISA = BTLA_ISA::AMX_BF16;++ protected:+ void cvt_fp32_bf16(const Xbyak::Ymm& _bf16, const Xbyak::Zmm& _fp32) { vcvtneps2bf16(_bf16, _fp32); }+};++class JitAmxint8 : protected JitAmxtile {+ public:+ static auto constexpr ISA = BTLA_ISA::AMX_INT8;++ protected:+ template <class, class>+ void _tdpb(const Xbyak::Tmm& x1, const Xbyak::Tmm& x2, const Xbyak::Tmm& x3);+};+template <>+inline void JitAmxint8::_tdpb<int8_t, int8_t>(const Xbyak::Tmm& x1, const Xbyak::Tmm& x2, const Xbyak::Tmm& x3) {+ tdpbssd(x1, x2, x3);+}+template <>+inline void JitAmxint8::_tdpb<int8_t, uint8_t>(const Xbyak::Tmm& x1, const Xbyak::Tmm& x2, const Xbyak::Tmm& x3) {+ tdpbsud(x1, x2, x3);+}+template <>+inline void JitAmxint8::_tdpb<uint8_t, int8_t>(const Xbyak::Tmm& x1, const Xbyak::Tmm& x2, const Xbyak::Tmm& x3) {+ tdpbusd(x1, x2, x3);+}+template <>+inline void JitAmxint8::_tdpb<uint8_t, uint8_t>(const Xbyak::Tmm& x1, const Xbyak::Tmm& x2, const Xbyak::Tmm& x3) {+ tdpbuud(x1, x2, x3);+}++class JitAmxfp16 : protected JitAmxtile {+ public:+ static auto constexpr ISA = BTLA_ISA::AMX_FP16;++ protected:+ void cvt_fp32_fp16(const Xbyak::Ymm& _fp16, const Xbyak::Zmm& _fp32) { vcvtps2phx(_fp16, _fp32); }+};++template <BTLA_ISA ISA_T>+class JitInfo {+ public:+ static auto constexpr ISA = ISA_T;+ static bool constexpr Has512 = ISA >= BTLA_ISA::AVX512F;+ static bool constexpr HasTile = ISA >= BTLA_ISA::AMX_BF16;+ static int constexpr VBits = Has512 ? 512 : 256;+ static int constexpr VecBytes = VBits / 8;+ static int constexpr VecLen32 = VBits / 32;+ static int constexpr RegCount = Has512 ? 32 : 16;+ static int constexpr TileCount = HasTile ? 8 : 0;+ using vreg_t = std::conditional_t<Has512, Xbyak::Zmm, Xbyak::Ymm>;+ using hvreg_t = std::conditional_t<Has512, Xbyak::Ymm, Xbyak::Xmm>;+ using qvreg_t = Xbyak::Xmm;+};++class JitCode : protected Xbyak::CodeGenerator {+ public:+ BTLA_ISA ISA;++ protected:+ JitCode(size_t size = 16 * 1024) : CodeGenerator(size) {}++ void load32(const Xbyak::Reg64& reg, const Xbyak::Address& addr) {+ xor_(reg, reg);+ mov(reg.cvt32(), addr);+ }++ void vreg_push(const Xbyak::Reg64& baseaddr) {+#ifdef _WIN32+ for (int i = 0; i < 10; i++) {+ movaps(xword[baseaddr + i * 16], Xbyak::Xmm(6 + i));+ }+#endif+ }++ void vreg_pop(const Xbyak::Reg64& baseaddr) {+#ifdef _WIN32+ for (int i = 0; i < 10; i++) {+ movaps(Xbyak::Xmm(6 + i), xword[baseaddr + i * 16]);+ }+#endif+ }++ void padto_le(const Xbyak::Reg64& _src, int padding) {+ // _src=_src/padding*padding+ if (padding == 1) {+ return;+ }+ for (int i = 1; i < 16; i++) {+ if ((1 << i) == padding) {+ shr(_src, i);+ shl(_src, i);+ return;+ }+ }+ assert(0);+ }++ void generate_Nbitsmask(const Xbyak::Opmask& _msk, const Xbyak::Reg64& _pos, const Xbyak::Address& _total,+ const Xbyak::Reg64& _tmp, const Xbyak::Reg64& _tmp1, int N) {+ inLocalLabel();+ lea(_tmp, _total);+ sub(_tmp, _pos);+ cmp(_tmp, N);+ jb(".maskflag");+ cmp(_tmp, 0);+ jl(".zeroflag");+ uint64_t allmask = (static_cast<uint64_t>(1) << N) - 1;+ if (N == 64) {+ allmask = static_cast<uint64_t>(-1);+ }+ mov(_tmp, allmask);+ kmovq(_msk, _tmp);+ jmp(".maskend");+ L(".maskflag");+ mov(_tmp1, 1);+ shlx(_tmp1, _tmp1, _tmp);+ sub(_tmp1, 1);+ kmovq(_msk, _tmp1);+ jmp(".maskend");+ L(".zeroflag");+ mov(_tmp1, 0);+ kmovq(_msk, _tmp1);+ L(".maskend");+ outLocalLabel();+ }++ void generate_Nbitsmask(const Xbyak::Opmask& _msk, const Xbyak::Reg64& _pos, const Xbyak::Reg64& _total,+ const Xbyak::Reg64& _tmp, const Xbyak::Reg64& _tmp1, int N) {+ generate_Nbitsmask(_msk, _pos, ptr[_total], _tmp, _tmp1, N);+ }++ void vpdpbusds_(const Xbyak::Xmm& x1, const Xbyak::Xmm& x2, const Xbyak::Operand& op,+ const Xbyak::Xmm& sum2 = Xbyak::Xmm{}, const Xbyak::Xmm& ones = Xbyak::Xmm{}) {+ if (ISA == BTLA_ISA::AVX512_VNNI) {+ vpdpbusds(x1, x2, op, Xbyak::EvexEncoding);+ } else if (ISA == BTLA_ISA::AVX_VNNI) {+ vpdpbusds(x1, x2, op, Xbyak::VexEncoding);+ } else {+ vpmaddubsw(sum2, x2, op);+ vpmaddwd(sum2, sum2, ones);+ vpaddd(x1, x1, sum2);+ }+ }++ void vxor(const Xbyak::Xmm& x1, const Xbyak::Xmm& x2, const Xbyak::Operand& op) { vxorps(x1, x2, op); }++ template <typename TA, typename TB>+ void _tdpb(const Xbyak::Tmm& x1, const Xbyak::Tmm& x2, const Xbyak::Tmm& x3) {+ assert(ISA >= BTLA_ISA::AMX_BF16);+ if (std::is_same_v<TA, uint8_t>) {+ if (std::is_same_v<TB, int8_t>) {+ tdpbusd(x1, x2, x3);+ } else {+ tdpbuud(x1, x2, x3);+ }+ } else {+ if (std::is_same_v<TB, int8_t>) {+ tdpbssd(x1, x2, x3);+ } else {+ tdpbsud(x1, x2, x3);+ }+ }+ }+};+} // namespace xbyak+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_parallel.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_parallel.hnew file mode 100644index 000000000..32308e0f9--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_parallel.h@@ -0,0 +1,1214 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include <atomic>+#include <functional>+#include <thread>+#include <vector>+#if BTLA_OPENMP+#include <omp.h>+#endif+#include <immintrin.h>+#include "bestla_utils.h"+#include "bestla_device.h"+#include <mutex>++namespace bestla {+namespace parallel {++using thread_func = std::function<void(int tid)>;++class IThreading {+ public:+ explicit IThreading(bool supportPE) : isSupportPE(supportPE) {}++ // equal to "for(int i=begin1;i<end1;i+=step1)"+ void parallel_for_collapse(const int& begin1, const int& end1, const int& step1,+ const std::function<void(int)>& func) {+ assert(end1 >= begin1 && step1 > 0);+ parallel_for([&](int tidx) {+ const int length = (end1 - begin1 + step1 - 1) / step1;+ const int block_size = length / mThreadNum;+ const int block_remain = length % mThreadNum;+ const int offset = std::min(tidx, block_remain) + tidx * block_size;+ for (int i = begin1 + offset * step1; i < begin1 + (offset + block_size + (tidx < block_remain)) * step1;+ i += step1)+ func(i);+ });+ }++ // equal to "for(int i=begin1;i<end1;i+=step1)for(int j=begin2;i<end2;i+=step2)"+ void parallel_for_collapse(const int& begin1, const int& end1, const int& step1, const int& begin2, const int& end2,+ const int& step2, const std::function<void(int, int)>& func) {+ assert(end1 >= begin1 && step1 > 0 && end2 >= begin2 && step2 > 0);+ parallel_for([&](int tidx) {+ const int length1 = (end1 - begin1 + step1 - 1) / step1;+ const int length2 = (end2 - begin2 + step2 - 1) / step2;+ const int block_size = (length1 * length2) / mThreadNum;+ const int block_remain = (length1 * length2) % mThreadNum;+ const int offset = std::min(tidx, block_remain) + tidx * block_size;+ for (int i = offset; i < offset + block_size + (tidx < block_remain); ++i)+ func(begin1 + (i / length2) * step1, begin2 + (i % length2) * step2);+ });+ }+ virtual void parallel_for(const thread_func& func) = 0;+ virtual inline void sync(int tidx, int idx = 0) = 0;+ virtual int num_threads() const { return mThreadNum; };+ virtual int is_support_PE() const { return isSupportPE; };+ virtual void set_threads(int nthreads, bool _hyper = false) = 0;+ virtual std::pair<float, float> get_PEtime() const { return {0.0f, 0.0f}; };++ protected:+ int mThreadNum = 1;+ const bool isSupportPE = false;+};+struct Config {+ size_t size_ = 0;+ size_t step_ = 1;+ int kernel_id_ = -1;+};+struct ThreadInfo {+ int group_idx_;+ int thread_idx_;+};+struct ThreadRange {+ size_t loc_ = 0;+ size_t size_ = 0;+ ThreadInfo info_;+};++using tr_thread_func = std::function<void(const ThreadRange&)>;++class IGroupThreading {+ public:+ // virtual void parallel_for(const gthread_func& func, const int _kernel_id = -1) = 0;+ virtual inline void sync(int tidx, int idx = 0) = 0;+ virtual int num_threads() const { return num_threads_; };+ virtual void set_threads(const device::CpuRange& _thread_group) = 0;++ protected:+ std::vector<std::vector<int>> thread_groups_;+ int num_groups_ = 0;+ int num_threads_ = 0;+};++#if BTLA_OPENMP++class OMPThreading : public IThreading {+ public:+ explicit OMPThreading() : IThreading(false) {+ GetCPUDevice();+ const char* _env = getenv("OMP_NUM_THREADS");+ mThreadNum = _cd->get_actual_usable_cores();+ if (_env != nullptr) mThreadNum = std::stoi(_env);+ omp_set_num_threads(mThreadNum);+ }++ void parallel_for(const thread_func& func) override {+ if (mThreadNum > 1) {+#pragma omp parallel+ {+ int tidx = omp_get_thread_num();+ func(tidx);+ (void)(tidx);+ (void)(0);+ }+ } else {+ func(0);+ void(0);+ }+ }+ virtual void set_threads(int nthreads, bool _hyper) override {+ if (nthreads == 0) {+ nthreads = device::CpuDevice::getInstance()->getCores();+ }+ mThreadNum = nthreads;+ omp_set_num_threads(nthreads);+ }+ virtual inline void sync(int tidx, int idx = 0) override {+ (void)(tidx);+ (void)(idx);+#pragma omp barrier+ (void)(0); // make msvc happy with c++20+ }+};++#endif++class StdThreading : public IThreading {+ public:+ using Timer_T = utils::timer<utils::microseconds>;+ explicit StdThreading() : IThreading(true) {+ cr = nullptr;+ memset(func_, 0, sizeof(func_));+ memset(flag, 0, sizeof(flag));+ stop = true;+ time_per_p = -1.f;+ time_per_e = -1.f;+ }++ void parallel_for(const thread_func& func) override {+ time_per_p = 0;+ time_per_e = 0;+ Timer_T tm;+ if (mThreadNum > 1) {+ running.store(mThreadNum - 1);+ for (int i = 0; i < 10; i++) flag[i].store(mThreadNum);+ if (cr->mHybrid) {+ int64_t time_p = 0, time_e = 0;++ for (size_t i = 0; i < mThreadNum - 1; i++) func_[i] = &func;+ thread_time[0] = 0;+ tm.start();+ func(0);+ thread_time[0] += int(tm.stop());+ while (true) {+ if (running.load() == 0)+ break;+ else+ std::this_thread::yield();+ }+ for (int i = 0; i < mThreadNum; i++)+ if (i >= cr->P_core_num && i < cr->P_core_num + cr->E_core_num)+ time_e += thread_time[i];+ else+ time_p += thread_time[i];+ time_per_p = (time_p) / (1.0f * (mThreadNum - cr->E_core_num));+ time_per_e = (time_e) / (1.0f * cr->E_core_num);+ // printf("%d %d %f %f\n", time_p, time_e, time_per_p, time_per_e);+ } else {+ for (size_t i = 0; i < mThreadNum - 1; i++) {+ func_[i] = &func;+ }+ func(0);+ while (true) {+ if (running.load() == 0)+ break;+ else+ std::this_thread::yield();+ }+ }+ } else {+ func(0);+ }+ }++ void set_threads(int nthreads, bool _hyper) override {+ if (nthreads == 0) {+ nthreads = device::CpuDevice::getInstance()->getCores();+ }+ if (nthreads != mThreadNum) {+ stop_threads();+ mThreadNum = nthreads;+ cr = &device::CpuRuntime::getInstance(nthreads);+ create_threads();+ }+ }++ inline void sync(int tidx, int idx = 0) override {+ if (mThreadNum > 1) {+ flag[idx]--;+ if (cr->mHybrid) {+ Timer_T tm;+ tm.start();+ while (true) {+ if (flag[idx].load() == 0)+ break;+ else+ std::this_thread::yield();+ }+ thread_time[tidx] -= int(tm.stop());+ } else {+ while (true) {+ if (flag[idx].load() == 0)+ break;+ else+ std::this_thread::yield();+ }+ }+ }+ }++ std::pair<float, float> get_PEtime() const override { return {time_per_p, time_per_e}; };++ ~StdThreading() { stop_threads(); }++ private:+ void stop_threads() {+ stop = 1;+ for (int i = 0; i < mThreadNum - 1; i++) thdset[i].join();+ thdset.clear();+ }+ void create_threads() {+ thdset.resize(mThreadNum - 1);+ stop = 0;+ GetCPUDevice();+ std::vector<int> core_order;+ if (_cd->isHybrid()) {+ core_order.resize(_cd->getThreads());+ memcpy(reinterpret_cast<void*>(core_order.data()), reinterpret_cast<void*>(_cd->getPCores()),+ _cd->getPcoreNum() * sizeof(int));+ memcpy(reinterpret_cast<void*>(core_order.data() + _cd->getPcoreNum()), reinterpret_cast<void*>(_cd->getECores()),+ _cd->getEcoreNum() * sizeof(int));+ memcpy(reinterpret_cast<void*>(core_order.data() + _cd->getPcoreNum() + _cd->getEcoreNum()),+ reinterpret_cast<void*>(_cd->getSMTCores()), _cd->getSMTcoreNum() * sizeof(int));+ } else {+ core_order.resize(_cd->getCores() * 2); // *2 for SMT+ if (_cd->isClient()) {+ for (int i = 0; i < _cd->getCores(); i++) {+ core_order[i] = 2 * i;+ core_order[i + _cd->getCores()] = 2 * i + 1;+ }+ } else {+ for (int i = 0; i < _cd->getCores() * 2; i++) core_order[i] = i;+ }+ }+ _cd->core_bond(core_order[0]);+ if (cr->mHybrid) {+ thread_time.resize(mThreadNum);+ for (size_t i = 0; i < mThreadNum - 1; i++) {+ thdset[i] = std::thread(+ [&](int tidx, int core_id) {+ _cd->core_bond(core_id);+ Timer_T tm;+ while (true) {+ if (stop) break;+ if (func_[tidx] != nullptr) {+ thread_time[tidx + 1] = 0;+ tm.start();+ (*func_[tidx])(tidx + 1);+ func_[tidx] = nullptr;+ thread_time[tidx + 1] += int(tm.stop());+ running.fetch_sub(1);+ } else {+ std::this_thread::yield();+ }+ }+ },+ int(i), core_order[i + 1]);+ }+ } else+ for (size_t i = 0; i < mThreadNum - 1; i++) {+ thdset[i] = std::thread(+ [&](int tidx, int core_id) {+ _cd->core_bond(core_id);+ while (true) {+ if (stop) break;+ if (func_[tidx] != nullptr) {+ (*func_[tidx])(tidx + 1);+ func_[tidx] = nullptr;+ running--;+ } else {+ std::this_thread::yield();+ }+ }+ },+ int(i), core_order[i + 1]);+ }+ }+ device::CpuRuntime* cr;+ std::atomic_int64_t running;+ std::atomic_int64_t flag[64];+ const thread_func* func_[1024];+ bool stop;+ std::vector<std::thread> thdset;+ std::vector<int64_t> thread_time;+ float time_per_p, time_per_e;+};+#if 0+class GroupScheduler {+ public:+ void schedule(const Config& config, float*) {+ cfg_ = config;+ g_range_ = {0, cfg_.size};+ n_group_ = _th->group_threads_.size();+ if (n_group_ == 1) {+ group_range_[0].loc = 0;+ group_range_[0].size = cfg_.size;+ } else {+ size_t loc = 0;+ int off = 0;+ for (size_t i = 0; i < n_group_; i++) {+ group_off_[i] = off;+ off += _th->group_threads_[i];+ group_range_[i].loc = loc;+ size_t size = cfg_.size * _th->pe[i];+ size = utils::padto(size, cfg_.step);+ size = utils::remainsize(loc, cfg_.size, size);+ if (i == n_group_ - 1) {+ size = cfg_.size - loc;+ }+ group_range_[i].size = size;+ loc += size;+ }+ }++ for (size_t i = 0; i < n_group_; i++) {+ size_t loc = group_range_[i].loc;+ size_t size = group_range_[i].size;+ size_t off = group_off_[i];+ int nt = _th->group_threads_[i];+ size_t avg_size = utils::padto(size / nt, cfg_.step);+ for (size_t j = 0; j < nt; j++) {+ auto rsize = utils::remainsize(loc, group_range_[i].loc + size, avg_size);+ if (j == nt - 1) {+ rsize = group_range_[i].loc + size - loc;+ }+ threads_range_[off + j] = ThreadRange{loc, rsize};+ loc += rsize;+ }+ }+ }++ inline ThreadRange get_range(const GroupThread& gt) {+ return threads_range_[group_off_[gt.group_idx_] + gt.thread_idx_];+ }++ Config cfg_;+ static int constexpr MAX_GROUP = 8;+ static int constexpr MAX_THREADS = 256;+ ThreadRange group_range_[MAX_GROUP], threads_range_[MAX_THREADS];+ int group_off_[MAX_GROUP]{0};+ int n_group_ = 0;+ ThreadRange g_range_{0};+};+#endif+class StdGroupThreading : public IGroupThreading {+ public:+ using Timer_T = utils::timer<utils::microseconds>;+ StdGroupThreading() {+ memset(func_, 0, sizeof(func_));+ memset(flag, 0, sizeof(flag));+ stop = true;+ }+ ~StdGroupThreading() { stop_threads(); }+ Config cfg_;+ std::vector<ThreadRange> group_range_, threads_range_;+ std::vector<int> group_off_;+ void schedule(const Config& config) {+ cfg_ = config;+ bool bdynamic = (config.kernel_id_ >= 0 && config.kernel_id_ < MAX_KERNELS);+ if (num_groups_ == 1) {+ group_range_[0].loc_ = 0;+ group_range_[0].size_ = cfg_.size_;+ } else {+ size_t loc = 0;+ for (size_t i = 0; i < num_groups_; i++) {+ group_range_[i].loc_ = loc;+ float ratio = bdynamic ? group_kernelperf_[config.kernel_id_][i] : 1.f / num_groups_;+ size_t size = cfg_.size_ * ratio;+ size = utils::padto(size, cfg_.step_);+ size = utils::remainsize(loc, cfg_.size_, size);+ if (i == num_groups_ - 1) {+ size = cfg_.size_ - loc;+ }+ group_range_[i].size_ = size;+ loc += size;+ }+ }++ for (size_t i = 0; i < num_groups_; i++) {+ size_t loc = group_range_[i].loc_;+ size_t size = group_range_[i].size_;+ size_t off = group_off_[i];+ int nt = group_threads_[i];+ size_t avg_size = utils::padto(size / nt, cfg_.step_);+ for (size_t j = 0; j < nt; j++) {+ auto rsize = utils::remainsize(loc, group_range_[i].loc_ + size, avg_size);+ if (j == nt - 1) {+ rsize = group_range_[i].loc_ + size - loc;+ }+ threads_range_[off + j] = ThreadRange{loc, rsize};+ loc += rsize;+ }+ }+ }++ inline ThreadRange get_range(const ThreadInfo& ti) {+ auto tr = threads_range_[group_off_[ti.group_idx_] + ti.thread_idx_];+ tr.info_ = ti;+ return tr;+ }++ void parallel_for(const Config& config, const tr_thread_func& func) {+ Timer_T tm;+ profiling_ = profile_flag_ && (num_groups_ > 1) && (iter_ % update_iter_ == 0);+ schedule(config);+ if (thread_pool_.size() == 0) {+ func(ThreadRange{0, cfg_.size_});+ } else {+ running.store(thread_pool_.size());+ int64_t time_p = 0, time_e = 0;+ for (size_t i = 0; i < thread_pool_.size(); i++) func_[i] = &func;+ this_thread_time_ = 0;+ if (profiling_) tm.start();+ func(get_range(this_info_));+ if (profiling_) this_thread_time_ += int(tm.stop());+ while (true) {+ if (running.load() == 0)+ break;+ else+ std::this_thread::yield();+ }+ }+ iter_++;+ if (profiling_) {+ update_performance(config.kernel_id_);+ }+ }++ void update_performance(const int _kernel_id) {+ if (_kernel_id < 0 || _kernel_id >= MAX_KERNELS) {+ return;+ }+ std::fill(group_time_.begin(), group_time_.end(), 0);+ group_time_[this_info_.group_idx_] = this_thread_time_;+ for (size_t i = 0; i < thread_time_.size(); i++) {+ group_time_[thread_info_[i].group_idx_] = std::max(group_time_[thread_info_[i].group_idx_], thread_time_[i]);+ }+ float sum = 0;+ float eps = 1e-9;+ for (size_t i = 0; i < group_time_.size(); i++) {+ sum += group_kernelperf_[_kernel_id][i] / (group_time_[i] + eps);+ }+ sum = 1.f / sum;+ float constexpr filter = 0.7f;+ for (size_t i = 0; i < group_time_.size(); i++) {+ auto new_perf = group_kernelperf_[_kernel_id][i] / (group_time_[i] + eps) * sum;+ // printf("Group %d, from %f to %f\n", i, group_kernelperf_[_kernel_id][i], new_perf);+ group_kernelperf_[_kernel_id][i] = new_perf * filter + (1 - filter) * group_kernelperf_[_kernel_id][i];+ }+ }++ void set_threads(const device::CpuRange& _crange) override {+ stop_threads();+ cpu_range_ = _crange;+ thread_groups_ = cpu_range_.get_groups();+ num_groups_ = thread_groups_.size();+ this_core_ = thread_groups_[0][0];+ num_threads_ = 0;+ for (size_t i = 0; i < num_groups_; i++) {+ for (size_t j = 0; j < thread_groups_[i].size(); j++) {+ num_threads_++;+ }+ }+ create_threads();+ }++ void enable_profiling(int update_iter = 1) {+ update_iter_ = update_iter;+ profile_flag_ = true;+ }++ void disable_profiling() { profile_flag_ = false; }++ inline void sync(int tidx, int idx = 0) override {+ if (thread_pool_.size()) {+ flag[idx].fetch_add(-1);+ Timer_T tm;+ if (profiling_) tm.start();+ while (true) {+ if (flag[idx].load() == 0)+ break;+ else+ std::this_thread::yield();+ }+ if (profiling_) thread_time_[tidx] -= int(tm.stop());+ }+ }++ void stop_threads() {+ stop = true;+ for (int i = 0; i < thread_pool_.size(); i++) thread_pool_[i].join();+ }++ void create_threads() {+ stop = false;+ iter_ = 0;+ thread_pool_.clear();+ cpu_range_.core_bond(this_core_);+ auto sub_thread_func = [&](int tidx, int core_id) {+ cpu_range_.core_bond(core_id);+ Timer_T tm;+ while (true) {+ if (func_[tidx] != nullptr) {+ if (profiling_) {+ thread_time_[tidx] = 0;+ tm.start();+ }+ (*func_[tidx])(get_range(thread_info_[tidx]));+ if (profiling_) {+ thread_time_[tidx] += int(tm.stop());+ }+ func_[tidx] = nullptr;+ running--;+ } else {+ std::this_thread::yield();+ }+ if (stop) break;+ }+ };+ for (size_t i = 0; i < MAX_KERNELS; i++) {+ group_kernelperf_[i].resize(num_groups_);+ std::fill(group_kernelperf_[i].begin(), group_kernelperf_[i].end(), 1.f / num_groups_);+ }+ group_threads_.resize(num_groups_, 1);+ group_time_.resize(num_groups_, 0);+ group_off_.resize(num_groups_);+ group_range_.resize(num_groups_);+ group_thread_idx_ = thread_groups_;+ this_info_.group_idx_ = 0;+ this_info_.thread_idx_ = 0;+ threads_range_.resize(num_threads_);+ if (num_threads_ > 1) {+ thread_pool_.resize(num_threads_ - 1);+ thread_time_.resize(num_threads_ - 1);+ thread_info_.resize(num_threads_ - 1);+ int a = 0;+ int off = 0;+ for (size_t i = 0; i < num_groups_; i++) {+ group_off_[i] = off;+ off += thread_groups_[i].size();+ group_threads_[i] = thread_groups_[i].size();+ for (size_t j = 0; j < thread_groups_[i].size(); j++) {+ group_thread_idx_[i][j] = a;+ if (thread_groups_[i][j] == this_core_) {+ group_thread_idx_[i][j] = -1;+ continue;+ }+ thread_info_[a] = ThreadInfo{(int)i, (int)j};+ thread_pool_[a] = std::thread(sub_thread_func, a, thread_groups_[i][j]);+ a++;+ }+ }+ }+ }+ std::atomic_int64_t running;+ std::atomic_int64_t flag[64];+ const tr_thread_func* func_[1024];+ volatile bool stop;+ volatile bool profiling_ = false;+ bool profile_flag_ = false;+ std::vector<std::vector<int>> group_thread_idx_;+ std::vector<std::thread> thread_pool_;+ std::vector<ThreadInfo> thread_info_;+ ThreadInfo this_info_;+ int this_core_;+ std::vector<int64_t> thread_time_, group_time_;+ int64_t this_thread_time_;+ uint32_t iter_ = 0, update_iter_ = 1;+ device::CpuRange cpu_range_;+ std::vector<int> group_threads_;+ static int constexpr MAX_KERNELS = 256;+ std::vector<float> group_kernelperf_[256];+};++class SingleThread : public IThreading {+ public:+ SingleThread() : IThreading(false) { mThreadNum = 1; }++ void set_threads(int nthreads, bool _hyper) override {+ assert(0);+ (void)(nthreads);+ }++ inline void parallel_for(const thread_func& func) override { func(0); }++ inline void sync(int tidx, int idx = 0) override {+ (void)(tidx);+ (void)(idx);+ }+};++struct Config2D {+ int threads;+ int size[2];+ int step[2];+ int offset[2];+};+struct ThreadProblem2D {+ int tid;+ int tidx[2];+ int loc[2];+ int size[2];+ bool valid;+ void print() {+ printf("Thread %d indice:(%d,%d)\n", tid, tidx[0], tidx[1]);+ printf("Thread location:(%d,%d)\n", loc[0], loc[1]);+ printf("Thread problem size:(%d,%d)\n", size[0], size[1]);+ }+};+class Scheduler2D {+ public:+ Scheduler2D() = default;+ Scheduler2D(const Config2D& config) { update(config); }+ using ThreadProblem = ThreadProblem2D;++ virtual void getIndex(ThreadProblem& problem) const {+ if (problem.tid >= mThdValid) {+ problem.size[0] = 0;+ problem.size[1] = 0;+ problem.valid = false;+ return;+ }+ auto& tid = problem.tid;+ problem.tidx[1] = tid % mThdPerRow;+ problem.tidx[0] = tid / mThdPerRow;+ problem.loc[0] = problem.tidx[0] * mThdSize[0];+ problem.loc[1] = problem.tidx[1] * mThdSize[1];+ problem.size[0] = utils::remainsize(problem.loc[0], mSize[0], mThdSize[0]);+ problem.size[1] = utils::remainsize(problem.loc[1], mSize[1], mThdSize[1]);+ problem.loc[0] += moffset[0];+ problem.loc[1] += moffset[1];+ problem.valid = true;+ }++ virtual void update(const Config2D& config) {+ mThdCount = config.threads;+ for (size_t i = 0; i < 2; i++) {+ mSize[i] = config.size[i];+ mStep[i] = config.step[i];+ moffset[i] = config.offset[i];+ }+ schedule();+ }++ constexpr static BTLA_ISA gemm_ISA() { return BTLA_ISA::NoSIMD; }++ void print() {+ printf("Thread Block:(%d,%d)\n", mThdSize[0], mThdSize[1]);+ printf("Thread in use:%d of %d, Nx%d\n", mThdValid, mThdCount, mThdPerRow);+ }++ constexpr int* thread_size() { return mThdSize; }++ protected:+ void set(const int* thdsize, const int* size, const int* step) {+ for (size_t i = 0; i < 2; i++) {+ mThdSize[i] = thdsize[i];+ mSize[i] = size[i];+ mStep[i] = step[i];+ }+ }+ void schedule() {+ int rownum = utils::updiv(mSize[0], mStep[0]);+ int colnum = utils::updiv(mSize[1], mStep[1]);+ float ratio = colnum * rownum / static_cast<float>(mThdCount);+ if (ratio <= 1) {+ mThdSize[0] = mStep[0];+ mThdSize[1] = mStep[1];+ mThdPerRow = colnum;+ calc_valid_threads();+ return;+ }+ float colratio = ratio > colnum ? colnum : ceil(ratio);+ mThdSize[1] = static_cast<int>(colratio * mStep[1]);+ mThdPerRow = static_cast<int>(ceil(static_cast<float>(colnum) / colratio));+ mThdSize[0] = static_cast<int>(ceil(rownum / (static_cast<float>(mThdCount) / mThdPerRow)) * mStep[0]);+ calc_valid_threads();+ }+ void calc_valid_threads() {+ mThdValid = mThdPerRow * static_cast<int>(std::ceil(static_cast<float>(mSize[0]) / mThdSize[0]));+ }++ int mThdPerRow = 0;+ int mThdValid = 0;+ int mThdCount = 0;+ int moffset[2] = {0, 0};++ private:+ int mThdSize[2] = {0, 0};+ int mSize[2] = {0, 0};+ int mStep[2] = {0, 0};+};++namespace gemm {++struct Config {+ const int threads;+ const utils::GemmProblem problem;+ const int offset[2];+ const size_t l2cache = 1024ULL * 1024;+ const size_t l1cache = 32ULL * 1024;+};++struct ThreadProblemBase : ThreadProblem2D {+ int block[3];+ size_t stacksize;+ size_t tmpcachesize;+};++template <class _GemmCore_T>+class SchedulerBase : public Scheduler2D {+ public:+ using ThreadProblem = ThreadProblemBase;+ SchedulerBase() = default;+ SchedulerBase(const Config& config) { update(config); }+ virtual void getIndex(ThreadProblem& problem) {+ problem.tmpcachesize = mL2Size - mL2Use;+ problem.stacksize = mL2Size;+ problem.block[0] = mBlock[0];+ problem.block[1] = mBlock[1];+ problem.block[2] = mBlock[2];+ Scheduler2D::getIndex(problem);+ }++ virtual void update(const Config& config) {+ for (size_t i = 0; i < 3; i++) {+ mSize[i] = config.problem.dims[i + 1]; // skip 0th batch+ mSizePadded[i] = utils::padto(mSize[i], mStep[i]);+ }+ mThdCount = config.threads;+ mL2Size = config.l2cache;+ mL1Size = config.l1cache;+ Scheduler2D::moffset[0] = config.offset[0];+ Scheduler2D::moffset[1] = config.offset[1];+ if (mSize[0] <= 0 || mSize[1] <= 0 || mSize[2] <= 0) {+ return;+ }+ schedule();+ assert(this->mL2Use <= this->mL2Size - ReservedSize);+ assert(this->mBlock[0] > 0);+ assert(this->mBlock[1] > 0);+ assert(this->mBlock[2] > 0);+ }++ constexpr static BTLA_ISA gemm_ISA() { return _GemmCore_T::ISA; }++ constexpr int valid_theads() { return mThdValid; }++ virtual void print() {+ printf("Thread Block:(%d,%d)\n", mThdSize[0], mThdSize[1]);+ printf("Thread in use:%d of %d, Nx%d\n", mThdValid, mThdCount, mThdPerRow);+ printf("GEMM MStep:%d NStep:%d KStep:%d\n", mBlock[0], mBlock[1], mBlock[2]);+ printf("Cache Size:%zu used:%zu\n", mL2Size, mL2Use);+ }++ template <class T>+ friend class SchedulerDispatcher;++ protected:+ virtual void schedule() {+ int rownum = utils::updiv(mSize[0], mStep[0]);+ int colnum = utils::updiv(mSize[1], mStep[1]);+ mDensity = static_cast<float>(mSize[0]) * mSize[1] / (mSize[0] + mSize[1]);+ int maxN = 0;+ float maxScore = std::numeric_limits<float>::min();+ int core_enum = static_cast<int>(std::sqrt(mThdCount));+ for (int i = 1; i <= core_enum; i += 1) {+ generate_by_cores(i, mThdCount / i, rownum, colnum);+ auto thdscore = calculate_score();+ if (maxScore < thdscore) {+ maxScore = thdscore;+ maxN = i;+ }+ generate_by_cores(mThdCount / i, i, rownum, colnum);+ thdscore = calculate_score();+ if (maxScore < thdscore) {+ maxScore = thdscore;+ maxN = mThdCount / i;+ }+ }+ generate_by_cores(maxN, mThdCount / maxN, rownum, colnum);+ update_cache_blocking();+ Scheduler2D::set(mThdSize, mSize, mStep);+ mL2Use = static_cast<size_t>(mBlock[0]) * mBlock[1] * mEleSize[2];+ mL2Use += static_cast<size_t>(mBlock[1]) * mBlock[2] * mEleSize[1] * 2;+ mL2Use += static_cast<size_t>(mStep[0]) * mBlock[2] * mEleSize[0] * 2;+ }+ static float constexpr DensityThres = 16;+ static size_t constexpr ReservedSize = 32ULL * 1024ULL;++ virtual float calculate_score() {+ int tmpnstep = mThdSize[1] < _GemmCore_T::PREFERRED_N ? mThdSize[1] : _GemmCore_T::PREFERRED_N;+ float threadratio = static_cast<float>(mThdValid) / mThdCount;+ float density = static_cast<float>(tmpnstep) * mThdSize[0] / (tmpnstep + mThdSize[0]);+ if (mDensity < DensityThres) {+ return threadratio;+ }+ return (threadratio * 1.f + density * 0.0016f);+ }++ virtual void generate_by_cores(int ny, int nx, int rownum, int colnum) {+ mThdSize[0] = utils::updiv(rownum, ny) * mStep[0];+ mThdSize[1] = utils::updiv(colnum, nx) * mStep[1];+ mThdPerRow = utils::updiv(mSize[1], mThdSize[1]);+ mThdValid = utils::updiv(mSize[0], mThdSize[0]) * mThdPerRow;+ }++ // cache = mMStep * mNStep * CSize + mNStep * mKStep * BSize+ // = mNStep * (mMStep*CSize + mKStep*BSize)+ // C Access = K/mKStep+ // B Access = M/mMStep+ // A Access = N/mNStep+ virtual void update_cache_blocking() {+ if (mDensity <= DensityThres) {+ return cache_blocking_memory();+ } else {+ return cache_blocking_compute();+ }+ }++ virtual void cache_blocking_compute() {+ size_t constexpr KRef = 256;+ size_t valid_total = mL2Size - ReservedSize;+ size_t asize = KRef * mStep[0] * mEleSize[0] * 2;+ size_t bsize = _GemmCore_T::PREFERRED_N * KRef * mEleSize[1] * 2;+ int maxM = static_cast<int>((valid_total - bsize - asize) / (_GemmCore_T::PREFERRED_N * mEleSize[2]));+ maxM = utils::downdiv(maxM, mStep[0]);+ int nthdm = mThdSize[0] / mStep[0];+ if (maxM < nthdm) {+ int niter = utils::updiv(nthdm, maxM);+ mBlock[0] = utils::updiv(nthdm, niter) * mStep[0];+ } else {+ mBlock[0] = mThdSize[0];+ }+ int maxN = static_cast<int>((valid_total - asize) / (mBlock[0] * mEleSize[2] + KRef * mEleSize[1] * 2));+ maxN = utils::downdiv(maxN, mStep[1]);+ int nthdn = mThdSize[1] / mStep[1];+ if (maxN < nthdn) {+ int niter = utils::updiv(nthdn, maxN);+ mBlock[1] = utils::updiv(nthdn, niter) * mStep[1];+ } else {+ mBlock[1] = mThdSize[1];+ }+ bsize = KRef * mBlock[1] * mEleSize[1] * 2;+ size_t csize = static_cast<size_t>(mBlock[0]) * mBlock[1] * mEleSize[2];+ auto rawk = static_cast<int>((valid_total - csize) / (mStep[0] * mEleSize[0] + mBlock[1] * mEleSize[1]) / 2);+ rawk = std::min(rawk, mSizePadded[2]);+ mBlock[2] = utils::padto_le(rawk, mStep[2]);+ }++ virtual void cache_blocking_memory() {+ mBlock[0] = mThdSize[0];+ mBlock[1] = mStep[1];+ size_t reservsize = static_cast<size_t>(mBlock[0]) * mBlock[1] * mEleSize[2];+ size_t maxK = (mL1Size - reservsize) / (mBlock[1] * mEleSize[1] + mBlock[0] * mEleSize[0]);+ size_t Bsize = maxK * mBlock[1] * mEleSize[1];+ size_t Bsize_1K = utils::padto_le(Bsize, 1024);+ mBlock[2] = static_cast<int>(Bsize_1K / mEleSize[1] / mBlock[1]);+ mBlock[2] = utils::padto_le(mBlock[2], mStep[2]);+ }++ size_t mL2Size = 0, mL1Size = 0, mL2Use = 0;+ float mDensity = 0.f;+ int mSize[3] = {0, 0, 0};+ int mThdSize[3] = {0, 0, 0};+ static constexpr int mStep[3] = {_GemmCore_T::MTILE, _GemmCore_T::NTILE, _GemmCore_T::KTILE};+ static constexpr int mEleSize[3] = {sizeof(typename _GemmCore_T::AType), sizeof(typename _GemmCore_T::BType),+ sizeof(typename _GemmCore_T::CType)};+ int mSizePadded[3] = {0, 0, 0};+ int mBlock[3] = {0, 0, 0};+};++template <class _GemmCore_T>+class SchedulerIntKBlock : public SchedulerBase<_GemmCore_T> {+ // Block[2]: block size of K must be multiplier of mKBlock+ // or factor of mKBlock+ public:+ using ThreadProblem = ThreadProblemBase;+ using BaseScheduler = SchedulerBase<_GemmCore_T>;+ SchedulerIntKBlock() = default;+ SchedulerIntKBlock(const Config& config) { update(config); }++ void update(const Config& config) {+ mKBlock = config.problem.dims[4];+ BaseScheduler::update(config);+ auto blks = utils::updiv(this->mBlock[2], mKBlock);+ this->mL2Use += static_cast<size_t>(blks) * (this->mBlock[1] + this->mStep[0]) *+ (sizeof(float) + sizeof(int8_t) + sizeof(float)); // scale+zp+reduce+ assert(this->mL2Use <= this->mL2Size - ReservedSize);+ assert(this->mBlock[0] >= 0);+ assert(this->mBlock[1] >= 0);+ assert(this->mBlock[2] >= 0);+ assert(this->mBlock[2] % _GemmCore_T::KTILE == 0);+ }++ constexpr static BTLA_ISA gemm_ISA() { return _GemmCore_T::ISA; }++ template <class T>+ friend class SchedulerDispatcher;++ protected:+ static float constexpr DensityThres = 16;+ static size_t constexpr ReservedSize = 32ULL * 1024ULL;++ virtual void update_cache_blocking() override {+ size_t KRef = this->mKBlock;+ size_t constexpr CorSize = sizeof(float) + sizeof(int8_t) + sizeof(float);+ size_t valid_total = this->mL2Size - ReservedSize;+ auto blks = utils::updiv(KRef, this->mKBlock);+ auto asize = KRef * this->mStep[0] * this->mEleSize[0] + blks * this->mStep[0] * CorSize;+ asize *= 2;+ auto bsize = _GemmCore_T::PREFERRED_N * KRef * this->mEleSize[1] + _GemmCore_T::PREFERRED_N * blks * CorSize;+ asize *= 2;+ size_t csize_total = valid_total - asize - bsize;+ int maxM = static_cast<int>(csize_total / _GemmCore_T::PREFERRED_N / this->mEleSize[2]);+ maxM = utils::downdiv(maxM, this->mStep[0]);+ int nthdm = this->mThdSize[0] / this->mStep[0];+ if (maxM < nthdm) {+ int niter = utils::updiv(nthdm, maxM);+ this->mBlock[0] = utils::updiv(nthdm, niter) * this->mStep[0];+ } else {+ this->mBlock[0] = this->mThdSize[0];+ }+ int maxN = static_cast<int>((valid_total - asize) / (this->mBlock[0] * this->mEleSize[2] ++ (KRef * this->mEleSize[1] + blks * CorSize) * 2));+ maxN = utils::downdiv(maxN, this->mStep[1]);+ int nthdn = this->mThdSize[1] / this->mStep[1];+ if (maxN < nthdn) {+ int niter = utils::updiv(nthdn, maxN);+ this->mBlock[1] = utils::updiv(nthdn, niter) * this->mStep[1];+ } else {+ this->mBlock[1] = this->mThdSize[1];+ }+ size_t csize = static_cast<size_t>(this->mBlock[0]) * this->mBlock[1] * this->mEleSize[2];+ auto rawk = static_cast<int>((valid_total - csize) / 2 /+ (this->mStep[0] * this->mEleSize[0] ++ float(CorSize * (this->mStep[0] + this->mBlock[1])) / this->mKBlock ++ this->mBlock[1] * this->mEleSize[1]));+ assert(rawk >= this->mKBlock);+ rawk = std::min(rawk, this->mSizePadded[2]);+ this->mBlock[2] = utils::padto_le(rawk, this->mStep[2]);+ if (this->mBlock[2] > this->mKBlock) {+ this->mBlock[2] = utils::padto_le(this->mBlock[2], this->mKBlock);+ }+ }++ int mKBlock{0};+};++template <class Scheduler>+class SchedulerDispatcher {+ public:+ using ThreadProblem = ThreadProblemBase;+ SchedulerDispatcher() = default;+ ~SchedulerDispatcher() {+ std::pair<float, float> PEtime = th_->get_PEtime();+ if (needDispatch && int(PEtime.first) > 0 && int(PEtime.second) > 0)+ cr->adjustPE(Scheduler::gemm_ISA(), PEtime.second / PEtime.first);+ }+ SchedulerDispatcher(const IThreading* th, const utils::GemmProblem& problem) {+ th_ = th;+ cr = &device::CpuRuntime::getInstance(th->num_threads());+ needDispatch = cr->mHybrid && th->is_support_PE();+ if (!needDispatch) {+ Scheduler_P = std::move(Scheduler({th->num_threads(), problem, {0, 0}, cr->mL2Cache, cr->mL1Cache}));+ } else {+ Pcore_num = cr->P_core_num;+ Ecore_num = cr->E_core_num;+ utils::GemmProblem problem_P = problem, problem_E = problem;+ const int N = problem.dims[2];+ auto PE_Ratio = cr->getPE(Scheduler::gemm_ISA());+ int N_offset = utils::padto(N - int(N / (1 + PE_Ratio)), Scheduler::mStep[1]);+ N_offset = N_offset <= N ? N_offset : N;+ problem_P.dims[2] = N_offset;+ Scheduler_P =+ std::move(Scheduler({th->num_threads() - cr->E_core_num, problem_P, {0, 0}, cr->mL2Cache_P, cr->mL1Cache_P}));+ problem_E.dims[2] = N - N_offset;+ Scheduler_E = std::move(Scheduler({cr->E_core_num, problem_E, {0, N_offset}, cr->mL2Cache_E, cr->mL1Cache_E}));+ }+ }++ void getIndex(ThreadProblem& problem) {+ if (!needDispatch) {+ Scheduler_P.getIndex(problem);+ } else {+ if (problem.tid >= Pcore_num + Ecore_num) {+ problem.tid -= Ecore_num;+ Scheduler_P.getIndex(problem);+ } else if (problem.tid >= Pcore_num) {+ problem.tid -= Pcore_num;+ Scheduler_E.getIndex(problem);+ } else {+ Scheduler_P.getIndex(problem);+ }+ }+ }++ void print() {+ printf("dispatch to hybrid:%d\n", needDispatch);+ Scheduler_P.print();+ if (needDispatch) Scheduler_E.print();+ }++ private:+ Scheduler Scheduler_P, Scheduler_E;+ const IThreading* th_;+ device::CpuRuntime* cr;+ bool needDispatch = false;+ int Pcore_num = 0, Ecore_num = 0;+};++template <>+class SchedulerDispatcher<Scheduler2D> {+ public:+ using ThreadProblem = ThreadProblem2D;+ SchedulerDispatcher() = default;+ ~SchedulerDispatcher() {}+ SchedulerDispatcher(const IThreading* th, const Config2D& config) {+ device::CpuRuntime& cr = device::CpuRuntime::getInstance(config.threads);+ needDispatch = cr.mHybrid && th->is_support_PE();+ if (!needDispatch) {+ Scheduler_P = std::move(Scheduler2D(config));+ } else {+ Pcore_num = cr.P_core_num;+ Ecore_num = cr.E_core_num;+ Config2D config_P = config, config_E = config;+ const int N = config.size[1];+ const auto pe = cr.getPE(BTLA_ISA::NoSIMD);+ const int N_offset = utils::padto(N - int(N / (1 + pe)), config.step[1]);+ config_P.threads = config.threads - cr.E_core_num;+ config_P.size[1] = N_offset;+ Scheduler_P = std::move(Scheduler2D(config_P));+ config_E.threads = cr.E_core_num;+ config_E.size[1] = N - N_offset;+ config_E.offset[1] += N_offset;+ Scheduler_E = std::move(Scheduler2D(config_E));+ }+ }++ void getIndex(ThreadProblem& problem) {+ if (!needDispatch) {+ Scheduler_P.getIndex(problem);+ } else {+ if (problem.tid >= Pcore_num + Ecore_num) {+ problem.tid -= Ecore_num;+ Scheduler_P.getIndex(problem);+ } else if (problem.tid >= Pcore_num) {+ problem.tid -= Pcore_num;+ Scheduler_E.getIndex(problem);+ } else {+ Scheduler_P.getIndex(problem);+ }+ }+ }++ void print() {+ printf("dispatch to hybrid:%d\n", needDispatch);+ Scheduler_P.print();+ if (needDispatch) Scheduler_E.print();+ }++ private:+ Scheduler2D Scheduler_P, Scheduler_E;+ bool needDispatch = false;+ int Pcore_num = 0, Ecore_num = 0;+};+#if 0+class GroupScheduler {+ public:+ GroupScheduler(const utils::GemmProblem& _prob, device::CpuRange& _range) {+ cfg_ = config;+ g_range_ = {0, cfg_.size};+ n_group_ = n_group;+ if (n_group_ == 1) {+ group_range_[0].loc = 0;+ group_range_[0].size = cfg_.size;+ } else {+ size_t loc = 0;+ int off = 0;+ for (size_t i = 0; i < n_group; i++) {+ group_off_[i] = off;+ off += _gthreads[i];+ group_range_[i].loc = loc;+ size_t size = cfg_.size * _gperf[i];+ size = utils::padto(size, cfg_.step);+ size = utils::remainsize(loc, cfg_.size, size);+ if (i == n_group - 1) {+ size = cfg_.size - loc;+ }+ group_range_[i].size = size;+ loc += size;+ }+ }++ for (size_t i = 0; i < n_group_; i++) {+ size_t loc = group_range_[i].loc;+ size_t size = group_range_[i].size;+ size_t off = group_off_[i];+ int nt = _gthreads[i];+ size_t avg_size = utils::padto(size / nt, cfg_.step);+ for (size_t j = 0; j < nt; j++) {+ auto rsize = utils::remainsize(loc, group_range_[i].loc + size, avg_size);+ if (j == nt - 1) {+ rsize = group_range_[i].loc + size - loc;+ }+ threads_range_[off + j] = ThreadRange{loc, rsize};+ loc += rsize;+ }+ }+ }++ inline ThreadRange get_range(const GroupThread& gt) {+ return threads_range_[group_off_[gt.group_idx_] + gt.thread_idx_];+ }++ Config cfg_;+ static int constexpr MAX_GROUP = 8;+ static int constexpr MAX_THREADS = 256;+ ThreadRange group_range_[MAX_GROUP], threads_range_[MAX_THREADS];+ int group_off_[MAX_GROUP]{0};+ int n_group_ = 0;+ ThreadRange g_range_{0};+};+#endif+} // namespace gemm++template <class Parallel_T, class Launch_T>+void GemmRun(const typename Launch_T::Param& args, parallel::IThreading* th) {+ gemm::SchedulerDispatcher<Parallel_T> para(th, args.problem);+ static bool flag = false;+ if (flag) {+ printf("%s\n", __FUNCTION__);+ para.print();+ flag = false;+ }+ th->parallel_for([&](int tidx) {+ typename Parallel_T::ThreadProblem thdp{tidx};+ para.getIndex(thdp);+ if (thdp.valid) {+ Launch_T::run(args, thdp);+ }+ });+}++template <class Parallel_T, class Launch_T>+void GemmRunWithA(const typename Launch_T::Param& args, parallel::IThreading* th) {+ gemm::SchedulerDispatcher<Parallel_T> para(th, args.problem);+ using AParall = typename Launch_T::PrologueA::Parallel;+ AParall apara = Launch_T::PrologueA::createParallel(th->num_threads(), args.problem);+ static bool flag = false;+ if (flag) {+ printf("%s\n", __FUNCTION__);+ para.print();+ flag = false;+ }+ th->parallel_for([&](int tidx) {+ typename AParall::ThreadProblem thdpA{tidx};+ apara.getIndex(thdpA);+ if (thdpA.valid) {+ Launch_T::PrologueA::run(args.paramA, thdpA);+ }+ th->sync(tidx);+ typename Parallel_T::ThreadProblem thdp{tidx};+ para.getIndex(thdp);+ if (thdp.valid) {+ Launch_T::run(args, thdp);+ }+ });+}++} // namespace parallel+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_prologue_a.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_prologue_a.hnew file mode 100644index 000000000..8c113f552--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_prologue_a.h@@ -0,0 +1,431 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include <immintrin.h>++#include <cassert>+#include <type_traits>++#include "bestla.h"+#include "bestla_device.h"+#include "bestla_gemm.h"+#include "bestla_parallel.h"+#include "bestla_storage.h"+#include "bestla_utils.h"+#include "kernel_wrapper.h"++namespace bestla {+namespace prologue_a {+namespace gemm {++template <typename AType>+struct ParamActivationBase {+ const AType* A;+ int lda;+};+template <class _GemmCore_T>+class ActivationBase {+ public:+ using AType = typename _GemmCore_T::AType;+ using SRCType = AType;+ using Param = ParamActivationBase<AType>;+ TLACALL BTLA_CODE getActivation(AType** dstptr, int* dststep, const Param& _param, int m_size, int k_size,+ int m_offset, int k_offset, void* tmpcache, size_t cachesize) {+ auto aptr = const_cast<AType*>(_param.A) + m_offset * _param.lda + k_offset;+ auto alignedptr = utils::cpu_pointer_align(aptr);+ bool use_rawptr = k_size % _GemmCore_T::KTILE == 0 && m_size >= _GemmCore_T::MTILE;+ use_rawptr = use_rawptr && (alignedptr == aptr);+ if (use_rawptr) {+ *dstptr = aptr;+ *dststep = _param.lda;+ return BTLA_CODE::Success;+ } else {+ auto k_pad = utils::padto(k_size, _GemmCore_T::KTILE);+ *dststep = k_pad;+ return kernel::wrapper::Memcpy2D::forward<BTLA_ISA::NoSIMD, AType, AType>(aptr, *dstptr, m_size, k_size,+ _param.lda, k_pad);+ }+ }+};++template <class _GemmCore_T, typename SRC_T>+class ActivationConverter : public ActivationBase<_GemmCore_T> {+ public:+ using AType = typename _GemmCore_T::AType;+ using SRCType = SRC_T;+ using Param = ParamActivationBase<SRC_T>;+ TLACALL BTLA_CODE getActivation(AType** dstptr, int* dststep, const Param& _param, int m_size, int k_size,+ int m_offset, int k_offset, void* tmpcache, size_t cachesize) {+ auto aptr = const_cast<SRC_T*>(_param.A);+ auto k_pad = utils::padto(k_size, _GemmCore_T::KTILE);+ *dststep = k_pad;+ if constexpr (std::is_same_v<AType, utils::bf16> && std::is_same_v<SRC_T, float>) {+ return kernel::wrapper::Memcpy2DFp32CvtBf16::forward<ISA_T>(aptr + m_offset * _param.lda + k_offset, *dstptr,+ m_size, k_size, _param.lda * sizeof(SRC_T),+ k_pad * sizeof(AType), true);+ } else if constexpr (std::is_same_v<AType, utils::fp16> && std::is_same_v<SRC_T, float>) {+ return kernel::wrapper::Memcpy2DFp32CvtFp16::forward<ISA_T>(aptr + m_offset * _param.lda + k_offset, *dstptr,+ m_size, k_size, _param.lda * sizeof(SRC_T),+ k_pad * sizeof(AType), true);+ } else if constexpr (std::is_same_v<AType, float> && std::is_same_v<SRC_T, utils::bf16>) {+ return kernel::wrapper::Memcpy2DBf16CvtFp32::forward<ISA_T>(aptr + m_offset * _param.lda + k_offset, *dstptr,+ m_size, k_size, _param.lda * sizeof(SRC_T),+ k_pad * sizeof(AType), true);+ } else if constexpr (std::is_same_v<AType, SRC_T>) {+ return ActivationBase<_GemmCore_T>::template getActivation<ISA_T>(+ dstptr, dststep, {_param.A, _param.lda}, m_size, k_size, m_offset, k_offset, tmpcache, cachesize);+ } else {+ assert(0);+ }+ return BTLA_CODE::NotSupport;+ }+};++template <class _GemmCore_T>+using ActivationConverterFp32 = ActivationConverter<_GemmCore_T, float>;+template <class _GemmCore_T>+using ActivationConverterBf16 = ActivationConverter<_GemmCore_T, utils::bf16>;++template <typename AType>+struct ParamActivationKBlockQuantize : ParamActivationBase<AType> {+ storage::gemm::StorageQuantActivation* quan;+};+template <class _GemmCore_T, typename SRC_T>+class ActivationKBlockQuantize {+ public:+ using AType = typename _GemmCore_T::AType;+ using SType = float;+ using QParam = storage::gemm::StorageQuantActivation;+ using SRCType = SRC_T;+ using Param = ParamActivationKBlockQuantize<SRC_T>;+ using Parallel = parallel::Scheduler2D;+ using ThreadProblem = parallel::ThreadProblem2D;++ AUTOCALL Parallel createParallel(int nthreads, const utils::GemmProblem& prbm) {+ return Parallel({+ nthreads, prbm.dims[1], // m+ prbm.dims[3], // k+ 1,+ prbm.dims[4] // kblock+ });+ }++ AUTOCALL QParam createStorage(int m, int k, int kblock, bool hasreduce) {+ QParam tmp;+ int kpad = utils::padto(k, _GemmCore_T::KTILE);+ int mpad = utils::padto(m, _GemmCore_T::MTILE);+ tmp.resize(mpad, kpad, m, k, kblock == -1 ? kpad : kblock, BTLA_DTYPE::U8, BTLA_DTYPE::F32, BTLA_DTYPE::U8,+ std::is_same_v<AType, uint8_t>, hasreduce);+ return tmp;+ }++ AUTOCALL void run(const Param& _param, ThreadProblem& thdp) {+ auto quan = _param.quan;+ if (thdp.valid) {+ // min max+ auto srcptr = const_cast<SRC_T*>(_param.A) + thdp.loc[0] * _param.lda + thdp.loc[1];+ auto thdqptr = quan->template APtr<AType>() + thdp.loc[0] * quan->mKPad + thdp.loc[1];+ auto blk_offset = thdp.loc[0] * quan->CStep() + thdp.loc[1] / quan->mBlockSize;+ auto thdsptr = quan->template SPtr<float>() + blk_offset;+ auto thdzptr = quan->template ZPtr<AType>() + blk_offset;+ auto thdrptr = quan->template RPtr<float>() == nullptr ? nullptr : quan->template RPtr<float>() + blk_offset;+ if constexpr (std::is_same_v<AType, uint8_t>) {+ kernel::wrapper::QuantizeU8ColBlock<SRC_T>::forward_auto(thdp.size[0], thdp.size[1], srcptr, _param.lda,+ thdqptr, quan->mKPad, thdsptr, quan->CStep(), thdzptr,+ quan->mBlockSize, thdrptr);+ }+ if constexpr (std::is_same_v<AType, int8_t>) {+ kernel::wrapper::QuantizeS8ColBlock<SRC_T>::forward_auto(thdp.size[0], thdp.size[1], srcptr, _param.lda,+ thdqptr, quan->mKPad, thdsptr, quan->CStep(),+ quan->mBlockSize, thdrptr);+ }+ }+ }++ AUTOCALL BTLA_CODE quantize(const Param& _param, int m, int k, parallel::IThreading* threading) {+ auto paral = Parallel({threading->num_threads(), m, k, 1, _param.quan->mBlockSize});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ paral.getIndex(thdp);+ if (thdp.valid) run(_param, thdp);+ });+ return BTLA_CODE::Success;+ }++ public: // Runtime get by launcher+ TLACALL BTLA_CODE getActivation(AType** dstptr, int* dststep, const Param& _param, int m_size, int k_size,+ int m_offset, int k_offset, void* tmpcache, size_t cachesize) {+ (void)m_size;+ (void)k_size;+ auto quan = _param.quan;+ auto aptr = quan->template APtr<AType>();+ *dstptr = aptr + m_offset * quan->mKPad + k_offset;+ *dststep = quan->mKPad;+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getZp(AType** dstptr, int* dststep, const Param& _param, int m_size, int k_size, int m_offset,+ int k_offset, void* tmpcache, size_t cachesize) {+ auto quan = _param.quan;+ auto aptr = quan->template ZPtr<AType>();+ if (aptr == nullptr) { // optional+ *dstptr = nullptr;+ return BTLA_CODE::Success;+ }+ int kele = utils::updiv(k_size, quan->mBlockSize);+ *dststep = kele;+ kernel::ref::memcpy2d(aptr + m_offset * quan->CStep() + k_offset / quan->mBlockSize, *dstptr, m_size,+ kele * sizeof(AType), quan->CStep() * sizeof(AType), kele * sizeof(AType));+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getScale(float** dstptr, int* dststep, const Param& _param, int m_size, int k_size, int m_offset,+ int k_offset, void* tmpcache, size_t cachesize) {+ auto quan = _param.quan;+ auto aptr = quan->template SPtr<float>();+ int kele = utils::updiv(k_size, quan->mBlockSize);+ *dststep = kele;+ kernel::ref::memcpy2d(aptr + m_offset * quan->CStep() + k_offset / quan->mBlockSize, *dstptr, m_size,+ kele * sizeof(float), quan->CStep() * sizeof(float), kele * sizeof(float));+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getReduce(float** dstptr, int* dststep, const Param& _param, int m_size, int k_size, int m_offset,+ int k_offset, void* tmpcache, size_t cachesize) {+ auto quan = _param.quan;+ auto aptr = quan->template RPtr<float>();+ int kele = utils::updiv(k_size, quan->mBlockSize);+ *dststep = kele;+ kernel::ref::memcpy2d(aptr + m_offset * quan->CStep() + k_offset / quan->mBlockSize, *dstptr, m_size,+ kele * sizeof(float), quan->CStep() * sizeof(float), kele * sizeof(float));+ return BTLA_CODE::Success;+ }+};++template <class _GemmCore_T>+using ActivationF32KBlockQuantize = ActivationKBlockQuantize<_GemmCore_T, float>;+template <class _GemmCore_T>+using ActivationBf16KBlockQuantize = ActivationKBlockQuantize<_GemmCore_T, utils::bf16>;++template <typename AType>+struct ParamActivationKBlockBase : ParamActivationBase<AType> {+ storage::gemm::StorageReduce* reduce;+};+template <class _GemmCore_T, typename SRC_T>+class ActivationKBlockBase : public ActivationConverter<_GemmCore_T, SRC_T> {+ public:+ using AType = typename _GemmCore_T::AType;+ using SType = storage::gemm::StorageReduce;+ using SRCType = SRC_T;+ using Param = ParamActivationKBlockBase<SRC_T>;+ using Parallel = parallel::Scheduler2D;+ using ThreadProblem = parallel::ThreadProblem2D;++ AUTOCALL Parallel createParallel(int nthreads, const utils::GemmProblem& prbm) {+ return Parallel({+ nthreads, prbm.dims[1], // m+ prbm.dims[3], // k+ 1,+ prbm.dims[4] // kblock+ });+ }+ AUTOCALL SType createStorage(int m, int k, int kblock) {+ SType tmp;+ tmp.resize(m, k, kblock == -1 ? k : kblock, BTLA_DTYPE::F32);+ return tmp;+ }++ AUTOCALL void run(const Param& _param, ThreadProblem& thdp) {+ auto stor = _param.reduce;+ if (thdp.valid) {+ // min max+ auto srcptr = const_cast<SRC_T*>(_param.A) + thdp.loc[0] * _param.lda + thdp.loc[1];+ auto blk_offset = thdp.loc[0] * stor->lda + thdp.loc[1] / stor->kblock;+ auto thdrptr = stor->template RPtr<float>() + blk_offset;+ auto ret = kernel::wrapper::ColBlockReduceSum<SRC_T>::forward_auto(srcptr, _param.lda, thdp.size[0], thdp.size[1],+ stor->kblock, thdrptr, stor->lda);+ assert(ret == BTLA_CODE::Success);+ }+ }++ AUTOCALL BTLA_CODE reduce(const Param& _param, int m, int k, int kblock, parallel::IThreading* threading) {+ auto paral = Parallel({threading->num_threads(), m, k, 1, kblock});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ paral.getIndex(thdp);+ if (thdp.valid) run(_param, thdp);+ });+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getActivation(AType** dstptr, int* dststep, const Param& _param, int m_size, int k_size,+ int m_offset, int k_offset, void* tmpcache, size_t cachesize) {+ return ActivationConverter<_GemmCore_T, SRC_T>::template getActivation<ISA_T>(+ dstptr, dststep, {_param.A, _param.lda}, m_size, k_size, m_offset, k_offset, tmpcache, cachesize);+ }++ TLACALL BTLA_CODE getReduce(float** dstptr, int* dststep, const Param& _param, int m_size, int k_size, int m_offset,+ int k_offset, void* tmpcache, size_t cachesize) {+ auto reduce = _param.reduce;+ auto aptr = reduce->template RPtr<float>();+ int kele = utils::updiv(k_size, reduce->kblock);+ *dststep = kele;+ kernel::ref::memcpy2d(aptr + m_offset * reduce->lda + k_offset / reduce->kblock, *dstptr, m_size,+ kele * sizeof(float), reduce->lda * sizeof(float), kele * sizeof(float));+ return BTLA_CODE::Success;+ }+};++template <class _GemmCore_T>+using ActivationKBlockBaseF32 = ActivationKBlockBase<_GemmCore_T, float>;++template <typename AType>+struct ParamShuffleActivationKBlockBase : ParamActivationKBlockBase<AType> {+ int* indices = nullptr;+ storage::gemm::StorageReorderActivation* reordered = nullptr;+};+template <class _GemmCore_T, typename SRC_T>+class ShuffleActivationKBlockBase : public ActivationKBlockBase<_GemmCore_T, SRC_T> {+ public:+ using AType = typename _GemmCore_T::AType;+ using RedType = storage::gemm::StorageReduce;+ using RAType = storage::gemm::StorageReorderActivation;+ using SRCType = SRC_T;+ using Param = ParamShuffleActivationKBlockBase<SRC_T>;+ using Parallel = parallel::Scheduler2D;+ using ThreadProblem = parallel::ThreadProblem2D;+ static RAType createReorderStorage(int m, int k, int kblock) {+ RAType tmp(_GemmCore_T::ID);+ int kpad = utils::padto(k, _GemmCore_T::KTILE);+ int mpad = utils::padto(m, _GemmCore_T::MTILE);+ tmp.resize(mpad, kpad, m, k, kblock == -1 ? kpad : kblock, utils::bestla_dtype<SRC_T>);+ return tmp;+ }++ static RedType createReduceStorage(int m, int k, int kblock) {+ RedType tmp;+ tmp.resize(m, k, kblock == -1 ? k : kblock, BTLA_DTYPE::F32);+ return tmp;+ }++ AUTOCALL void run(const Param& _param, ThreadProblem& thdp) {+ auto stor = _param.reduce;+ auto reordered = _param.reordered;+ if (thdp.valid) {+ auto srcptr = const_cast<SRC_T*>(_param.A) + thdp.loc[0] * _param.lda + thdp.loc[1];+ if (reordered && _param.indices) {+ auto rptr = reordered->template APtr<SRC_T>() + thdp.loc[0] * reordered->mKPad + thdp.loc[1];+ auto ret =+ kernel::ref::shuffle_activation(const_cast<SRC_T*>(_param.A), rptr, thdp.size[0], thdp.size[1], thdp.loc[0],+ thdp.loc[1], _param.indices, _param.lda, reordered->mKPad);+ srcptr = rptr;+ }+ if (stor) {+ // min max+ auto blk_offset = thdp.loc[0] * stor->lda + thdp.loc[1] / stor->kblock;+ auto thdrptr = stor->template RPtr<float>() + blk_offset;+ auto ret = kernel::wrapper::ColBlockReduceSum<SRC_T>::forward_auto(+ srcptr, _param.lda, thdp.size[0], thdp.size[1], stor->kblock, thdrptr, stor->lda);+ assert(ret == BTLA_CODE::Success);+ }+ }+ }++ AUTOCALL BTLA_CODE preprocess(const Param& _param, int m, int k, int kblock, parallel::IThreading* threading) {+ auto paral = Parallel({threading->num_threads(), m, k, 1, kblock});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ paral.getIndex(thdp);+ run(_param, thdp);+ });+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getActivation(AType** dstptr, int* dststep, const Param& _param, int m_size, int k_size,+ int m_offset, int k_offset, void* tmpcache, size_t cachesize) {+ if (_param.indices == nullptr) {+ return ActivationConverter<_GemmCore_T, SRC_T>::template getActivation<ISA_T>(+ dstptr, dststep, {_param.A, _param.lda}, m_size, k_size, m_offset, k_offset, tmpcache, cachesize);+ } else {+ return ActivationConverter<_GemmCore_T, SRC_T>::template getActivation<ISA_T>(+ dstptr, dststep, {_param.reordered->template APtr<SRC_T>(), _param.reordered->mKPad}, m_size, k_size,+ m_offset, k_offset, tmpcache, cachesize);+ }+ }+};++template <class _GemmCore_T>+using ShuffleActivationKBlockBaseF32 = ShuffleActivationKBlockBase<_GemmCore_T, float>;+template <class _GemmCore_T>+using ShuffleActivationKBlockBaseBf16 = ShuffleActivationKBlockBase<_GemmCore_T, utils::bf16>;++template <typename AType>+struct ParamShuffleActivationKBlockQuantize : ParamActivationKBlockQuantize<AType> {+ int* indices = nullptr;+ storage::gemm::StorageReorderActivation* reordered = nullptr;+};+template <class _GemmCore_T, typename SRC_T>+class ShuffleActivationKBlockQuantize : public ActivationKBlockQuantize<_GemmCore_T, SRC_T> {+ public:+ using AType = typename _GemmCore_T::AType;+ using SType = float;+ using QParam = storage::gemm::StorageQuantActivation;+ using RAType = storage::gemm::StorageReorderActivation;+ using SRCType = SRC_T;+ using Param = ParamShuffleActivationKBlockQuantize<SRC_T>;+ using Parallel = parallel::Scheduler2D;+ using ThreadProblem = parallel::ThreadProblem2D;++ AUTOCALL QParam createQuantStorage(int m, int k, int kblock, bool hasreduce) {+ QParam tmp;+ int kpad = utils::padto(k, _GemmCore_T::KTILE);+ int mpad = utils::padto(m, _GemmCore_T::MTILE);+ tmp.resize(mpad, kpad, m, k, kblock == -1 ? kpad : kblock, BTLA_DTYPE::U8, BTLA_DTYPE::F32, BTLA_DTYPE::U8,+ std::is_same_v<AType, uint8_t>, hasreduce);+ return tmp;+ }++ AUTOCALL RAType createReorderStorage(int m, int k, int kblock) {+ RAType tmp(_GemmCore_T::ID);+ int kpad = utils::padto(k, _GemmCore_T::KTILE);+ int mpad = utils::padto(m, _GemmCore_T::MTILE);+ tmp.resize(mpad, kpad, m, k, kblock == -1 ? kpad : kblock, utils::bestla_dtype<SRC_T>);+ return tmp;+ }++ AUTOCALL BTLA_CODE quantize(const Param& _param, int m, int k, parallel::IThreading* threading) {+ auto srcptr = const_cast<SRC_T*>(_param.A);+ if (_param.indices) {+ auto shuffle_src = _param.reordered->template APtr<SRC_T>();+ threading->parallel_for([&](int tidx) {+ auto enable_thr = threading->num_threads();+ auto align_m = m / enable_thr;+ auto process_m = (tidx + 1) == enable_thr ? (m - tidx * align_m) : align_m;+ kernel::ref::shuffle_activation(const_cast<SRC_T*>(_param.A), shuffle_src + tidx * align_m * k, process_m, k,+ tidx * align_m, 0, _param.indices, k, k);+ });+ srcptr = shuffle_src;+ }+ ActivationKBlockQuantize<_GemmCore_T, SRC_T>::quantize({srcptr, k, _param.quan}, m, k, threading);+ return BTLA_CODE::Success;+ }+};++template <class _GemmCore_T>+using ShuffleActivationKBlockQuantizeF32 = ShuffleActivationKBlockQuantize<_GemmCore_T, float>;+template <class _GemmCore_T>+using ShuffleActivationKBlockQuantizeBf16 = ShuffleActivationKBlockQuantize<_GemmCore_T, utils::bf16>;+} // namespace gemm+} // namespace prologue_a+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_prologue_b.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_prologue_b.hnew file mode 100644index 000000000..228619792--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_prologue_b.h@@ -0,0 +1,2399 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include <cassert>+#include "bestla_utils.h"+#include "bestla_storage.h"+#include "bestla_device.h"+#include "bestla_parallel.h"+#include "kernel_wrapper.h"++namespace bestla {+namespace prologue_b {+namespace gemm {++template <typename WT>+static inline void transposeWeight(const int Row, const int Col, const WT* src, const int ld_src, WT* dst,+ const int ld_dst, parallel::IThreading* threading) {+ bestla::parallel::Scheduler2D _para;+ _para.update({threading->num_threads(), Row, Col, 16, 16});+ threading->parallel_for([&](int tidx) {+ bestla::parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ kernel::wrapper::Transpose2D<WT>::forward_auto(src + thdp.loc[0] * ld_src + thdp.loc[1],+ dst + thdp.loc[0] + thdp.loc[1] * ld_dst, thdp.size[0],+ thdp.size[1], ld_src, ld_dst);+ }+ });+}++static inline void transpose(int Row, int Col, const int8_t* src, BTLA_DTYPE dtype, int ld_src, int8_t* dst, int ld_dst,+ parallel::IThreading* threading) {+ bestla::parallel::Scheduler2D _para;+ _para.update({threading->num_threads(), Row, Col, 16, 16});+ threading->parallel_for([&](int tidx) {+ bestla::parallel::ThreadProblem2D thdp{tidx};+ auto byte = utils::bestla_dtype_bytes(dtype);+ _para.getIndex(thdp);+ if (thdp.valid) {+ if (byte == 1) {+ kernel::wrapper::Transpose2D<int8_t>::forward_auto(src + thdp.loc[0] * ld_src + thdp.loc[1],+ dst + thdp.loc[0] + thdp.loc[1] * ld_dst, thdp.size[0],+ thdp.size[1], ld_src, ld_dst);+ }+ if (byte == 2) {+ kernel::wrapper::Transpose2D<int16_t>::forward_auto((int16_t*)src + thdp.loc[0] * ld_src + thdp.loc[1],+ (int16_t*)dst + thdp.loc[0] + thdp.loc[1] * ld_dst,+ thdp.size[0], thdp.size[1], ld_src, ld_dst);+ }+ if (byte == 4) {+ kernel::wrapper::Transpose2D<int32_t>::forward_auto((int32_t*)src + thdp.loc[0] * ld_src + thdp.loc[1],+ (int32_t*)dst + thdp.loc[0] + thdp.loc[1] * ld_dst,+ thdp.size[0], thdp.size[1], ld_src, ld_dst);+ }+ }+ });+}++template <typename WType>+struct ParamWeightPack {+ const WType* B;+ const int ldb;+ storage::gemm::StoragePackedWeight* packedW;+};++template <class _GemmCore_T>+class WeightPack {+ public:+ using WType = typename _GemmCore_T::BType;+ using StorageType = storage::gemm::StoragePackedWeight;+ using Param = ParamWeightPack<WType>;++ AUTOCALL StorageType createStorage(int n, int k, storage::gemm::Layout _layout = storage::gemm::Layout::NPack) {+ int KPad = k, NPad = n;+ switch (_layout) {+ case bestla::storage::gemm::Layout::NPack: {+ KPad = utils::padto(k, _GemmCore_T::KTILE);+ NPad = utils::padto(n, _GemmCore_T::NTILE);+ } break;+ case bestla::storage::gemm::Layout::T:+ case bestla::storage::gemm::Layout::NT:+ default:+ break;+ }+ StorageType tmp(_GemmCore_T::ID);+ tmp.resize(NPad, KPad, n, k, utils::bestla_dtype<WType>, _layout);+ return tmp;+ }++ AUTOCALL void packWeightTranspose(const int N, const int K, const Param& _param, parallel::IThreading* threading) {+ auto B_NT = utils::amalloc<WType>(static_cast<size_t>(N) * K);+ transposeWeight<WType>(N, K, _param.B, _param.ldb, B_NT, N, threading);+ packWeight(N, K, {B_NT, N, _param.packedW}, threading);+ utils::afree(B_NT);+ }++ // from KxN int8 symmetric weight to packed N//NtilexKPadxNTile int4 weight+ AUTOCALL void packWeight(const int N, const int K, const Param& _param, parallel::IThreading* threading) {+ parallel::Scheduler2D _para({threading->num_threads(), K, N, _GemmCore_T::KTILE, _GemmCore_T::NTILE});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ run(_param, thdp);+ }+ });+ }++ AUTOCALL void unpackWeight(const int N, const int K, StorageType* stor, WType* B, const int ldb,+ parallel::IThreading* threading) {+ parallel::Scheduler2D _para({threading->num_threads(), K, N, _GemmCore_T::KTILE, _GemmCore_T::NTILE});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ auto rowpad = utils::padto(thdp.size[0], _GemmCore_T::KTILE);+ auto colpad = utils::padto(thdp.size[1], _GemmCore_T::NTILE);+ auto dequant = utils::amalloc<WType>((size_t)rowpad * colpad);+ auto dstptr = dequant;+ int dststep = 0;+ size_t constexpr CacheSize = size_t(100) << 10;+ int8_t tmpcache[CacheSize];+ GetCPUDevice();+ if (_cd->AVX512F()) {+ getWeight<BTLA_ISA::AVX512F>(&dstptr, &dststep, rowpad, colpad, thdp.loc[0], thdp.loc[1], {nullptr, 0, stor},+ tmpcache, CacheSize);+ } else if (_cd->AVX2()) {+ getWeight<BTLA_ISA::AVX2>(&dstptr, &dststep, rowpad, colpad, thdp.loc[0], thdp.loc[1], {nullptr, 0, stor},+ tmpcache, CacheSize);+ } else {+ getWeight<BTLA_ISA::NoSIMD>(&dstptr, &dststep, rowpad, colpad, thdp.loc[0], thdp.loc[1], {nullptr, 0, stor},+ tmpcache, CacheSize);+ }+ kernel::wrapper::RevertPaddingInterleaveMN<_GemmCore_T::NTILE, _GemmCore_T::PACK_ROW, WType>::forward_auto(+ dstptr, B + thdp.loc[0] * ldb + thdp.loc[1], thdp.size[0], thdp.size[1], rowpad, colpad, dststep, ldb);+ utils::afree(dequant);+ }+ });+ }++ AUTOCALL void run(const Param& _param, parallel::ThreadProblem2D& thdp) {+ auto packedw = _param.packedW;+ auto rowpadded = utils::padto(thdp.size[0], _GemmCore_T::KTILE);+ auto colpadded = utils::padto(thdp.size[1], _GemmCore_T::NTILE);+ const auto src = _param.B + thdp.loc[0] * _param.ldb + thdp.loc[1];+ const auto dst = packedw->template WPtr<WType>() + thdp.loc[0] * _GemmCore_T::NTILE + thdp.loc[1] * packedw->mKPad;+ using PaddingInterleaveMNWType =+ kernel::wrapper::PaddingInterleaveMN<_GemmCore_T::NTILE, _GemmCore_T::PACK_ROW, WType>;+ auto ret = PaddingInterleaveMNWType::forward_auto( //+ src, dst, thdp.size[0], thdp.size[1], rowpadded, colpadded, _param.ldb, packedw->mKPad);+ _param.packedW->mLayout = storage::gemm::Layout::NPack;+ assert(ret == BTLA_CODE::Success);+ (void)ret;+ }++ TLACALL BTLA_CODE getWeight(WType** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param param, void* tmpcache, size_t cachesize) {+ auto wptr = param.packedW;+ auto KPad = wptr->mKPad;+ auto bptr = wptr->template WPtr<WType>() + n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ kernel::wrapper::Memcpy2D::template forward<BTLA_ISA::NoSIMD, WType, WType>(+ bptr, *dstptr, n_size / _GemmCore_T::NTILE, _GemmCore_T::NTILE * k_size, _GemmCore_T::NTILE * KPad,+ _GemmCore_T::NTILE * k_size);+ *dststep = k_size;+ return BTLA_CODE::Success;+ }+};++template <typename _Wt, class _St>+struct ParamWeightNPack {+ const _Wt* B_;+ const int ldb_;+ const _St* stor_;+};++template <class _GemmCore_T>+class WeightNPack {+ public:+ using WType = typename _GemmCore_T::BType;+ using StorageType = storage::gemm::StorageWeight;+ using Param = ParamWeightNPack<WType, StorageType>;++ AUTOCALL StorageType create_storage(int n, int k, storage::gemm::Layout _layout = storage::gemm::Layout::NPack) {+ int KPad = k, NPad = n;+ switch (_layout) {+ case bestla::storage::gemm::Layout::NPack: {+ KPad = utils::padto(k, _GemmCore_T::KTILE);+ NPad = utils::padto(n, _GemmCore_T::NTILE);+ } break;+ case bestla::storage::gemm::Layout::T:+ case bestla::storage::gemm::Layout::NT:+ default:+ break;+ }+ StorageType tmp(_GemmCore_T::ID);+ tmp.resize(NPad, KPad, n, k, utils::bestla_dtype<WType>, _layout);+ return tmp;+ }++ AUTOCALL void unpack_weight(const int N, const int K, StorageType* stor, WType* B, const int ldb,+ parallel::IThreading* threading) {+ parallel::Scheduler2D _para({threading->num_threads(), K, N, _GemmCore_T::KTILE, _GemmCore_T::NTILE});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ auto rowpad = utils::padto(thdp.size[0], _GemmCore_T::KTILE);+ auto colpad = utils::padto(thdp.size[1], _GemmCore_T::NTILE);+ auto dequant = utils::amalloc<WType>((size_t)rowpad * colpad);+ auto dstptr = dequant;+ int dststep = 0;+ size_t constexpr CacheSize = size_t(100) << 10;+ int8_t tmpcache[CacheSize];+ GetCPUDevice();+ if (_cd->AVX512F()) {+ get_weight<BTLA_ISA::AVX512F>(&dstptr, &dststep, rowpad, colpad, thdp.loc[0], thdp.loc[1], {nullptr, 0, stor},+ tmpcache, CacheSize);+ } else if (_cd->AVX2()) {+ get_weight<BTLA_ISA::AVX2>(&dstptr, &dststep, rowpad, colpad, thdp.loc[0], thdp.loc[1], {nullptr, 0, stor},+ tmpcache, CacheSize);+ } else {+ get_weight<BTLA_ISA::NoSIMD>(&dstptr, &dststep, rowpad, colpad, thdp.loc[0], thdp.loc[1], {nullptr, 0, stor},+ tmpcache, CacheSize);+ }+ kernel::wrapper::RevertPaddingInterleaveMN<_GemmCore_T::NTILE, _GemmCore_T::PACK_ROW, WType>::forward_auto(+ dstptr, B + thdp.loc[0] * ldb + thdp.loc[1], thdp.size[0], thdp.size[1], rowpad, colpad, dststep, ldb);+ utils::afree(dequant);+ }+ });+ }++ using PaddingInterleaveMNWType =+ kernel::wrapper::PaddingInterleaveMN<_GemmCore_T::NTILE, _GemmCore_T::PACK_ROW, WType>;++ AUTOCALL void convert(const storage::gemm::StorageWeight& src, const storage::gemm::StorageWeight& dst,+ parallel::IThreading* threading) {+ assert(src.info_.dtype_ == utils::bestla_dtype<WType>); // No type conversion+ assert(dst.info_.dtype_ == utils::bestla_dtype<WType>); // No type conversion+ using storage::gemm::Layout;+ if (src.info_.layout_ == storage::gemm::Layout::NT) {+ convert_NT(src, dst, threading);+ }+ if (src.info_.layout_ == storage::gemm::Layout::T) {+ auto tmpptr = utils::amalloc<int8_t>(src.buf_size_);+ storage::gemm::StorageWeight nt_stor;+ nt_stor.set_nt_data(src.info_.n_, src.info_.k_, src.info_.n_, src.info_.dtype_, tmpptr);+ convert_T(src, nt_stor, threading);+ convert_NT(nt_stor, dst, threading);+ utils::afree(tmpptr);+ }+ if (src.info_.layout_ == storage::gemm::Layout::NPack) {+ convert_NPack(src, dst, threading);+ }+ }++ TLACALL BTLA_CODE get_weight(WType** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param param, void* tmpcache, size_t cachesize) {+ auto wptr = param.stor_;+ auto KPad = wptr->info_.kpad_;+ auto bptr = (WType*)wptr->ptr() + n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ auto bptr_align = utils::cpu_pointer_align<WType>(bptr);+ if (bptr_align == bptr) {+ *dstptr = bptr;+ *dststep = KPad;+ } else {+ kernel::wrapper::Memcpy2D::template forward<BTLA_ISA::NoSIMD, WType, WType>(+ bptr, *dstptr, n_size / _GemmCore_T::NTILE, _GemmCore_T::NTILE * k_size, _GemmCore_T::NTILE * KPad,+ _GemmCore_T::NTILE * k_size);+ *dststep = k_size;+ }+ return BTLA_CODE::Success;+ }++ AUTOCALL BTLA_CODE get_weight_auto(WType** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param param, void* tmpcache, size_t cachesize) {+ GetCPUDevice();+ if (_cd->AVX512F()) {+ return get_weight<BTLA_ISA::AVX512F>(dstptr, dststep, k_size, n_size, k_offset, n_offset, param, tmpcache,+ cachesize);+ } else if (_cd->AVX2()) {+ return get_weight<BTLA_ISA::AVX2>(dstptr, dststep, k_size, n_size, k_offset, n_offset, param, tmpcache,+ cachesize);+ } else {+ return get_weight<BTLA_ISA::NoSIMD>(dstptr, dststep, k_size, n_size, k_offset, n_offset, param, tmpcache,+ cachesize);+ }+ return BTLA_CODE::NotSupport;+ }+ AUTOCALL void convert_NT(const storage::gemm::StorageWeight& src, const storage::gemm::StorageWeight& dst,+ parallel::IThreading* threading) {+ assert(src.info_.dtype_ == utils::bestla_dtype<WType>); // No type conversion+ assert(dst.info_.dtype_ == utils::bestla_dtype<WType>); // No type conversion+ assert(src.info_.layout_ == storage::gemm::Layout::NT);+ using storage::gemm::Layout;+ if (dst.info_.layout_ == storage::gemm::Layout::NPack) {+ parallel::Scheduler2D _para(+ {threading->num_threads(), src.info_.k_, src.info_.n_, _GemmCore_T::KTILE, _GemmCore_T::NTILE});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ auto sptr = (const WType*)src.ptr() + (thdp.loc[0] * src.info_.npad_ + thdp.loc[1]);+ auto dptr = (WType*)dst.ptr() + (thdp.loc[0] * _GemmCore_T::NTILE + thdp.loc[1] * dst.info_.kpad_);+ auto rowpadded = utils::padto(thdp.size[0], _GemmCore_T::KTILE);+ auto colpadded = utils::padto(thdp.size[1], _GemmCore_T::NTILE);+ auto ret = PaddingInterleaveMNWType::forward_auto( //+ sptr, dptr, thdp.size[0], thdp.size[1], rowpadded, colpadded, src.info_.npad_, dst.info_.kpad_);+ assert(ret == BTLA_CODE::Success);+ }+ });+ }++ if (dst.info_.layout_ == storage::gemm::Layout::T) {+ transposeWeight<WType>(src.info_.k_, src.info_.n_, (const WType*)src.buf_.buf_ptr_, src.info_.npad_,+ (WType*)dst.buf_.buf_ptr_, dst.info_.kpad_, threading);+ }+ }++ AUTOCALL void convert_NPack(const storage::gemm::StorageWeight& src, const storage::gemm::StorageWeight& dst,+ parallel::IThreading* threading) {+ using storage::gemm::Layout;+ assert(src.info_.dtype_ == utils::bestla_dtype<WType>); // No type conversion+ assert(dst.info_.dtype_ == utils::bestla_dtype<WType>); // No type conversion+ assert(src.info_.layout_ == storage::gemm::Layout::NPack);+ if (dst.info_.layout_ == storage::gemm::Layout::NPack) {+ if (dst.info_.core_id_ == src.info_.core_id_) {+ std::memcpy(dst.buf_.buf_ptr_, src.buf_.buf_ptr_, src.buf_size_);+ return;+ }+ // reorder for different Cores+ }+ if (dst.info_.layout_ == Layout::NT || dst.info_.layout_ == Layout::T) {+ auto ntptr = (WType*)dst.buf_.buf_ptr_;+ auto nt_ld = dst.info_.npad_;+ if (dst.info_.layout_ == Layout::T) {+ ntptr = (WType*)utils::amalloc<int8_t>(dst.buf_size_);+ }+ parallel::Scheduler2D _para(+ {threading->num_threads(), src.info_.k_, src.info_.n_, _GemmCore_T::KTILE, _GemmCore_T::NTILE});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ auto rowpad = utils::padto(thdp.size[0], _GemmCore_T::KTILE);+ auto colpad = utils::padto(thdp.size[1], _GemmCore_T::NTILE);+ auto dequant = utils::amalloc<WType>((size_t)rowpad * colpad);+ auto dstptr = dequant;+ int dststep = 0;+ size_t constexpr CacheSize = size_t(100) << 10;+ int8_t tmpcache[CacheSize];+ get_weight_auto(&dstptr, &dststep, rowpad, colpad, thdp.loc[0], thdp.loc[1], {nullptr, 0, &src}, tmpcache,+ CacheSize);+ kernel::wrapper::RevertPaddingInterleaveMN<_GemmCore_T::NTILE, _GemmCore_T::PACK_ROW, WType>::forward_auto(+ dstptr, ntptr + thdp.loc[0] * nt_ld + thdp.loc[1], thdp.size[0], thdp.size[1], rowpad, colpad, dststep,+ nt_ld);+ utils::afree(dequant);+ }+ });+ if (dst.info_.layout_ == Layout::T) {+ storage::gemm::StorageWeight nt_stor;+ nt_stor.set_nt_data(src.info_.n_, src.info_.k_, src.info_.n_, src.info_.dtype_, ntptr);+ convert_NT(nt_stor, dst, threading);+ utils::afree(ntptr);+ }+ }+ }++ AUTOCALL void convert_T(const storage::gemm::StorageWeight& src, const storage::gemm::StorageWeight& dst,+ parallel::IThreading* threading) {+ assert(src.info_.dtype_ == utils::bestla_dtype<WType>); // No type conversion+ assert(dst.info_.dtype_ == utils::bestla_dtype<WType>); // No type conversion+ assert(src.info_.layout_ == storage::gemm::Layout::T);+ using storage::gemm::Layout;+ if (dst.info_.layout_ == storage::gemm::Layout::NT) {+ transposeWeight<WType>(src.info_.n_, src.info_.k_, (const WType*)src.buf_.buf_ptr_, src.info_.kpad_,+ (WType*)dst.buf_.buf_ptr_, dst.info_.npad_, threading);+ }+ }+};++struct ParamWeightKBlockNInteger {+ storage::gemm::StorageWeightKBlockNInteger* packedW;+};++template <class _GemmCore_T>+class WeightKBlockNInteger {+ public:+ using StorageWeight = storage::gemm::StorageWeightKBlockNInteger;+ using BType = typename _GemmCore_T::BType;+ using Param = ParamWeightKBlockNInteger;++ AUTOCALL StorageWeight createStorage(int n, int k, int blocksize, BTLA_DTYPE qtype, BTLA_DTYPE scat, bool is_asym) {+ int KPad = utils::padto(k, _GemmCore_T::KTILE);+ int NPad = utils::padto(n, _GemmCore_T::NTILE);+ StorageWeight tmp(_GemmCore_T::ID);+ tmp.resize(NPad, KPad, blocksize <= 0 ? KPad : blocksize, n, k, qtype, scat, is_asym);+ return tmp;+ }++ AUTOCALL void convertTransStorage(StorageWeight& srcstor, StorageWeight& dststor, parallel::IThreading* threading) {+ auto s8buf = utils::amalloc<int8_t>((size_t)srcstor.mK * srcstor.mN);+ auto s8transbuf = utils::amalloc<int8_t>((size_t)srcstor.mKPad * srcstor.mNPad);+ unpackWeight(srcstor.mN, srcstor.mK, &srcstor, s8buf, srcstor.mN, threading);+ transposeWeight<int8_t>(srcstor.mK, srcstor.mN, s8buf, srcstor.mN, s8transbuf, srcstor.mKPad, threading);+ compressWeight(srcstor.mKPad, srcstor.mNPad, s8transbuf, srcstor.mKPad, dststor.WPtr<int8_t>(), srcstor.mDType,+ threading);+ if (s8buf) {+ utils::afree(s8buf);+ }+ if (s8transbuf) {+ utils::afree(s8transbuf);+ }+ int nk_scale = utils::updiv(srcstor.mKPad, srcstor.mBlockSize);+ if (srcstor.mCorrection.mScaEleSize == 4) {+ transposeWeight<float>(nk_scale, srcstor.mNPad, srcstor.template SPtr<float>(), srcstor.mNPad,+ dststor.template SPtr<float>(), dststor.CStep(), threading);+ } else if (srcstor.mCorrection.mScaEleSize == 2) {+ transposeWeight<uint16_t>(nk_scale, srcstor.mNPad, srcstor.template SPtr<uint16_t>(), srcstor.mNPad,+ dststor.template SPtr<uint16_t>(), dststor.CStep(), threading);+ }+ }+ AUTOCALL void doubleQuantScale(float* scale, size_t scale_size, int dq_blocksize, BTLA_DTYPE qtype,+ utils::aligned_vector<float>* dq_buf) {+ if (qtype == BTLA_DTYPE::DQ8_BNB) {+ dq_buf->resize(utils::updiv(scale_size, dq_blocksize) + 1); // add 1 for offset.+ kernel::ref::dq8_bnb_double_quant<false>(scale, scale_size, dq_blocksize, dq_buf->data());+ } else {+ assert(0);+ }+ }++ AUTOCALL void setDoubleQuantCorrection(utils::avector<float>* dq_buf, StorageWeight* ptr) {+ if (ptr->SDtype() == BTLA_DTYPE::DQ8_BNB) {+ auto packw_dqbuf_ptr = ptr->DQPtr<float>();+ memcpy(packw_dqbuf_ptr, dq_buf->data(), dq_buf->size() * sizeof(float));+ } else {+ assert(0);+ }+ }++ AUTOCALL void enableShuffle(StorageWeight* stor) { stor->enable_shuffle(); }++ AUTOCALL void setDoubleQuantBlkSize(StorageWeight* stor, BTLA_DTYPE stype, int dq_blksize) {+ stor->mDqBlockSize = dq_blksize;+ auto nk_scale = utils::updiv(stor->mKPad, stor->mBlockSize);+ if (stor->IsAsym() || dq_blksize % 8 != 0) assert(0);+ stor->mCorrection.enable_double_quant(utils::updiv(nk_scale * stor->mN, dq_blksize), stype);+ stor->update_size();+ }++ AUTOCALL void packTransposeWeight(const int N, const int K, const float* B, const int ldb, StorageWeight* stor,+ parallel::IThreading* threading) {+ auto B_NT = utils::amalloc<float>(static_cast<size_t>(N) * K);+ transposeWeight<float>(N, K, B, ldb, B_NT, N, threading);+ packWeight(N, K, B_NT, N, stor, threading);+ utils::afree(B_NT);+ }++ // from packed N//NtilexKPadxNTile int8 weight to KxN f32 weight+ AUTOCALL void unpackTransposeWeight(const int N, const int K, StorageWeight* stor, float* B, const int ldb,+ parallel::IThreading* threading) {+ auto B_NT = utils::amalloc<float>(static_cast<size_t>(N) * K);+ unpackWeight(N, K, stor, B_NT, N, threading);+ transposeWeight<float>(K, N, B_NT, N, B, ldb, threading);+ utils::afree(B_NT);+ }++ // from KxN f32 weight to packed N//NtilexKPadxNTile int8 weight+ AUTOCALL void packWeight(const int N, const int K, const float* B, const int ldb, StorageWeight* ptr,+ parallel::IThreading* threading) {+ auto tmpq = utils::amalloc<int8_t>(static_cast<size_t>(N) * K);+ int nk_scale = utils::updiv(K, ptr->mBlockSize);+ auto ssize = static_cast<size_t>(N) * nk_scale;+ auto Tscales = utils::amalloc<float>(ssize);+ auto Tzps = utils::amalloc<int8_t>(ptr->IsAsym() ? ssize : 0);+ quantizeWeight(N, K, B, ldb, tmpq, Tscales, Tzps, ptr, threading);+ packQWeight(N, K, tmpq, N, Tscales, Tzps, ptr, threading);+ utils::afree(tmpq);+ utils::afree(Tscales);+ utils::afree(Tzps);+ }+ template <typename T>+ AUTOCALL void unpackWeight(const int N, const int K, StorageWeight* stor, T* B, const int ldb,+ parallel::IThreading* threading) {+ parallel::Scheduler2D _para({threading->num_threads(), K, N, _GemmCore_T::KTILE, _GemmCore_T::NTILE});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ auto rowpad = utils::padto(thdp.size[0], _GemmCore_T::KTILE);+ auto colpad = utils::padto(thdp.size[1], _GemmCore_T::NTILE);+ auto dequant = utils::amalloc<T>((size_t)rowpad * colpad);+ auto dstptr = dequant;+ int dststep = 0;+ size_t constexpr CacheSize = size_t(100) << 10;+ int8_t tmpcache[CacheSize];+ GetCPUDevice();+ if (_cd->AVX512F()) {+ getWeight<BTLA_ISA::AVX512F>(&dstptr, &dststep, rowpad, colpad, thdp.loc[0], thdp.loc[1], {stor}, tmpcache,+ CacheSize);+ } else if (_cd->AVX2()) {+ getWeight<BTLA_ISA::AVX2>(&dstptr, &dststep, rowpad, colpad, thdp.loc[0], thdp.loc[1], {stor}, tmpcache,+ CacheSize);+ } else {+ getWeight<BTLA_ISA::NoSIMD>(&dstptr, &dststep, rowpad, colpad, thdp.loc[0], thdp.loc[1], {stor}, tmpcache,+ CacheSize);+ }+ kernel::wrapper::RevertPaddingInterleaveMN<_GemmCore_T::NTILE, _GemmCore_T::PACK_ROW, T>::forward_auto(+ dstptr, B + thdp.loc[0] * ldb + thdp.loc[1], thdp.size[0], thdp.size[1], rowpad, colpad, dststep, ldb);+ utils::afree(dequant);+ }+ });+ }++ AUTOCALL void setQuantCorrection(const int N, const int K, const int8_t* zero_points, const float* scales,+ StorageWeight* stor, parallel::IThreading* threading) {+ int rawnk_scale = utils::updiv(K, stor->mBlockSize);+ int nk_scale = utils::updiv(stor->mKPad, stor->mBlockSize);+ parallel::Scheduler2D _para({threading->num_threads(), 1, nk_scale, 1, 1});+ if (stor->SDtype() == BTLA_DTYPE::BF16 || stor->SDtype() == BTLA_DTYPE::F16 || stor->SDtype() == BTLA_DTYPE::F32) {+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ int rows = thdp.loc[1] + thdp.size[1] <= rawnk_scale ? thdp.size[1] : rawnk_scale - thdp.loc[1];+ if (scales) {+ if (stor->SDtype() == BTLA_DTYPE::BF16) {+ kernel::wrapper::Memcpy2DFp32TPadding<utils::bf16>::forward_auto(+ scales + thdp.loc[1] * N, stor->template SPtr<utils::bf16>() + thdp.loc[1] * stor->mNPad, rows, N,+ N * sizeof(scales[0]), stor->mNPad * sizeof(utils::bf16), true);+ } else if (stor->SDtype() == BTLA_DTYPE::F32) {+ kernel::wrapper::Memcpy2DPadding::forward(+ scales + thdp.loc[1] * N, stor->template SPtr<float>() + thdp.loc[1] * stor->mNPad, rows,+ N * sizeof(float), N * sizeof(scales[0]), stor->mNPad * sizeof(float), true);+ } else if (stor->SDtype() == BTLA_DTYPE::F16) {+ kernel::wrapper::Memcpy2DFp32TPadding<utils::fp16>::forward_auto(+ scales + thdp.loc[1] * N, stor->template SPtr<utils::fp16>() + thdp.loc[1] * stor->mNPad, rows, N,+ N * sizeof(scales[0]), stor->mNPad * sizeof(utils::fp16), true);+ }+ if (rows < thdp.size[1]) {+ auto sb = bestla::utils::bestla_dtype_bytes(stor->SDtype());+ if (sb == 2) {+ std::memset(stor->template SPtr<utils::fp16>() + (thdp.loc[1] + rows) * stor->mNPad, 0,+ sb * (thdp.size[1] - rows) * stor->mNPad);+ } else if (sb == 4) {+ std::memset(stor->template SPtr<float>() + (thdp.loc[1] + rows) * stor->mNPad, 0,+ sb * (thdp.size[1] - rows) * stor->mNPad);+ } else {+ assert(0);+ }+ }+ }+ if (zero_points) {+ kernel::wrapper::Memcpy2DPadding::forward(+ zero_points + thdp.loc[1] * N, stor->template ZPtr<int8_t>() + thdp.loc[1] * stor->mNPad, rows,+ N * sizeof(zero_points[0]), N * sizeof(zero_points[0]), sizeof(int8_t) * stor->mNPad, true);++ if (rows < thdp.size[1]) {+ std::memset(stor->template ZPtr<int8_t>() + (thdp.loc[1] + rows) * stor->mNPad, 0,+ sizeof(int8_t) * (thdp.size[1] - rows) * stor->mNPad);+ }+ }+ }+ });+ } else if (stor->SDtype() == BTLA_DTYPE::F8_E8M0) {+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ for (int i = thdp.loc[1]; i < thdp.loc[1] + thdp.size[1]; i++) {+ if (i < rawnk_scale) {+ if (scales != nullptr) {+ for (size_t j = 0; j < N; j++) {+ stor->template SPtr<utils::f8>()[j + i * stor->mNPad] = static_cast<int8_t>(scales[i * N + j]);+ }+ }+ } else {+ if (scales != nullptr)+ std::memset(stor->template SPtr<utils::f8>() + i * stor->mNPad, 0, stor->mNPad * sizeof(utils::f8));+ }+ }+ }+ });+ } else if (stor->SDtype() == BTLA_DTYPE::DQ8_BNB) {+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ for (int i = thdp.loc[1]; i < thdp.loc[1] + thdp.size[1]; i++) {+ if (i < rawnk_scale) {+ if (scales != nullptr) {+ for (size_t j = 0; j < N; j++) {+ stor->template SPtr<uint8_t>()[j + i * stor->mNPad] = static_cast<uint8_t>(scales[i * N + j]);+ }+ }+ } else {+ if (scales != nullptr)+ std::memset(stor->template SPtr<uint8_t>() + i * stor->mNPad, 0, stor->mNPad * sizeof(uint8_t));+ }+ }+ }+ });+ } else {+ assert(0);+ }+ }++ AUTOCALL void setShuffleIndices(const int* groupindices, StorageWeight* stor, parallel::IThreading* threading) {+ auto groupsize = utils::updiv(stor->mK, stor->mBlockSize);+ parallel::Scheduler2D _para({threading->num_threads(), 1, groupsize, 1, 1});+ auto countptr = utils::amalloc<int>(groupsize);+ std::memset(countptr, 0, groupsize * sizeof(int));+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ auto siptr = stor->ShfIndice();+ for (int i = 0; i < stor->mK; i++) {+ if (groupindices[i] >= thdp.loc[1] && groupindices[i] < thdp.loc[1] + thdp.size[1]) {+ siptr[groupindices[i] * stor->mBlockSize + countptr[groupindices[i]]] = i;+ countptr[groupindices[i]]++;+ }+ }+ }+ });+ utils::afree(countptr);+ }++ AUTOCALL void setTransposeQuantCorrection(const int N, const int K, const int8_t* zero_pointsT, const float* scalesT,+ StorageWeight* stor, parallel::IThreading* threading) {+ int rawnk_scale = utils::updiv(K, stor->mBlockSize);+ auto scales = scalesT ? utils::amalloc<float>(rawnk_scale * N) : nullptr;+ auto zero_points = zero_pointsT ? utils::amalloc<int8_t>(rawnk_scale * N) : nullptr;+ if (scales) {+ transposeWeight<float>(N, rawnk_scale, scalesT, rawnk_scale, scales, N, threading);+ }+ if (zero_points) {+ transposeWeight<int8_t>(N, rawnk_scale, zero_pointsT, rawnk_scale, zero_points, N, threading);+ }+ setQuantCorrection(N, K, zero_points, scales, stor, threading);+ if (scales) {+ utils::afree(scales);+ }+ if (zero_points) {+ utils::afree(zero_points);+ }+ }++ AUTOCALL void packQWeight(const int N, const int K, const int8_t* B, const int ldb, const float* scales,+ const int8_t* zero_points, StorageWeight* stor, parallel::IThreading* threading) {+ if (stor->SDtype() == BTLA_DTYPE::DQ8_BNB) assert(stor->mDqBlockSize != 0);+ if (stor->IsDoubleQuant()) {+ int nk_scale = utils::updiv(K, stor->mBlockSize);+ auto ssize = static_cast<size_t>(N) * nk_scale;+ utils::avector<float> dq_buf;+ doubleQuantScale(const_cast<float*>(scales), ssize, stor->mDqBlockSize, stor->SDtype(), &dq_buf);+ setDoubleQuantCorrection(&dq_buf, stor);+ }+ setQuantCorrection(N, K, zero_points, scales, stor, threading);+ if (stor->mDType == BTLA_DTYPE::S8) {+ reorderWeight(N, K, B, ldb, stor->WPtr<int8_t>(), threading);+ } else {+ auto reordered = utils::amalloc<int8_t>((size_t)stor->mKPad * stor->mNPad);+ reorderWeight(N, K, B, ldb, reordered, threading);+ compressWeight(stor->mNPad, stor->mKPad, reordered, stor->mNPad, stor->WPtr<int8_t>(), stor->mDType, threading);+ utils::afree(reordered);+ }+ reduceWeight(stor, threading);+ }++ AUTOCALL void packNbitsWeightQ4(const int N, const int K, bool isasym, const uint8_t* B, const int ldb,+ const float* scales, const uint8_t* zero_points, void* ptr,+ parallel::IThreading* threading) {+ auto stor = reinterpret_cast<StorageWeight*>(ptr);+ auto tmp = utils::amalloc<float>(static_cast<size_t>(stor->mKPad) * stor->mNPad);+ auto blks = utils::updiv(K, stor->mBlockSize);+ auto blks_padding2 = utils::padto(blks, 2);+ auto tmpscales = tmp;+ auto tmpzeropoints = reinterpret_cast<int8_t*>(tmpscales + N * blks);+ assert(isasym == (zero_points != nullptr));+ if (scales) {+ for (size_t i = 0; i < N * blks; i += 1) {+ tmpscales[i] = scales[i];+ }+ }+ if (zero_points) {+ for (size_t i = 0; i < N; i += 1) {+ for (size_t ib = 0; ib < blks; ib += 2) {+ auto tmpzp = *(zero_points + i * blks_padding2 / 2 + ib / 2);+ tmpzeropoints[i * blks + ib] = (tmpzp & 0x0f) - 8;+ if (ib + 1 < blks) {+ tmpzeropoints[i * blks + ib + 1] = ((tmpzp & 0xf0) >> 4) - 8;+ }+ }+ }+ }++ setTransposeQuantCorrection(N, K, zero_points ? tmpzeropoints : nullptr, scales ? tmpscales : nullptr, stor,+ threading);+ if (B) {+ auto s8ptr = (int8_t*)tmp;+ auto transposeunpackfunc_u4s4 = [&]() {+ parallel::Scheduler2D para({threading->num_threads(), N, K, 1, 2});+ threading->parallel_for([&](int tid) {+ parallel::ThreadProblem2D thdp{tid};+ para.getIndex(thdp);+ if (thdp.valid) {+ for (size_t i = thdp.loc[0]; i < thdp.loc[0] + thdp.size[0]; i++) {+ for (size_t j = thdp.loc[1]; j < thdp.loc[1] + thdp.size[1]; j += 2) {+ auto src = *(B + i * ldb / 2 + j / 2);+ s8ptr[(j + 0) * N + i] = ((src & 0xf) - 8);+ s8ptr[(j + 1) * N + i] = (((src & 0xf0) >> 4) - 8);+ }+ }+ }+ });+ };+ transposeunpackfunc_u4s4();+ auto reordered = s8ptr + static_cast<size_t>(K) * N;+ reorderWeight(N, K, s8ptr, N, reordered, threading);+ compressWeight(stor->mNPad, stor->mKPad, reordered, stor->mNPad, stor->WPtr<int8_t>(), stor->mDType, threading);+ }+ utils::afree(tmp);+ }++ AUTOCALL void reduceWeight(StorageWeight* stor, parallel::IThreading* threading) {+ if (stor->HasReduce()) {+ auto deq = utils::amalloc<int8_t>((size_t)stor->mK * stor->mN);+ unpackWeight(stor->mN, stor->mK, stor, deq, stor->mN, threading);+ reduce(stor->mN, stor->mK, stor->mBlockSize, deq, stor->mN, stor->template RPtr<int32_t>(), stor->CStep(),+ threading);+ utils::afree(deq);+ }+ }++ AUTOCALL void quantizeWeight(const int N, const int K, const float* B, const int ldb, int8_t* qB, float* scales,+ int8_t* zero_points, void* stor, parallel::IThreading* threading) {+ auto ptr = reinterpret_cast<StorageWeight*>(stor);+ int bsize = ptr->mBlockSize == -1 ? K : ptr->mBlockSize;+ parallel::Scheduler2D _para({threading->num_threads(), K, N, bsize, 16});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp({tidx});+ _para.getIndex(thdp);+ if (thdp.valid) {+ kernel::wrapper::QuantizeSignIntRowBlock::forward_auto(+ B + thdp.loc[0] * ldb + thdp.loc[1], qB + thdp.loc[0] * N + thdp.loc[1], thdp.size[0], thdp.size[1], ldb, N,+ scales + thdp.loc[0] / bsize * N + thdp.loc[1],+ zero_points == nullptr ? zero_points : zero_points + thdp.loc[0] / bsize * N + thdp.loc[1], ptr->mBlockSize,+ ptr->mDType);+ }+ });+ }++ AUTOCALL void reorderWeight(const int N, const int K, const int8_t* B, const int ldb, int8_t* dstptr,+ parallel::IThreading* threading) {+ int KPad = utils::padto(K, _GemmCore_T::KTILE);+ parallel::Scheduler2D _para({threading->num_threads(), K, N, _GemmCore_T::KTILE, _GemmCore_T::NTILE});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp({tidx});+ _para.getIndex(thdp);+ if (thdp.valid) {+ auto rowpadded = utils::padto(thdp.size[0], _GemmCore_T::KTILE);+ auto colpadded = utils::padto(thdp.size[1], _GemmCore_T::NTILE);+ const auto src = B + thdp.loc[0] * ldb + thdp.loc[1];+ const auto dst = dstptr + thdp.loc[0] * _GemmCore_T::NTILE + thdp.loc[1] * KPad;+ using PaddingInterleaveMNWType =+ kernel::wrapper::PaddingInterleaveMN<_GemmCore_T::NTILE, _GemmCore_T::PACK_ROW, int8_t>;+ auto ret = PaddingInterleaveMNWType::forward_auto( //+ src, dst, thdp.size[0], thdp.size[1], rowpadded, colpadded, ldb, KPad);+ assert(ret == BTLA_CODE::Success);+ (void)ret;+ }+ });+ }++ AUTOCALL void compressBit3Weight(const int N, const int K, const int8_t* B, int8_t* dstptr, BTLA_DTYPE qtype,+ parallel::IThreading* threading) {+ auto bit1_offset = size_t(N) * K;+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(dstptr);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(dstptr + bit1_offset / 4);+ auto ret = kernel::wrapper::CompressBit3::forward_auto(B, bit2ptr, bit1ptr, bit1_offset);+ assert(ret == BTLA_CODE::Success);+ }++ AUTOCALL void compressBit5Weight(const int N, const int K, const int8_t* B, int8_t* dstptr, BTLA_DTYPE qtype,+ parallel::IThreading* threading) {+ auto bit1_offset = size_t(N) * K;+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(dstptr);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(dstptr + bit1_offset / 2);+ auto ret = kernel::wrapper::CompressBit5::forward_auto(B, bit4ptr, bit1ptr, bit1_offset);+ assert(ret == BTLA_CODE::Success);+ }++ AUTOCALL void compressBit6Weight(const int N, const int K, const int8_t* B, int8_t* dstptr, BTLA_DTYPE qtype,+ parallel::IThreading* threading) {+ auto bit2_offset = size_t(N) * K;+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(dstptr);+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(dstptr + bit2_offset / 2);+ auto ret = kernel::wrapper::CompressBit6::forward_auto(B, bit4ptr, bit2ptr, bit2_offset);+ assert(ret == BTLA_CODE::Success);+ }++ AUTOCALL void compressBit7Weight(const int N, const int K, const int8_t* B, int8_t* dstptr, BTLA_DTYPE qtype,+ parallel::IThreading* threading) {+ auto eltsize = size_t(N) * K;+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(dstptr);+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit4ptr + eltsize / 2);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit2ptr + eltsize / 4);+ auto ret = kernel::wrapper::CompressBit7::forward_auto(B, bit4ptr, bit2ptr, bit1ptr, eltsize);+ assert(ret == BTLA_CODE::Success);+ }++ AUTOCALL void compressBit2Weight(const int N, const int K, const int8_t* B, int8_t* dstptr, BTLA_DTYPE qtype,+ parallel::IThreading* threading) {+ // TODO(zhe): 1D parallel compress+ parallel::Scheduler2D _para({threading->num_threads(), 1, K * N, 1, 64});+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(dstptr);+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp({tidx});+ _para.getIndex(thdp);+ if (thdp.valid) {+ auto ret =+ kernel::wrapper::CompressBit2::forward_auto(B + thdp.loc[1], bit2ptr + thdp.loc[1] / 4, thdp.size[1]);+ assert(ret == BTLA_CODE::Success);+ (void)ret;+ }+ });+ }++ AUTOCALL void compressBit1Weight(const int N, const int K, const int8_t* B, int8_t* dstptr, BTLA_DTYPE qtype,+ parallel::IThreading* threading) {+ // TODO(zhe): 1D parallel compress+ parallel::Scheduler2D _para({threading->num_threads(), 1, K * N, 1, 64});+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(dstptr);+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp({tidx});+ _para.getIndex(thdp);+ if (thdp.valid) {+ auto ret =+ kernel::wrapper::CompressBit1::forward_auto(B + thdp.loc[1], bit1ptr + thdp.loc[1] / 8, thdp.size[1]);+ assert(ret == BTLA_CODE::Success);+ (void)ret;+ }+ });+ }++ AUTOCALL void compressBit4Weight(const int N, const int K, const int8_t* B, int8_t* dstptr, BTLA_DTYPE qtype,+ parallel::IThreading* threading) {+ parallel::Scheduler2D _para({threading->num_threads(), 1, K * N, 1, 64});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp({tidx});+ _para.getIndex(thdp);+ if (thdp.valid) {+ BTLA_CODE ret = BTLA_CODE::NotSupport;+ if (qtype == BTLA_DTYPE::S4_CLIP) {+ auto bit4ptr = reinterpret_cast<utils::int4x2*>(dstptr);+ ret = kernel::wrapper::CompressS8S4::forward_auto(B + thdp.loc[1], bit4ptr + thdp.loc[1] / 2, thdp.size[1]);+ } else if (qtype == BTLA_DTYPE::F4_BNB || qtype == BTLA_DTYPE::F4_NF4 || qtype == BTLA_DTYPE::F4_E2M1) {+ auto bit4ptr = reinterpret_cast<utils::f4x2*>(dstptr);+ ret = kernel::wrapper::CompressFp4::forward_auto(B + thdp.loc[1], bit4ptr + thdp.loc[1] / 2, thdp.size[1]);+ } else {+ assert(0);+ }+ assert(ret == BTLA_CODE::Success);+ (void)ret;+ }+ });+ }++ AUTOCALL void compressWeight(const int N, const int K, const int8_t* B, const int ldb, int8_t* dstptr,+ BTLA_DTYPE qtype, parallel::IThreading* threading) {+ if (qtype == BTLA_DTYPE::S7_CLIP) return compressBit7Weight(N, K, B, dstptr, qtype, threading);+ if (qtype == BTLA_DTYPE::S6_CLIP) return compressBit6Weight(N, K, B, dstptr, qtype, threading);+ if (qtype == BTLA_DTYPE::S5_CLIP) return compressBit5Weight(N, K, B, dstptr, qtype, threading);+ if (qtype == BTLA_DTYPE::S4_CLIP) return compressBit4Weight(N, K, B, dstptr, qtype, threading);+ if (qtype == BTLA_DTYPE::S3_CLIP) return compressBit3Weight(N, K, B, dstptr, qtype, threading);+ if (qtype == BTLA_DTYPE::S2_CLIP) return compressBit2Weight(N, K, B, dstptr, qtype, threading);+ if (qtype == BTLA_DTYPE::S1_CLIP) return compressBit1Weight(N, K, B, dstptr, qtype, threading);+ if (qtype == BTLA_DTYPE::F4_BNB || qtype == BTLA_DTYPE::F4_NF4 || qtype == BTLA_DTYPE::F4_E2M1)+ return compressBit4Weight(N, K, B, dstptr, qtype, threading);+ }++ AUTOCALL void reduce(const int N, const int K, const int KBlock, const int8_t* B, const int ldb, int32_t* rptr,+ const int ldr, parallel::IThreading* threading) {+ parallel::Scheduler2D _para({threading->num_threads(), K, N, KBlock, 16});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp({tidx});+ _para.getIndex(thdp);+ if (thdp.valid) {+ const auto src = B + thdp.loc[0] * ldb + thdp.loc[1];+ const auto dst = rptr + thdp.loc[1] + thdp.loc[0] / KBlock * ldr;+ using RowReduceSum = kernel::wrapper::RowReduceSum<int8_t, int32_t>;+ for (int i = 0; i < thdp.size[0]; i += KBlock) {+ int rowremain = utils::remainsize(thdp.loc[0] + i, K, KBlock);+ auto ret = RowReduceSum::forward_auto(src + i * ldb, ldb, rowremain, thdp.size[1], dst + i / KBlock * ldr);+ assert(ret == BTLA_CODE::Success);+ (void)ret;+ }+ }+ });+ }++ public:+ template <BTLA_ISA ISA_T, typename _T>+ static inline BTLA_CODE getWeight(_T** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ if constexpr (std::is_same_v<_T, int8_t>) {+ return getQWeight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ }+ return getFpWeight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ }++ TLACALL BTLA_CODE getQWeight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.packedW;+ if (wptr->mDType == BTLA_DTYPE::S8) {+ return getQ8Weight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::S6_CLIP) {+ return getQ6Weight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::S5_CLIP) {+ return getQ5Weight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::S4_CLIP) {+ return getQ4Weight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::S3_CLIP) {+ return getQ3Weight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::S2_CLIP) {+ return getQ2Weight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::S7_CLIP) {+ return getQ7Weight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::S1_CLIP) {+ return getQ1Weight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ } else {+ assert(0);+ }+ return BTLA_CODE::NotSupport;+ }++ TLACALL BTLA_CODE getScale(float** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.packedW;+ if (wptr->SDtype() == BTLA_DTYPE::F32) {+ auto aptr = wptr->template SPtr<float>();+ kernel::wrapper::Memcpy2D::template forward<BTLA_ISA::NoSIMD>(+ aptr + k_offset / wptr->mBlockSize * wptr->CStep() + n_offset, *dstptr,+ utils::updiv(k_size, wptr->mBlockSize), n_size, wptr->CStep(), n_size);+ *dststep = n_size;+ }+ if (wptr->SDtype() == BTLA_DTYPE::BF16) {+ auto aptr = wptr->template SPtr<utils::bf16>();+ kernel::wrapper::Memcpy2DBf16CvtFp32::forward<ISA_T>(+ aptr + k_offset / wptr->mBlockSize * wptr->CStep() + n_offset, *dstptr,+ utils::updiv(k_size, wptr->mBlockSize), n_size, wptr->CStep() * 2, n_size * 4, false);+ *dststep = n_size;+ }+ if (wptr->SDtype() == BTLA_DTYPE::F16) {+ auto aptr = wptr->template SPtr<utils::fp16>();+ kernel::wrapper::Memcpy2DFp16CvtFp32::forward<ISA_T>(+ aptr + k_offset / wptr->mBlockSize * wptr->CStep() + n_offset, *dstptr,+ utils::updiv(k_size, wptr->mBlockSize), n_size, wptr->CStep() * 2, n_size * 4, false);+ *dststep = n_size;+ }+ if (wptr->SDtype() == BTLA_DTYPE::DQ8_BNB) {+ auto aptr = wptr->template SPtr<uint8_t>();+ auto internal_k_offset = k_offset / wptr->mBlockSize;+ auto dq_offset_idx = static_cast<int>(wptr->mCorrection.mDQCorrectionBuf.mBufSize / sizeof(float) - 1);+ kernel::wrapper::Dq8GetScale::template forward<ISA_T>(+ aptr + internal_k_offset * wptr->CStep() + n_offset, *dstptr, utils::updiv(k_size, wptr->mBlockSize), n_size,+ internal_k_offset * wptr->mN + n_offset, wptr->mDqBlockSize, dq_offset_idx, wptr->DQPtr<float>(),+ wptr->CStep(), n_size, false, wptr->mN);+ }+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getReduce(int32_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.packedW;+ auto rptr = wptr->template RPtr<int32_t>();+ kernel::wrapper::Memcpy2D::template forward<BTLA_ISA::NoSIMD>(+ rptr + k_offset / wptr->mBlockSize * wptr->CStep() + n_offset, *dstptr, utils::updiv(k_size, wptr->mBlockSize),+ n_size, wptr->CStep(), n_size);+ *dststep = n_size;+ return BTLA_CODE::Success;+ }++ protected:+ template <BTLA_ISA ISA_T, typename _T>+ static inline BTLA_CODE getFpWeight(_T** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.packedW;+ auto NPad = wptr->mNPad;+ auto KPad = wptr->mKPad;+ int constexpr ColSize = _GemmCore_T::NTILE * _GemmCore_T::PACK_ROW;+ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ auto zptr = wptr->template ZPtr<int8_t>();+ if (wptr->mDType == BTLA_DTYPE::S4_CLIP) {+ if (wptr->SDtype() == BTLA_DTYPE::DQ8_BNB) {+ auto internal_n_offset = n_offset + i;+ int dq_offset = static_cast<int>(wptr->mCorrection.mDQCorrectionBuf.mBufSize / sizeof(float) - 1);+ kernel::wrapper::DecompressDQKBlockS4Fp<_T, _GemmCore_T::PACK_ROW>::template forward<ISA_T,+ BTLA_DTYPE::S4_CLIP>(+ wptr->template WPtr<utils::int4x2>() + n_offset * KPad / 2 + k_offset * _GemmCore_T::NTILE / 2 ++ i * KPad / 2,+ *dstptr + i * k_size, k_size / _GemmCore_T::PACK_ROW, ColSize, ColSize, ColSize,+ wptr->template SPtr<uint8_t>(), wptr->template DQPtr<float>(), k_offset / _GemmCore_T::PACK_ROW,+ internal_n_offset, wptr->mBlockSize / _GemmCore_T::PACK_ROW, NPad, wptr->mN, wptr->mDqBlockSize,+ dq_offset, tmpcache, cachesize);+ } else {+ auto sptr = wptr->template SPtr<void>();+ kernel::wrapper::DecompressKBlockS4Fp<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE, _T>::template forward<ISA_T>(+ wptr->template WPtr<utils::int4x2>() + n_offset * KPad / 2 + k_offset * _GemmCore_T::NTILE / 2 ++ i * KPad / 2,+ *dstptr + i * k_size, k_size, _GemmCore_T::NTILE, sptr, wptr->SDtype(), zptr, k_offset, n_offset + i,+ wptr->mBlockSize, NPad, tmpcache, cachesize);+ }++ } else if (wptr->mDType == BTLA_DTYPE::S3_CLIP) {+ auto sptr = wptr->template SPtr<void>();+ int8_t* bit3_ptr = wptr->template WPtr<int8_t>();+ auto elt_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE + i * KPad;+ assert(elt_offset % 8 == 0);+ size_t bit1_offset = size_t(NPad) * KPad;+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit3_ptr + elt_offset / 4);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit3_ptr + bit1_offset / 4 + elt_offset / 8);+ kernel::wrapper::DecompressKBlockS3Fp<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE, _T>::template forward<ISA_T>(+ bit2ptr, bit1ptr, *dstptr + i * k_size, k_size, _GemmCore_T::NTILE, sptr, wptr->SDtype(), zptr, k_offset,+ n_offset + i, wptr->mBlockSize, NPad, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::S2_CLIP) {+ auto sptr = wptr->template SPtr<void>();+ int8_t* bit2_ptr = wptr->template WPtr<int8_t>();+ auto elt_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE + i * KPad;+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit2_ptr + elt_offset / 4);+ kernel::wrapper::DecompressKBlockS2Fp<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE, _T>::template forward<ISA_T>(+ bit2ptr, *dstptr + i * k_size, k_size, _GemmCore_T::NTILE, sptr, wptr->SDtype(), zptr, k_offset,+ n_offset + i, wptr->mBlockSize, NPad, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::S8) {+ auto sptr = wptr->template SPtr<void>();+ auto elt_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE + i * KPad;+ int8_t* bptr = wptr->template WPtr<int8_t>() + elt_offset;+ kernel::wrapper::DecompressKBlockS8Fp<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE, _T>::template forward<ISA_T>(+ bptr, *dstptr + i * k_size, k_size, _GemmCore_T::NTILE, sptr, wptr->SDtype(), zptr, k_offset, n_offset + i,+ wptr->mBlockSize, NPad, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::S5_CLIP) {+ auto sptr = wptr->template SPtr<void>();+ int8_t* bit5_ptr = wptr->template WPtr<int8_t>();+ auto elt_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE + i * KPad;+ assert(elt_offset % 8 == 0);+ size_t bit1_offset = size_t(NPad) * KPad;+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(bit5_ptr + elt_offset / 2);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit5_ptr + bit1_offset / 2 + elt_offset / 8);+ kernel::wrapper::DecompressKBlockS5Fp<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE, _T>::template forward<ISA_T>(+ bit4ptr, bit1ptr, *dstptr + i * k_size, k_size, _GemmCore_T::NTILE, sptr, wptr->SDtype(), zptr, k_offset,+ n_offset + i, wptr->mBlockSize, NPad, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::S6_CLIP) {+ auto sptr = wptr->template SPtr<void>();+ int8_t* bit6_ptr = wptr->template WPtr<int8_t>();+ auto elt_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE + i * KPad;+ assert(elt_offset % 4 == 0);+ size_t bit2_offset = size_t(NPad) * KPad;+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(bit6_ptr + elt_offset / 2);+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit6_ptr + bit2_offset / 2 + elt_offset / 4);+ kernel::wrapper::DecompressKBlockS6Fp<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE, _T>::template forward<ISA_T>(+ bit4ptr, bit2ptr, *dstptr + i * k_size, k_size, _GemmCore_T::NTILE, sptr, wptr->SDtype(), zptr, k_offset,+ n_offset + i, wptr->mBlockSize, NPad, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::S7_CLIP) {+ auto sptr = wptr->template SPtr<void>();+ int8_t* bit7_ptr = wptr->template WPtr<int8_t>();+ auto elt_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE + i * KPad;+ assert(elt_offset % 8 == 0);+ size_t bit2_offset = size_t(NPad) * KPad;+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(bit7_ptr + elt_offset / 2);+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit7_ptr + bit2_offset / 2 + elt_offset / 4);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit7_ptr + bit2_offset / 2 + bit2_offset / 4 + elt_offset / 8);+ kernel::wrapper::DecompressKBlockS7Fp<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE, _T>::template forward<ISA_T>(+ bit4ptr, bit2ptr, bit1ptr, *dstptr + i * k_size, k_size, _GemmCore_T::NTILE, sptr, wptr->SDtype(), zptr,+ k_offset, n_offset + i, wptr->mBlockSize, NPad, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::S1_CLIP) {+ auto sptr = wptr->template SPtr<void>();+ int8_t* bit1_ptr = wptr->template WPtr<int8_t>();+ auto elt_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE + i * KPad;+ assert(elt_offset % 8 == 0);+ size_t bit1_offset = size_t(NPad) * KPad;+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit1_ptr + elt_offset / 8);+ kernel::wrapper::DecompressKBlockS1Fp<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE, _T>::template forward<ISA_T>(+ bit1ptr, *dstptr + i * k_size, k_size, _GemmCore_T::NTILE, sptr, wptr->SDtype(), zptr, k_offset,+ n_offset + i, wptr->mBlockSize, NPad, tmpcache, cachesize);+ } else {+ assert(0);+ }+ }+ *dststep = k_size;+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getQ8Weight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.packedW;+ auto KPad = wptr->mKPad;+ auto bptr = wptr->template WPtr<int8_t>() + n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ kernel::wrapper::Memcpy2D::template forward<BTLA_ISA::NoSIMD, int8_t, int8_t>(+ bptr, *dstptr, n_size / _GemmCore_T::NTILE, _GemmCore_T::NTILE * k_size, _GemmCore_T::NTILE * KPad,+ _GemmCore_T::NTILE * k_size);+ *dststep = k_size;+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getQ4Weight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.packedW;+ auto KPad = wptr->mKPad;+ auto bptr = wptr->template WPtr<utils::int4x2>() + n_offset * KPad / 2 + k_offset * _GemmCore_T::NTILE / 2;+ auto zpptr = wptr->template ZPtr<int8_t>();+ int constexpr ColSize = _GemmCore_T::NTILE * _GemmCore_T::PACK_ROW;++ assert(wptr->mDType == BTLA_DTYPE::S4_CLIP);++ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ kernel::wrapper::DecompressKBlockS4S8<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE>::template forward<ISA_T>(+ bptr + i * KPad / 2, wptr->IsAsym() ? zpptr : nullptr, *dstptr + i * k_size, wptr->mBlockSize, wptr->CStep(),+ n_offset + i, k_offset, k_size, _GemmCore_T::NTILE, tmpcache, cachesize);+ }+ *dststep = k_size;+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getQ3Weight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.packedW;+ int8_t* bit3_ptr = wptr->template WPtr<int8_t>();+ auto zpptr = wptr->template ZPtr<int8_t>();+ auto KPad = wptr->mKPad;+ auto NPad = wptr->mNPad;+ size_t bit1_offset = size_t(NPad) * KPad;+ auto base_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ auto elt_offset = base_offset + i * KPad;+ assert(elt_offset % 8 == 0);+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit3_ptr + elt_offset / 4);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit3_ptr + bit1_offset / 4 + elt_offset / 8);+ kernel::wrapper::DecompressKBlockS3S8<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE>::template forward<ISA_T>(+ bit2ptr, bit1ptr, wptr->IsAsym() ? zpptr : nullptr, *dstptr + i * k_size, wptr->mBlockSize, wptr->CStep(),+ n_offset + i, k_offset, k_size, _GemmCore_T::NTILE, tmpcache, cachesize);+ }+ *dststep = k_size;+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getQ5Weight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.packedW;+ int8_t* bit5_ptr = wptr->template WPtr<int8_t>();+ auto zpptr = wptr->template ZPtr<int8_t>();+ auto KPad = wptr->mKPad;+ auto NPad = wptr->mNPad;+ size_t bit1_offset = size_t(NPad) * KPad;+ auto base_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ auto elt_offset = base_offset + i * KPad;+ assert(elt_offset % 8 == 0);+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(bit5_ptr + elt_offset / 2);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit5_ptr + bit1_offset / 2 + elt_offset / 8);+ kernel::wrapper::DecompressKBlockS5S8<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE>::template forward<ISA_T>(+ bit4ptr, bit1ptr, wptr->IsAsym() ? zpptr : nullptr, *dstptr + i * k_size, wptr->mBlockSize, wptr->CStep(),+ n_offset + i, k_offset, k_size, _GemmCore_T::NTILE, tmpcache, cachesize);+ }+ *dststep = k_size;+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getQ6Weight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.packedW;+ int8_t* bit6_ptr = wptr->template WPtr<int8_t>();+ auto zpptr = wptr->template ZPtr<int8_t>();+ auto KPad = wptr->mKPad;+ auto NPad = wptr->mNPad;+ size_t bit2_offset = size_t(NPad) * KPad;+ auto base_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ auto elt_offset = base_offset + i * KPad;+ assert(elt_offset % 4 == 0);+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(bit6_ptr + elt_offset / 2);+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit6_ptr + bit2_offset / 2 + elt_offset / 4);+ kernel::wrapper::DecompressKBlockS6S8<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE>::template forward<ISA_T>(+ bit4ptr, bit2ptr, wptr->IsAsym() ? zpptr : nullptr, *dstptr + i * k_size, wptr->mBlockSize, wptr->CStep(),+ n_offset + i, k_offset, k_size, _GemmCore_T::NTILE, tmpcache, cachesize);+ }+ *dststep = k_size;+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getQ7Weight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.packedW;+ int8_t* bit6_ptr = wptr->template WPtr<int8_t>();+ auto zpptr = wptr->template ZPtr<int8_t>();+ auto KPad = wptr->mKPad;+ auto NPad = wptr->mNPad;+ size_t bit2_offset = size_t(NPad) * KPad;+ auto base_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ auto elt_offset = base_offset + i * KPad;+ assert(elt_offset % 8 == 0);+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(bit6_ptr + elt_offset / 2);+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit6_ptr + bit2_offset / 2 + elt_offset / 4);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit6_ptr + bit2_offset / 2 + bit2_offset / 4 + elt_offset / 8);+ kernel::wrapper::DecompressKBlockS7S8<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE>::template forward<ISA_T>(+ bit4ptr, bit2ptr, bit1ptr, wptr->IsAsym() ? zpptr : nullptr, *dstptr + i * k_size, wptr->mBlockSize,+ wptr->CStep(), n_offset + i, k_offset, k_size, _GemmCore_T::NTILE, tmpcache, cachesize);+ }+ *dststep = k_size;+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getQ2Weight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.packedW;+ int8_t* bit2_ptr = wptr->template WPtr<int8_t>();+ int8_t* zpptr = wptr->template ZPtr<int8_t>();+ auto KPad = wptr->mKPad;+ auto NPad = wptr->mNPad;+ auto base_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ auto elt_offset = base_offset + i * KPad;+ assert(elt_offset % 4 == 0);+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit2_ptr + elt_offset / 4);+ kernel::wrapper::DecompressKBlockS2S8<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE>::template forward<ISA_T>(+ bit2ptr, wptr->IsAsym() ? zpptr : nullptr, *dstptr + i * k_size, wptr->mBlockSize, wptr->CStep(),+ n_offset + i, k_offset, k_size, _GemmCore_T::NTILE, tmpcache, cachesize);+ }+ *dststep = k_size;+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getQ1Weight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.packedW;+ int8_t* bit1_ptr = wptr->template WPtr<int8_t>();+ int8_t* zpptr = wptr->template ZPtr<int8_t>();+ auto KPad = wptr->mKPad;+ auto NPad = wptr->mNPad;+ auto base_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ auto elt_offset = base_offset + i * KPad;+ assert(elt_offset % 8 == 0);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit1_ptr + elt_offset / 8);+ kernel::wrapper::DecompressKBlockS1S8<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE>::template forward<ISA_T>(+ bit1ptr, wptr->IsAsym() ? zpptr : nullptr, *dstptr + i * k_size, wptr->mBlockSize, wptr->CStep(),+ n_offset + i, k_offset, k_size, _GemmCore_T::NTILE, tmpcache, cachesize);+ }+ *dststep = k_size;+ return BTLA_CODE::Success;+ }+};++struct ParamWeightKBlockNInt {+ const storage::gemm::StorageWeightNInt* stor_;+};++template <class _GemmCore_T>+class WeightKBlockNInt {+ public:+ using Storage = storage::gemm::StorageWeightNInt;+ using BType = typename _GemmCore_T::BType;+ using Param = ParamWeightKBlockNInt;++ AUTOCALL Storage create_storage(int n, int k, int blocksize, BTLA_DTYPE qtype, BTLA_DTYPE scat, bool is_asym) {+ int KPad = utils::padto(k, _GemmCore_T::KTILE);+ int NPad = utils::padto(n, _GemmCore_T::NTILE);+ Storage tmp(_GemmCore_T::ID);+ tmp.resize(NPad, KPad, blocksize <= 0 ? KPad : blocksize, n, k, qtype, storage::gemm::Layout::NPack, scat, is_asym);+ return tmp;+ }++ AUTOCALL void convert(const storage::gemm::StorageWeightNInt& src, const storage::gemm::StorageWeightNInt& dst,+ parallel::IThreading* threading) {+ using storage::gemm::Layout;+ if (src.info_.layout_ == storage::gemm::Layout::NT) {+ convert_NT(src, dst, threading);+ }+ if (src.info_.layout_ == storage::gemm::Layout::T) {+ if (src.info_.dtype_ == BTLA_DTYPE::F32) { // transpose to NT and quantize & reorder+ auto tmpptr = utils::amalloc<int8_t>(src.buf_size_);+ storage::gemm::StorageWeightNInt nt_stor;+ nt_stor.set_nt_data(src.info_.n_, src.info_.k_, src.info_.n_, src.info_.dtype_, tmpptr);+ transposeWeight<float>(src.info_.n_, src.info_.k_, (const float*)src.buf_.buf_ptr_, src.info_.kpad_,+ (float*)tmpptr, src.info_.n_, threading);+ convert_NT(nt_stor, dst, threading);+ utils::afree(tmpptr);+ }+ if (src.info_.dtype_ == dst.info_.dtype_) { // reorder+ reorder_T(src, dst, threading);+ }+ }+ if (src.info_.layout_ == storage::gemm::Layout::NPack) {+ convert_NPack(src, dst, threading);+ }+ }+ AUTOCALL void reorder_T(const storage::gemm::StorageWeightNInt& src, const storage::gemm::StorageWeightNInt& dst,+ parallel::IThreading* threading) {+ assert(src.info_.layout_ == storage::gemm::Layout::T);+ assert(src.info_.dtype_ == dst.info_.dtype_);+ using storage::gemm::Layout;+ if (dst.info_.layout_ == storage::gemm::Layout::NPack) {+ if (src.info_.dtype_ == dst.info_.dtype_) { // quantize and reorder+ auto tmpq = utils::amalloc<int8_t>(static_cast<size_t>(src.info_.n_) * src.info_.k_);+ utils::avector<int8_t> tmpqT(static_cast<size_t>(src.info_.n_) * src.info_.k_);+ auto ssize = static_cast<size_t>(src.info_.n_) * dst.n_block_;+ auto Tscales = utils::amalloc<float>(ssize);+ auto TscalesNT = utils::amalloc<float>(ssize);+ if (src.info_.dtype_ == BTLA_DTYPE::S8) {+ transpose(src.info_.n_, src.info_.k_, src.buf_.buf_ptr_, BTLA_DTYPE::S8, src.info_.k_, tmpq, src.info_.n_,+ threading);+ } else {+ decompress_weight(src.info_.n_, src.info_.k_, src, tmpqT.data(), src.info_.k_, threading);+ transpose(src.info_.n_, src.info_.k_, tmpqT.data(), BTLA_DTYPE::S8, src.info_.k_, tmpq, src.info_.n_,+ threading);+ }+ transpose(src.info_.n_, src.n_block_, src.corr_.scale_buf_.buf_ptr_, src.corr_.sdtype_, src.n_block_,+ (int8_t*)TscalesNT, src.info_.n_, threading);+ kernel::wrapper::Memcpy2DCvtPadding::forward_auto(TscalesNT, src.corr_.sdtype_, Tscales, BTLA_DTYPE::F32,+ src.n_block_, src.info_.n_, src.info_.n_, src.info_.n_, false,+ tmpqT.data(), tmpqT.size());+ auto Tzps = utils::amalloc<int8_t>(dst.corr_.zp_buf_.buf_size_ ? ssize : 0);+ if (src.corr_.zptr()) {+ transpose(src.info_.n_, src.n_block_, src.corr_.zp_buf_.buf_ptr_, src.corr_.zdtype_, src.n_block_,+ (int8_t*)Tzps, src.info_.n_, threading);+ }++ pack_qweight(src.info_.n_, src.info_.k_, tmpq, src.info_.n_, Tscales, Tzps, dst, threading);+ utils::afree(tmpq);+ utils::afree(Tscales);+ utils::afree(TscalesNT);+ utils::afree(Tzps);+ }+ }+ }++ AUTOCALL void convert_NT(const storage::gemm::StorageWeightNInt& src, const storage::gemm::StorageWeightNInt& dst,+ parallel::IThreading* threading) {+ assert(src.info_.layout_ == storage::gemm::Layout::NT);+ assert(src.info_.dtype_ == BTLA_DTYPE::F32);+ using storage::gemm::Layout;+ if (dst.info_.layout_ == storage::gemm::Layout::NPack) {+ if (src.info_.dtype_ == BTLA_DTYPE::F32) { // quantize and reorder+ auto tmpq = utils::amalloc<int8_t>(static_cast<size_t>(src.info_.n_) * src.info_.k_);+ auto ssize = static_cast<size_t>(src.info_.n_) * dst.n_block_;+ auto Tscales = utils::amalloc<float>(ssize);+ auto Tzps = utils::amalloc<int8_t>(dst.corr_.zp_buf_.buf_size_ ? ssize : 0);+ quantize_weight(src.info_.n_, src.info_.k_, (float*)src.buf_.buf_ptr_, src.info_.npad_, tmpq, Tscales, Tzps,+ dst.block_, dst.info_.dtype_, threading);+ pack_qweight(src.info_.n_, src.info_.k_, tmpq, src.info_.n_, Tscales, Tzps, dst, threading);+ utils::afree(tmpq);+ utils::afree(Tscales);+ utils::afree(Tzps);+ }+ }+ }++ AUTOCALL void convert_NPack(const storage::gemm::StorageWeightNInt& src, const storage::gemm::StorageWeightNInt& dst,+ parallel::IThreading* threading) {+ using storage::gemm::Layout;+ if (dst.info_.layout_ == Layout::NT || dst.info_.layout_ == Layout::T) {+ if (dst.info_.dtype_ == BTLA_DTYPE::F32) {+ float* upk_ptr = (float*)dst.buf_.buf_ptr_;+ if (dst.info_.layout_ == Layout::T) {+ upk_ptr = (float*)utils::amalloc<int8_t>(dst.buf_.buf_size_);+ }+ unpack_weight<float>(dst.info_.n_, dst.info_.k_, src, upk_ptr, dst.info_.n_, threading);+ if (dst.info_.layout_ == Layout::T) {+ transposeWeight<float>(dst.info_.k_, dst.info_.n_, upk_ptr, dst.info_.n_, (float*)dst.buf_.buf_ptr_,+ dst.info_.kpad_, threading);+ utils::afree(upk_ptr);+ }+ }+ if (dst.info_.dtype_ == src.info_.dtype_) {+ assert(dst.info_.layout_ == Layout::T);+ auto tmp_buf_NT = utils::amalloc<int8_t>(src.info_.k_ * src.info_.n_);+ if (dst.info_.dtype_ == BTLA_DTYPE::S8) {+ unpack_weight<int8_t>(dst.info_.n_, dst.info_.k_, src, tmp_buf_NT, dst.info_.n_, threading);+ transposeWeight<int8_t>(dst.info_.k_, dst.info_.n_, tmp_buf_NT, dst.info_.n_, dst.buf_.buf_ptr_, dst.info_.k_,+ threading);+ } else {+ auto tmp_buf_T = utils::amalloc<int8_t>(src.info_.k_ * src.info_.n_);+ unpack_weight<int8_t>(dst.info_.n_, dst.info_.k_, src, tmp_buf_NT, dst.info_.n_, threading);+ transposeWeight<int8_t>(dst.info_.k_, dst.info_.n_, tmp_buf_NT, dst.info_.n_, tmp_buf_T, dst.info_.k_,+ threading);+ compressWeight(dst.info_.n_, dst.info_.k_, tmp_buf_T, dst.info_.k_, dst.buf_.buf_ptr_, dst.info_.dtype_,+ threading);+ transpose(src.n_block_, src.info_.n_, src.corr_.scale_buf_.buf_ptr_, src.corr_.sdtype_, src.info_.npad_,+ dst.corr_.scale_buf_.buf_ptr_, dst.n_block_, threading);+ utils::afree(tmp_buf_T);+ }+ transpose(src.n_block_, src.info_.n_, src.corr_.scale_buf_.buf_ptr_, src.corr_.sdtype_, src.info_.npad_,+ dst.corr_.scale_buf_.buf_ptr_, dst.n_block_, threading);+ if (src.corr_.zptr()) {+ assert(dst.corr_.zptr());+ transpose(src.n_block_, src.info_.n_, src.corr_.zp_buf_.buf_ptr_, src.corr_.zdtype_, src.info_.npad_,+ dst.corr_.zp_buf_.buf_ptr_, dst.n_block_, threading);+ }+ if (src.corr_.rptr()) {+ assert(dst.corr_.rptr());+ transpose(src.n_block_, src.info_.n_, src.corr_.red_buf_.buf_ptr_, BTLA_DTYPE::S32, src.info_.npad_,+ dst.corr_.red_buf_.buf_ptr_, dst.n_block_, threading);+ }+ utils::afree(tmp_buf_NT);+ }+ }+ }++ template <typename T>+ AUTOCALL void unpack_weight(const int N, const int K, const Storage& stor, T* B, const int ldb,+ parallel::IThreading* threading) {+ assert(stor.info_.layout_ == storage::gemm::Layout::NPack);+ if (stor.info_.layout_ == storage::gemm::Layout::NPack) {+ parallel::Scheduler2D _para({threading->num_threads(), K, N, _GemmCore_T::KTILE, _GemmCore_T::NTILE});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ auto rowpad = utils::padto(thdp.size[0], _GemmCore_T::KTILE);+ auto colpad = utils::padto(thdp.size[1], _GemmCore_T::NTILE);+ auto dequant = utils::amalloc<T>((size_t)rowpad * colpad);+ auto dstptr = dequant;+ int dststep = 0;+ size_t constexpr CacheSize = size_t(100) << 10;+ int8_t tmpcache[CacheSize];+ GetCPUDevice();+ if (_cd->AVX512F()) {+ get_weight<BTLA_ISA::AVX512F>(&dstptr, &dststep, rowpad, colpad, thdp.loc[0], thdp.loc[1], {&stor},+ tmpcache, CacheSize);+ } else if (_cd->AVX2()) {+ get_weight<BTLA_ISA::AVX2>(&dstptr, &dststep, rowpad, colpad, thdp.loc[0], thdp.loc[1], {&stor}, tmpcache,+ CacheSize);+ } else {+ get_weight<BTLA_ISA::NoSIMD>(&dstptr, &dststep, rowpad, colpad, thdp.loc[0], thdp.loc[1], {&stor}, tmpcache,+ CacheSize);+ }+ kernel::wrapper::RevertPaddingInterleaveMN<_GemmCore_T::NTILE, _GemmCore_T::PACK_ROW, T>::forward_auto(+ dstptr, B + thdp.loc[0] * ldb + thdp.loc[1], thdp.size[0], thdp.size[1], rowpad, colpad, dststep, ldb);+ utils::afree(dequant);+ }+ });+ }+ }++ AUTOCALL void decompress_weight(const int N, const int K, const Storage& stor, int8_t* B, const int ldb,+ parallel::IThreading* threading) {+ if (stor.info_.layout_ == storage::gemm::Layout::T) {+ parallel::Scheduler2D _para({threading->num_threads(), N * K, 1, 512, 1});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ size_t constexpr CacheSize = size_t(100) << 10;+ int8_t tmpcache[CacheSize];+ GetCPUDevice();+ if (stor.info_.dtype_ == BTLA_DTYPE::S4_CLIP) {+ auto srcptr = (utils::int4x2*)stor.buf_.buf_ptr_ + thdp.loc[0] / 2;+ kernel::ref::decompress_s4_s8(srcptr, B + thdp.loc[0], thdp.size[0], tmpcache, CacheSize);+ }+ }+ });+ }+ }++ AUTOCALL void set_correction(const int N, const int K, const int8_t* zero_points, const float* scales,+ const storage::gemm::StorageWeightNInt& stor, parallel::IThreading* threading) {+ int rawnk_scale = utils::updiv(K, stor.block_);+ int nk_scale = stor.n_block_;+ parallel::Scheduler2D _para({threading->num_threads(), 1, nk_scale, 1, 1});+ if (stor.corr_.sdtype_ == BTLA_DTYPE::BF16 || stor.corr_.sdtype_ == BTLA_DTYPE::F16 ||+ stor.corr_.sdtype_ == BTLA_DTYPE::F32) {+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ int rows = thdp.loc[1] + thdp.size[1] <= rawnk_scale ? thdp.size[1] : rawnk_scale - thdp.loc[1];+ if (scales) {+ if (stor.corr_.sdtype_ == BTLA_DTYPE::BF16) {+ kernel::wrapper::Memcpy2DFp32TPadding<utils::bf16>::forward_auto(+ scales + thdp.loc[1] * N,+ (utils::bf16*)stor.corr_.scale_buf_.buf_ptr_ + thdp.loc[1] * stor.info_.npad_, rows, N,+ N * sizeof(scales[0]), stor.info_.npad_ * sizeof(utils::bf16), true);+ } else if (stor.corr_.sdtype_ == BTLA_DTYPE::F32) {+ kernel::wrapper::Memcpy2DFp32TPadding<float>::forward_auto(+ scales + thdp.loc[1] * N, (float*)stor.corr_.scale_buf_.buf_ptr_ + thdp.loc[1] * stor.info_.npad_,+ rows, N, N * sizeof(scales[0]), stor.info_.npad_ * sizeof(float), true);+ } else if (stor.corr_.sdtype_ == BTLA_DTYPE::F16) {+ kernel::wrapper::Memcpy2DFp32TPadding<utils::fp16>::forward_auto(+ scales + thdp.loc[1] * N,+ (utils::fp16*)stor.corr_.scale_buf_.buf_ptr_ + thdp.loc[1] * stor.info_.npad_, rows, N,+ N * sizeof(scales[0]), stor.info_.npad_ * sizeof(utils::fp16), true);+ }+ if (rows < thdp.size[1]) {+ auto sb = bestla::utils::bestla_dtype_bytes(stor.corr_.sdtype_);+ if (sb == 2) {+ std::memset((int16_t*)stor.corr_.scale_buf_.buf_ptr_ + (thdp.loc[1] + rows) * stor.info_.npad_, 0,+ sb * (thdp.size[1] - rows) * stor.info_.npad_);+ } else if (sb == 4) {+ std::memset((float*)stor.corr_.scale_buf_.buf_ptr_ + (thdp.loc[1] + rows) * stor.info_.npad_, 0,+ sb * (thdp.size[1] - rows) * stor.info_.npad_);+ } else {+ assert(0);+ }+ }+ }+ if (zero_points) {+ kernel::wrapper::Memcpy2DPadding::forward(+ zero_points + thdp.loc[1] * N, (int8_t*)stor.corr_.zp_buf_.buf_ptr_ + thdp.loc[1] * stor.info_.npad_,+ rows, N * sizeof(zero_points[0]), N * sizeof(zero_points[0]), sizeof(int8_t) * stor.info_.npad_, true);++ if (rows < thdp.size[1]) {+ std::memset((int8_t*)stor.corr_.zp_buf_.buf_ptr_ + (thdp.loc[1] + rows) * stor.info_.npad_, 0,+ sizeof(int8_t) * (thdp.size[1] - rows) * stor.info_.npad_);+ }+ }+ }+ });+ } else {+ assert(0);+ }+ }++ AUTOCALL void pack_qweight(const int N, const int K, const int8_t* B, const int ldb, const float* scales,+ const int8_t* zero_points, const storage::gemm::StorageWeightNInt& stor,+ parallel::IThreading* threading) {+ set_correction(N, K, zero_points, scales, stor, threading);+ if (stor.info_.dtype_ == BTLA_DTYPE::S8) {+ reorderWeight(N, K, B, ldb, stor.buf_.buf_ptr_, threading);+ } else {+ auto reordered = utils::amalloc<int8_t>((size_t)stor.info_.kpad_ * stor.info_.npad_);+ reorderWeight(N, K, B, ldb, reordered, threading);+ compressWeight(stor.info_.npad_, stor.info_.kpad_, reordered, stor.info_.npad_, stor.buf_.buf_ptr_,+ stor.info_.dtype_, threading);+ utils::afree(reordered);+ }+ reduce_weight(stor, threading);+ }++ AUTOCALL void reduce_weight(const Storage& stor, parallel::IThreading* threading) {+ if (stor.corr_.red_buf_.buf_size_) {+ auto deq = utils::amalloc<int8_t>((size_t)stor.info_.k_ * stor.info_.n_);+ unpack_weight(stor.info_.n_, stor.info_.k_, stor, deq, stor.info_.n_, threading);+ reduce(stor.info_.n_, stor.info_.k_, stor.block_, deq, stor.info_.n_, (int32_t*)stor.corr_.red_buf_.buf_ptr_,+ stor.corr_.step_, threading);+ utils::afree(deq);+ }+ }++ AUTOCALL void quantize_weight(const int N, const int K, const float* B, const int ldb, int8_t* qB, float* scales,+ int8_t* zero_points, int block, BTLA_DTYPE qtype, parallel::IThreading* threading) {+ int bsize = block == -1 ? K : block;+ parallel::Scheduler2D _para({threading->num_threads(), K, N, bsize, 16});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp({tidx});+ _para.getIndex(thdp);+ if (thdp.valid) {+ kernel::wrapper::QuantizeSignIntRowBlock::forward_auto(+ B + thdp.loc[0] * ldb + thdp.loc[1], qB + thdp.loc[0] * N + thdp.loc[1], thdp.size[0], thdp.size[1], ldb, N,+ scales + thdp.loc[0] / bsize * N + thdp.loc[1],+ zero_points == nullptr ? zero_points : zero_points + thdp.loc[0] / bsize * N + thdp.loc[1], block, qtype);+ }+ });+ }++ AUTOCALL void reorderWeight(const int N, const int K, const int8_t* B, const int ldb, int8_t* dstptr,+ parallel::IThreading* threading) {+ int KPad = utils::padto(K, _GemmCore_T::KTILE);+ parallel::Scheduler2D _para({threading->num_threads(), K, N, _GemmCore_T::KTILE, _GemmCore_T::NTILE});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp({tidx});+ _para.getIndex(thdp);+ if (thdp.valid) {+ auto rowpadded = utils::padto(thdp.size[0], _GemmCore_T::KTILE);+ auto colpadded = utils::padto(thdp.size[1], _GemmCore_T::NTILE);+ const auto src = B + thdp.loc[0] * ldb + thdp.loc[1];+ const auto dst = dstptr + thdp.loc[0] * _GemmCore_T::NTILE + thdp.loc[1] * KPad;+ using PaddingInterleaveMNWType =+ kernel::wrapper::PaddingInterleaveMN<_GemmCore_T::NTILE, _GemmCore_T::PACK_ROW, int8_t>;+ auto ret = PaddingInterleaveMNWType::forward_auto( //+ src, dst, thdp.size[0], thdp.size[1], rowpadded, colpadded, ldb, KPad);+ assert(ret == BTLA_CODE::Success);+ (void)ret;+ }+ });+ }++ AUTOCALL void compressBit3Weight(const int N, const int K, const int8_t* B, int8_t* dstptr, BTLA_DTYPE qtype,+ parallel::IThreading* threading) {+ auto bit1_offset = size_t(N) * K;+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(dstptr);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(dstptr + bit1_offset / 4);+ auto ret = kernel::wrapper::CompressBit3::forward_auto(B, bit2ptr, bit1ptr, bit1_offset);+ assert(ret == BTLA_CODE::Success);+ }++ AUTOCALL void compressBit5Weight(const int N, const int K, const int8_t* B, int8_t* dstptr, BTLA_DTYPE qtype,+ parallel::IThreading* threading) {+ auto bit1_offset = size_t(N) * K;+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(dstptr);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(dstptr + bit1_offset / 2);+ auto ret = kernel::wrapper::CompressBit5::forward_auto(B, bit4ptr, bit1ptr, bit1_offset);+ assert(ret == BTLA_CODE::Success);+ }++ AUTOCALL void compressBit6Weight(const int N, const int K, const int8_t* B, int8_t* dstptr, BTLA_DTYPE qtype,+ parallel::IThreading* threading) {+ auto bit2_offset = size_t(N) * K;+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(dstptr);+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(dstptr + bit2_offset / 2);+ auto ret = kernel::wrapper::CompressBit6::forward_auto(B, bit4ptr, bit2ptr, bit2_offset);+ assert(ret == BTLA_CODE::Success);+ }++ AUTOCALL void compressBit7Weight(const int N, const int K, const int8_t* B, int8_t* dstptr, BTLA_DTYPE qtype,+ parallel::IThreading* threading) {+ auto eltsize = size_t(N) * K;+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(dstptr);+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit4ptr + eltsize / 2);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit2ptr + eltsize / 4);+ auto ret = kernel::wrapper::CompressBit7::forward_auto(B, bit4ptr, bit2ptr, bit1ptr, eltsize);+ assert(ret == BTLA_CODE::Success);+ }++ AUTOCALL void compressBit2Weight(const int N, const int K, const int8_t* B, int8_t* dstptr, BTLA_DTYPE qtype,+ parallel::IThreading* threading) {+ // TODO(zhe): 1D parallel compress+ parallel::Scheduler2D _para({threading->num_threads(), 1, K * N, 1, 64});+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(dstptr);+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp({tidx});+ _para.getIndex(thdp);+ if (thdp.valid) {+ auto ret =+ kernel::wrapper::CompressBit2::forward_auto(B + thdp.loc[1], bit2ptr + thdp.loc[1] / 4, thdp.size[1]);+ assert(ret == BTLA_CODE::Success);+ (void)ret;+ }+ });+ }++ AUTOCALL void compressBit1Weight(const int N, const int K, const int8_t* B, int8_t* dstptr, BTLA_DTYPE qtype,+ parallel::IThreading* threading) {+ // TODO(zhe): 1D parallel compress+ parallel::Scheduler2D _para({threading->num_threads(), 1, K * N, 1, 64});+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(dstptr);+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp({tidx});+ _para.getIndex(thdp);+ if (thdp.valid) {+ auto ret =+ kernel::wrapper::CompressBit1::forward_auto(B + thdp.loc[1], bit1ptr + thdp.loc[1] / 8, thdp.size[1]);+ assert(ret == BTLA_CODE::Success);+ (void)ret;+ }+ });+ }++ AUTOCALL void compressBit4Weight(const int N, const int K, const int8_t* B, int8_t* dstptr, BTLA_DTYPE qtype,+ parallel::IThreading* threading) {+ parallel::Scheduler2D _para({threading->num_threads(), 1, K * N, 1, 64});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp({tidx});+ _para.getIndex(thdp);+ if (thdp.valid) {+ BTLA_CODE ret = BTLA_CODE::NotSupport;+ if (qtype == BTLA_DTYPE::S4_CLIP) {+ auto bit4ptr = reinterpret_cast<utils::int4x2*>(dstptr);+ ret = kernel::wrapper::CompressS8S4::forward_auto(B + thdp.loc[1], bit4ptr + thdp.loc[1] / 2, thdp.size[1]);+ } else if (qtype == BTLA_DTYPE::F4_BNB || qtype == BTLA_DTYPE::F4_NF4 || qtype == BTLA_DTYPE::F4_E2M1) {+ auto bit4ptr = reinterpret_cast<utils::f4x2*>(dstptr);+ ret = kernel::wrapper::CompressFp4::forward_auto(B + thdp.loc[1], bit4ptr + thdp.loc[1] / 2, thdp.size[1]);+ } else {+ assert(0);+ }+ assert(ret == BTLA_CODE::Success);+ (void)ret;+ }+ });+ }++ AUTOCALL void compressWeight(const int N, const int K, const int8_t* B, const int ldb, int8_t* dstptr,+ BTLA_DTYPE qtype, parallel::IThreading* threading) {+ if (qtype == BTLA_DTYPE::S7_CLIP) return compressBit7Weight(N, K, B, dstptr, qtype, threading);+ if (qtype == BTLA_DTYPE::S6_CLIP) return compressBit6Weight(N, K, B, dstptr, qtype, threading);+ if (qtype == BTLA_DTYPE::S5_CLIP) return compressBit5Weight(N, K, B, dstptr, qtype, threading);+ if (qtype == BTLA_DTYPE::S4_CLIP) return compressBit4Weight(N, K, B, dstptr, qtype, threading);+ if (qtype == BTLA_DTYPE::S3_CLIP) return compressBit3Weight(N, K, B, dstptr, qtype, threading);+ if (qtype == BTLA_DTYPE::S2_CLIP) return compressBit2Weight(N, K, B, dstptr, qtype, threading);+ if (qtype == BTLA_DTYPE::S1_CLIP) return compressBit1Weight(N, K, B, dstptr, qtype, threading);+ if (qtype == BTLA_DTYPE::F4_BNB || qtype == BTLA_DTYPE::F4_NF4 || qtype == BTLA_DTYPE::F4_E2M1)+ return compressBit4Weight(N, K, B, dstptr, qtype, threading);+ }++ AUTOCALL void reduce(const int N, const int K, const int KBlock, const int8_t* B, const int ldb, int32_t* rptr,+ const int ldr, parallel::IThreading* threading) {+ parallel::Scheduler2D _para({threading->num_threads(), K, N, KBlock, 16});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp({tidx});+ _para.getIndex(thdp);+ if (thdp.valid) {+ const auto src = B + thdp.loc[0] * ldb + thdp.loc[1];+ const auto dst = rptr + thdp.loc[1] + thdp.loc[0] / KBlock * ldr;+ using RowReduceSum = kernel::wrapper::RowReduceSum<int8_t, int32_t>;+ for (int i = 0; i < thdp.size[0]; i += KBlock) {+ int rowremain = utils::remainsize(thdp.loc[0] + i, K, KBlock);+ auto ret = RowReduceSum::forward_auto(src + i * ldb, ldb, rowremain, thdp.size[1], dst + i / KBlock * ldr);+ assert(ret == BTLA_CODE::Success);+ (void)ret;+ }+ }+ });+ }++ public:+ template <BTLA_ISA ISA_T, typename _T>+ static inline BTLA_CODE get_weight(_T** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ if constexpr (std::is_same_v<_T, int8_t>) {+ return get_int<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ }+ return get_fp<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ }++ TLACALL BTLA_CODE get_int(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.stor_;+ if (wptr->info_.dtype_ == BTLA_DTYPE::S8) {+ return getQ8Weight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ } else if (wptr->info_.dtype_ == BTLA_DTYPE::S6_CLIP) {+ return getQ6Weight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ } else if (wptr->info_.dtype_ == BTLA_DTYPE::S5_CLIP) {+ return getQ5Weight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ } else if (wptr->info_.dtype_ == BTLA_DTYPE::S4_CLIP) {+ return getQ4Weight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ } else if (wptr->info_.dtype_ == BTLA_DTYPE::S3_CLIP) {+ return getQ3Weight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ } else if (wptr->info_.dtype_ == BTLA_DTYPE::S2_CLIP) {+ return getQ2Weight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ } else if (wptr->info_.dtype_ == BTLA_DTYPE::S7_CLIP) {+ return getQ7Weight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ } else if (wptr->info_.dtype_ == BTLA_DTYPE::S1_CLIP) {+ return getQ1Weight<ISA_T>(dstptr, dststep, k_size, n_size, k_offset, n_offset, _param, tmpcache, cachesize);+ } else {+ assert(0);+ }+ return BTLA_CODE::NotSupport;+ }++ TLACALL BTLA_CODE get_scale(float** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.stor_;+ if (wptr->corr_.sdtype_ == BTLA_DTYPE::F32) {+ auto aptr = (float*)wptr->corr_.scale_buf_.buf_ptr_;+ kernel::wrapper::Memcpy2D::template forward<BTLA_ISA::NoSIMD>(+ aptr + k_offset / wptr->block_ * wptr->corr_.step_ + n_offset, *dstptr, utils::updiv(k_size, wptr->block_),+ n_size, wptr->corr_.step_, n_size);+ *dststep = n_size;+ }+ if (wptr->corr_.sdtype_ == BTLA_DTYPE::BF16) {+ auto aptr = (utils::bf16*)wptr->corr_.scale_buf_.buf_ptr_;+ kernel::wrapper::Memcpy2DBf16CvtFp32::forward<ISA_T>(+ aptr + k_offset / wptr->block_ * wptr->corr_.step_ + n_offset, *dstptr, utils::updiv(k_size, wptr->block_),+ n_size, wptr->corr_.step_ * 2, n_size * 4, false);+ *dststep = n_size;+ }+ if (wptr->corr_.sdtype_ == BTLA_DTYPE::F16) {+ auto aptr = (utils::fp16*)wptr->corr_.scale_buf_.buf_ptr_;+ kernel::wrapper::Memcpy2DFp16CvtFp32::forward<ISA_T>(+ aptr + k_offset / wptr->block_ * wptr->corr_.step_ + n_offset, *dstptr, utils::updiv(k_size, wptr->block_),+ n_size, wptr->corr_.step_ * 2, n_size * 4, false);+ *dststep = n_size;+ }+ /*if (wptr->corr_.sdtype_ == BTLA_DTYPE::DQ8_BNB) {+ auto aptr = wptr->template SPtr<uint8_t>();+ auto internal_k_offset = k_offset / wptr->block_;+ auto dq_offset_idx = static_cast<int>(wptr->mCorrection.mDQCorrectionBuf.mBufSize / sizeof(float) - 1);+ kernel::wrapper::Dq8GetScale::template forward<ISA_T>(+ aptr + internal_k_offset * wptr->corr_.step_ + n_offset, *dstptr, utils::updiv(k_size, wptr->block_), n_size,+ internal_k_offset * wptr->mN + n_offset, wptr->mDqBlockSize, dq_offset_idx, wptr->DQPtr<float>(),+ wptr->corr_.step_, n_size, false, wptr->mN);+ }*/+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE get_reduce(int32_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.stor_;+ auto rptr = (int32_t*)wptr->corr_.red_buf_.buf_ptr_;+ kernel::wrapper::Memcpy2D::template forward<BTLA_ISA::NoSIMD>(+ rptr + k_offset / wptr->block_ * wptr->corr_.step_ + n_offset, *dstptr, utils::updiv(k_size, wptr->block_),+ n_size, wptr->corr_.step_, n_size);+ *dststep = n_size;+ return BTLA_CODE::Success;+ }++ protected:+ template <BTLA_ISA ISA_T, typename _T>+ static inline BTLA_CODE get_fp(_T** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.stor_;+ auto NPad = wptr->info_.npad_;+ auto KPad = wptr->info_.kpad_;+ int constexpr ColSize = _GemmCore_T::NTILE * _GemmCore_T::PACK_ROW;+ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ auto zptr = wptr->corr_.zptr();+ auto sptr = wptr->corr_.scale_buf_.buf_ptr_;+ if (wptr->info_.dtype_ == BTLA_DTYPE::S4_CLIP) {+ kernel::wrapper::DecompressKBlockS4Fp<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE, _T>::template forward<ISA_T>(+ (utils::int4x2*)wptr->buf_.buf_ptr_ + n_offset * KPad / 2 + k_offset * _GemmCore_T::NTILE / 2 ++ i * KPad / 2,+ *dstptr + i * k_size, k_size, _GemmCore_T::NTILE, sptr, wptr->corr_.sdtype_, zptr, k_offset, n_offset + i,+ wptr->block_, NPad, tmpcache, cachesize);++ } else if (wptr->info_.dtype_ == BTLA_DTYPE::S3_CLIP) {+ int8_t* bit3_ptr = wptr->buf_.buf_ptr_;+ auto elt_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE + i * KPad;+ assert(elt_offset % 8 == 0);+ size_t bit1_offset = size_t(NPad) * KPad;+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit3_ptr + elt_offset / 4);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit3_ptr + bit1_offset / 4 + elt_offset / 8);+ kernel::wrapper::DecompressKBlockS3Fp<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE, _T>::template forward<ISA_T>(+ bit2ptr, bit1ptr, *dstptr + i * k_size, k_size, _GemmCore_T::NTILE, sptr, wptr->corr_.sdtype_, zptr,+ k_offset, n_offset + i, wptr->block_, NPad, tmpcache, cachesize);+ } else if (wptr->info_.dtype_ == BTLA_DTYPE::S2_CLIP) {+ int8_t* bit2_ptr = wptr->buf_.buf_ptr_;+ auto elt_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE + i * KPad;+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit2_ptr + elt_offset / 4);+ kernel::wrapper::DecompressKBlockS2Fp<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE, _T>::template forward<ISA_T>(+ bit2ptr, *dstptr + i * k_size, k_size, _GemmCore_T::NTILE, sptr, wptr->corr_.sdtype_, zptr, k_offset,+ n_offset + i, wptr->block_, NPad, tmpcache, cachesize);+ } else if (wptr->info_.dtype_ == BTLA_DTYPE::S8) {+ auto elt_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE + i * KPad;+ int8_t* bptr = wptr->buf_.buf_ptr_ + elt_offset;+ kernel::wrapper::DecompressKBlockS8Fp<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE, _T>::template forward<ISA_T>(+ bptr, *dstptr + i * k_size, k_size, _GemmCore_T::NTILE, sptr, wptr->corr_.sdtype_, zptr, k_offset,+ n_offset + i, wptr->block_, NPad, tmpcache, cachesize);+ } else if (wptr->info_.dtype_ == BTLA_DTYPE::S5_CLIP) {+ int8_t* bit5_ptr = wptr->buf_.buf_ptr_;+ auto elt_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE + i * KPad;+ assert(elt_offset % 8 == 0);+ size_t bit1_offset = size_t(NPad) * KPad;+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(bit5_ptr + elt_offset / 2);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit5_ptr + bit1_offset / 2 + elt_offset / 8);+ kernel::wrapper::DecompressKBlockS5Fp<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE, _T>::template forward<ISA_T>(+ bit4ptr, bit1ptr, *dstptr + i * k_size, k_size, _GemmCore_T::NTILE, sptr, wptr->corr_.sdtype_, zptr,+ k_offset, n_offset + i, wptr->block_, NPad, tmpcache, cachesize);+ } else if (wptr->info_.dtype_ == BTLA_DTYPE::S6_CLIP) {+ int8_t* bit6_ptr = wptr->buf_.buf_ptr_;+ auto elt_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE + i * KPad;+ assert(elt_offset % 4 == 0);+ size_t bit2_offset = size_t(NPad) * KPad;+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(bit6_ptr + elt_offset / 2);+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit6_ptr + bit2_offset / 2 + elt_offset / 4);+ kernel::wrapper::DecompressKBlockS6Fp<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE, _T>::template forward<ISA_T>(+ bit4ptr, bit2ptr, *dstptr + i * k_size, k_size, _GemmCore_T::NTILE, sptr, wptr->corr_.sdtype_, zptr,+ k_offset, n_offset + i, wptr->block_, NPad, tmpcache, cachesize);+ } else if (wptr->info_.dtype_ == BTLA_DTYPE::S7_CLIP) {+ int8_t* bit7_ptr = wptr->buf_.buf_ptr_;+ auto elt_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE + i * KPad;+ assert(elt_offset % 8 == 0);+ size_t bit2_offset = size_t(NPad) * KPad;+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(bit7_ptr + elt_offset / 2);+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit7_ptr + bit2_offset / 2 + elt_offset / 4);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit7_ptr + bit2_offset / 2 + bit2_offset / 4 + elt_offset / 8);+ kernel::wrapper::DecompressKBlockS7Fp<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE, _T>::template forward<ISA_T>(+ bit4ptr, bit2ptr, bit1ptr, *dstptr + i * k_size, k_size, _GemmCore_T::NTILE, sptr, wptr->corr_.sdtype_,+ zptr, k_offset, n_offset + i, wptr->block_, NPad, tmpcache, cachesize);+ } else if (wptr->info_.dtype_ == BTLA_DTYPE::S1_CLIP) {+ int8_t* bit1_ptr = wptr->buf_.buf_ptr_;+ auto elt_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE + i * KPad;+ assert(elt_offset % 8 == 0);+ size_t bit1_offset = size_t(NPad) * KPad;+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit1_ptr + elt_offset / 8);+ kernel::wrapper::DecompressKBlockS1Fp<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE, _T>::template forward<ISA_T>(+ bit1ptr, *dstptr + i * k_size, k_size, _GemmCore_T::NTILE, sptr, wptr->corr_.sdtype_, zptr, k_offset,+ n_offset + i, wptr->block_, NPad, tmpcache, cachesize);+ } else {+ assert(0);+ }+ }+ *dststep = k_size;+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getQ8Weight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.stor_;+ auto KPad = wptr->info_.kpad_;+ auto bptr = wptr->buf_.buf_ptr_ + n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ kernel::wrapper::Memcpy2D::template forward<BTLA_ISA::NoSIMD, int8_t, int8_t>(+ bptr, *dstptr, n_size / _GemmCore_T::NTILE, _GemmCore_T::NTILE * k_size, _GemmCore_T::NTILE * KPad,+ _GemmCore_T::NTILE * k_size);+ *dststep = k_size;+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getQ4Weight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.stor_;+ auto KPad = wptr->info_.kpad_;+ using storage::gemm::Layout;+ if (wptr->info_.layout_ == Layout::NPack) {+ auto bptr = (utils::int4x2*)wptr->buf_.buf_ptr_ + n_offset * KPad / 2 + k_offset * _GemmCore_T::NTILE / 2;+ auto zpptr = wptr->corr_.zptr();+ int constexpr ColSize = _GemmCore_T::NTILE * _GemmCore_T::PACK_ROW;+ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ kernel::wrapper::DecompressKBlockS4S8<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE>::template forward<ISA_T>(+ bptr + i * KPad / 2, zpptr, *dstptr + i * k_size, wptr->block_, wptr->corr_.step_, n_offset + i, k_offset,+ k_size, _GemmCore_T::NTILE, tmpcache, cachesize);+ }+ *dststep = k_size;+ }+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getQ3Weight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.stor_;+ int8_t* bit3_ptr = wptr->buf_.buf_ptr_;+ auto zpptr = wptr->corr_.zptr();+ auto KPad = wptr->info_.kpad_;+ auto NPad = wptr->info_.npad_;+ size_t bit1_offset = size_t(NPad) * KPad;+ auto base_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ auto elt_offset = base_offset + i * KPad;+ assert(elt_offset % 8 == 0);+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit3_ptr + elt_offset / 4);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit3_ptr + bit1_offset / 4 + elt_offset / 8);+ kernel::wrapper::DecompressKBlockS3S8<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE>::template forward<ISA_T>(+ bit2ptr, bit1ptr, zpptr, *dstptr + i * k_size, wptr->block_, wptr->corr_.step_, n_offset + i, k_offset,+ k_size, _GemmCore_T::NTILE, tmpcache, cachesize);+ }+ *dststep = k_size;+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getQ5Weight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.stor_;+ int8_t* bit5_ptr = wptr->buf_.buf_ptr_;+ auto zpptr = wptr->corr_.zptr();+ auto KPad = wptr->info_.kpad_;+ auto NPad = wptr->info_.npad_;+ size_t bit1_offset = size_t(NPad) * KPad;+ auto base_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ auto elt_offset = base_offset + i * KPad;+ assert(elt_offset % 8 == 0);+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(bit5_ptr + elt_offset / 2);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit5_ptr + bit1_offset / 2 + elt_offset / 8);+ kernel::wrapper::DecompressKBlockS5S8<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE>::template forward<ISA_T>(+ bit4ptr, bit1ptr, zpptr, *dstptr + i * k_size, wptr->block_, wptr->corr_.step_, n_offset + i, k_offset,+ k_size, _GemmCore_T::NTILE, tmpcache, cachesize);+ }+ *dststep = k_size;+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getQ6Weight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.stor_;+ int8_t* bit6_ptr = wptr->buf_.buf_ptr_;+ auto zpptr = wptr->corr_.zptr();+ auto KPad = wptr->info_.kpad_;+ auto NPad = wptr->info_.npad_;+ size_t bit2_offset = size_t(NPad) * KPad;+ auto base_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ auto elt_offset = base_offset + i * KPad;+ assert(elt_offset % 4 == 0);+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(bit6_ptr + elt_offset / 2);+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit6_ptr + bit2_offset / 2 + elt_offset / 4);+ kernel::wrapper::DecompressKBlockS6S8<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE>::template forward<ISA_T>(+ bit4ptr, bit2ptr, zpptr, *dstptr + i * k_size, wptr->block_, wptr->corr_.step_, n_offset + i, k_offset,+ k_size, _GemmCore_T::NTILE, tmpcache, cachesize);+ }+ *dststep = k_size;+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getQ7Weight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.stor_;+ int8_t* bit6_ptr = wptr->buf_.buf_ptr_;+ auto zpptr = wptr->corr_.zptr();+ auto KPad = wptr->info_.kpad_;+ auto NPad = wptr->info_.npad_;+ size_t bit2_offset = size_t(NPad) * KPad;+ auto base_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ auto elt_offset = base_offset + i * KPad;+ assert(elt_offset % 8 == 0);+ auto bit4ptr = reinterpret_cast<utils::bit4x2*>(bit6_ptr + elt_offset / 2);+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit6_ptr + bit2_offset / 2 + elt_offset / 4);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit6_ptr + bit2_offset / 2 + bit2_offset / 4 + elt_offset / 8);+ kernel::wrapper::DecompressKBlockS7S8<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE>::template forward<ISA_T>(+ bit4ptr, bit2ptr, bit1ptr, zpptr, *dstptr + i * k_size, wptr->block_, wptr->corr_.step_, n_offset + i,+ k_offset, k_size, _GemmCore_T::NTILE, tmpcache, cachesize);+ }+ *dststep = k_size;+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getQ2Weight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.stor_;+ int8_t* bit2_ptr = wptr->buf_.buf_ptr_;+ int8_t* zpptr = wptr->corr_.zptr();+ auto KPad = wptr->info_.kpad_;+ auto NPad = wptr->info_.npad_;+ auto base_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ auto elt_offset = base_offset + i * KPad;+ assert(elt_offset % 4 == 0);+ auto bit2ptr = reinterpret_cast<utils::bit2x4*>(bit2_ptr + elt_offset / 4);+ kernel::wrapper::DecompressKBlockS2S8<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE>::template forward<ISA_T>(+ bit2ptr, zpptr, *dstptr + i * k_size, wptr->block_, wptr->corr_.step_, n_offset + i, k_offset, k_size,+ _GemmCore_T::NTILE, tmpcache, cachesize);+ }+ *dststep = k_size;+ return BTLA_CODE::Success;+ }++ TLACALL BTLA_CODE getQ1Weight(int8_t** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = _param.stor_;+ int8_t* bit1_ptr = wptr->buf_.buf_ptr_;+ int8_t* zpptr = wptr->corr_.zptr();+ auto KPad = wptr->info_.kpad_;+ auto NPad = wptr->info_.npad_;+ auto base_offset = n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ auto elt_offset = base_offset + i * KPad;+ assert(elt_offset % 8 == 0);+ auto bit1ptr = reinterpret_cast<utils::bit1x8*>(bit1_ptr + elt_offset / 8);+ kernel::wrapper::DecompressKBlockS1S8<_GemmCore_T::PACK_ROW, _GemmCore_T::NTILE>::template forward<ISA_T>(+ bit1ptr, zpptr, *dstptr + i * k_size, wptr->block_, wptr->corr_.step_, n_offset + i, k_offset, k_size,+ _GemmCore_T::NTILE, tmpcache, cachesize);+ }+ *dststep = k_size;+ return BTLA_CODE::Success;+ }+};++struct ParamWeightKBlockNFloat {+ storage::gemm::StorageWeightKBlockNFloat* packedW;+};++template <class _GemmCore_T>+class WeightKBlockNFloat {+ public:+ using Param = ParamWeightKBlockNFloat; // NFloat storage Param same with NInteger storage.+ using StorageWeight = storage::gemm::StorageWeightKBlockNFloat;+ using QuantBaseT = WeightKBlockNInteger<_GemmCore_T>;+ AUTOCALL StorageWeight createStorage(const int N, const int K, int blocksize, BTLA_DTYPE fT, BTLA_DTYPE scaT) {+ int KPad = utils::padto(K, _GemmCore_T::KTILE);+ int NPad = utils::padto(N, _GemmCore_T::NTILE);+ StorageWeight tmp(_GemmCore_T::ID);+ tmp.resize(NPad, KPad, blocksize <= 0 ? KPad : blocksize, N, K, fT, scaT);+ return tmp;+ }++ AUTOCALL void packTransposeWeight(const int N, const int K, const float* B, const int ldb, StorageWeight* stor,+ parallel::IThreading* threading) {+ auto B_NT = utils::amalloc<float>(static_cast<size_t>(N) * K);+ transposeWeight<float>(N, K, B, ldb, B_NT, N, threading);+ packWeight(N, K, B_NT, N, stor, threading);+ utils::afree(B_NT);+ }++ // from KxN f32 weight to packed N//NtilexKPadxNTile int8 weight+ AUTOCALL void packWeight(const int N, const int K, const float* B, const int ldb, StorageWeight* ptr,+ parallel::IThreading* threading) {+ auto tmpq = utils::amalloc<int8_t>(static_cast<size_t>(N) * K);+ int nk_scale = utils::updiv(K, ptr->mBlockSize);+ auto ssize = static_cast<size_t>(N) * nk_scale;+ auto Tscales = utils::amalloc<float>(ssize);+ auto Tzps = utils::amalloc<int8_t>(ptr->IsAsym() ? ssize : 0);+ quantizeWeight(N, K, B, ldb, tmpq, Tscales, Tzps, ptr, threading);+ packQWeight(N, K, tmpq, N, Tscales, Tzps, ptr, threading);+ utils::afree(tmpq);+ utils::afree(Tscales);+ utils::afree(Tzps);+ }++ AUTOCALL void quantizeWeight(const int N, const int K, const float* B, const int ldb, int8_t* qB, float* scales,+ int8_t* zero_points, void* stor, parallel::IThreading* threading) {+ auto ptr = reinterpret_cast<StorageWeight*>(stor);+ int bsize = ptr->mBlockSize == -1 ? K : ptr->mBlockSize;+ parallel::Scheduler2D _para({threading->num_threads(), K, N, bsize, 16});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp({tidx});+ _para.getIndex(thdp);+ if (thdp.valid) {+ quantRowBlock(B + thdp.loc[0] * ldb + thdp.loc[1], qB + thdp.loc[0] * N + thdp.loc[1], thdp.size[0],+ thdp.size[1], ldb, N, scales + thdp.loc[0] / bsize * N + thdp.loc[1], ptr);+ }+ });+ }++ // from packed N//NtilexKPadxNTile int8 weight to KxN f32 weight+ AUTOCALL void unpackTransposeWeight(const int N, const int K, StorageWeight* stor, float* B, const int ldb,+ parallel::IThreading* threading) {+ auto B_NT = utils::amalloc<float>(static_cast<size_t>(N) * K);+ unpackWeight(N, K, stor, B_NT, N, threading);+ transposeWeight<float>(K, N, B_NT, N, B, ldb, threading);+ utils::afree(B_NT);+ }++ AUTOCALL void unpackWeight(const int N, const int K, StorageWeight* stor, float* B, const int ldb,+ parallel::IThreading* threading) {+ parallel::Scheduler2D _para({threading->num_threads(), K, N, _GemmCore_T::KTILE, _GemmCore_T::NTILE});+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ auto rowpad = utils::padto(thdp.size[0], _GemmCore_T::KTILE);+ auto colpad = utils::padto(thdp.size[1], _GemmCore_T::NTILE);+ auto dequant = utils::amalloc<float>((size_t)rowpad * colpad);+ auto dstptr = dequant;+ int dststep = 0;+ size_t constexpr CacheSize = size_t(100) << 10;+ int8_t tmpcache[CacheSize];+ GetCPUDevice();+ if (_cd->AVX512F()) {+ getWeight<BTLA_ISA::AVX512F>(&dstptr, &dststep, rowpad, colpad, thdp.loc[0], thdp.loc[1], {stor}, tmpcache,+ CacheSize);+ } else if (_cd->AVX2()) {+ getWeight<BTLA_ISA::AVX2>(&dstptr, &dststep, rowpad, colpad, thdp.loc[0], thdp.loc[1], {stor}, tmpcache,+ CacheSize);+ } else {+ getWeight<BTLA_ISA::NoSIMD>(&dstptr, &dststep, rowpad, colpad, thdp.loc[0], thdp.loc[1], {stor}, tmpcache,+ CacheSize);+ }+ kernel::wrapper::RevertPaddingInterleaveMN<_GemmCore_T::NTILE, _GemmCore_T::PACK_ROW, float>::forward_auto(+ dstptr, B + thdp.loc[0] * ldb + thdp.loc[1], thdp.size[0], thdp.size[1], rowpad, colpad, dststep, ldb);+ utils::afree(dequant);+ }+ });+ }++ AUTOCALL void setDoubleQuantCorrection(utils::avector<float>* dq_buf, StorageWeight* ptr) {+ if (ptr->SDtype() == BTLA_DTYPE::DQ8_BNB) {+ auto packw_dqbuf_ptr = ptr->DQPtr<float>();+ memcpy(packw_dqbuf_ptr, dq_buf->data(), dq_buf->size() * sizeof(float));+ } else {+ assert(0);+ }+ }++ AUTOCALL void packQWeight(const int N, const int K, const int8_t* B, const int ldb, const float* scales,+ const int8_t* zero_points, StorageWeight* stor, parallel::IThreading* threading) {+ if (stor->IsDoubleQuant()) {+ int nk_scale = utils::updiv(K, stor->mBlockSize);+ auto ssize = static_cast<size_t>(N) * nk_scale;+ utils::avector<float> dq_buf;+ QuantBaseT::doubleQuantScale(const_cast<float*>(scales), ssize, stor->mDqBlockSize, stor->SDtype(), &dq_buf);+ setDoubleQuantCorrection(&dq_buf, stor);+ }+ setQuantCorrection(N, K, zero_points, scales, stor, threading);+ if (stor->mDType == BTLA_DTYPE::F8_E4M3 || stor->mDType == BTLA_DTYPE::F8_E5M2) {+ QuantBaseT::reorderWeight(N, K, B, ldb, stor->WPtr<int8_t>(), threading);+ } else {+ auto reordered = utils::amalloc<int8_t>((size_t)stor->mKPad * stor->mNPad);+ QuantBaseT::reorderWeight(N, K, B, ldb, reordered, threading);+ QuantBaseT::compressWeight(stor->mNPad, stor->mKPad, reordered, stor->mNPad, stor->WPtr<int8_t>(), stor->mDType,+ threading);+ utils::afree(reordered);+ }+ }++ AUTOCALL void setQuantCorrection(const int N, const int K, const int8_t* zero_points, const float* scales,+ StorageWeight* stor, parallel::IThreading* threading) {+ int rawnk_scale = utils::updiv(K, stor->mBlockSize);+ int nk_scale = utils::updiv(stor->mKPad, stor->mBlockSize);+ parallel::Scheduler2D _para({threading->num_threads(), 1, nk_scale, 1, 1});+ if (stor->SDtype() == BTLA_DTYPE::F32) { // fp32 to fp32 direct copy+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ for (int i = thdp.loc[1]; i < thdp.loc[1] + thdp.size[1]; i++) {+ if (i < rawnk_scale) {+ if (scales != nullptr)+ std::memcpy(stor->template SPtr<float>() + i * stor->mNPad, scales + i * N, N * sizeof(scales[0]));+ if (zero_points != nullptr)+ std::memcpy(stor->template ZPtr<int8_t>() + i * stor->mNPad, zero_points + i * N,+ N * sizeof(zero_points[0]));+ } else {+ if (scales != nullptr)+ std::memset(stor->template SPtr<float>() + i * stor->mNPad, 0, stor->mNPad * sizeof(float));+ if (zero_points != nullptr)+ std::memset(stor->template ZPtr<int8_t>() + i * stor->mNPad, 0, stor->mNPad * sizeof(zero_points[0]));+ }+ }+ }+ });+ } else if (stor->SDtype() == BTLA_DTYPE::BF16) {+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ for (int i = thdp.loc[1]; i < thdp.loc[1] + thdp.size[1]; i++) {+ if (i < rawnk_scale) {+ if (scales != nullptr) {+ for (int j = 0; j < N; j++) {+ stor->template SPtr<utils::bf16>()[j + i * stor->mNPad] = static_cast<utils::bf16>(scales[i * N + j]);+ }+ }+ if (zero_points != nullptr) {+ std::memcpy(stor->template ZPtr<int8_t>() + i * stor->mNPad, zero_points + i * N,+ N * sizeof(zero_points[0]));+ }+ } else {+ if (scales != nullptr)+ std::memset(stor->template SPtr<utils::bf16>() + i * stor->mNPad, 0, stor->mNPad * sizeof(utils::bf16));+ if (zero_points != nullptr)+ std::memset(stor->template ZPtr<int8_t>() + i * stor->mNPad, 0, stor->mNPad * sizeof(zero_points[0]));+ }+ }+ }+ });+ } else if (stor->SDtype() == BTLA_DTYPE::F8_E8M0) {+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ for (int i = thdp.loc[1]; i < thdp.loc[1] + thdp.size[1]; i++) {+ if (i < rawnk_scale) {+ if (scales != nullptr) {+ for (int j = 0; j < N; j++) {+ stor->template SPtr<utils::f8>()[j + i * stor->mNPad] = static_cast<int8_t>(scales[i * N + j]);+ }+ }+ } else {+ if (scales != nullptr)+ std::memset(stor->template SPtr<utils::f8>() + i * stor->mNPad, 0, stor->mNPad * sizeof(utils::f8));+ }+ }+ }+ });+ } else if (stor->SDtype() == BTLA_DTYPE::DQ8_BNB) {+ threading->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ for (int i = thdp.loc[1]; i < thdp.loc[1] + thdp.size[1]; i++) {+ if (i < rawnk_scale) {+ if (scales != nullptr) {+ for (int j = 0; j < N; j++) {+ stor->template SPtr<uint8_t>()[j + i * stor->mNPad] = static_cast<uint8_t>(scales[i * N + j]);+ }+ }+ } else {+ if (scales != nullptr)+ std::memset(stor->template SPtr<uint8_t>() + i * stor->mNPad, 0, stor->mNPad * sizeof(uint8_t));+ }+ }+ }+ });+ } else {+ assert(0);+ }+ }++ template <BTLA_ISA ISA_T, typename _T>+ static inline BTLA_CODE getWeight(_T** dstptr, int* dststep, int k_size, int n_size, int k_offset, int n_offset,+ const Param& _param, void* tmpcache, size_t cachesize) {+ auto wptr = reinterpret_cast<StorageWeight*>(_param.packedW);+ auto NPad = wptr->mNPad;+ auto KPad = wptr->mKPad;+ char* bptr;+ if (wptr->mDType == BTLA_DTYPE::F8_E5M2 || wptr->mDType == BTLA_DTYPE::F8_E4M3) {+ bptr = wptr->template WPtr<char>() + n_offset * KPad + k_offset * _GemmCore_T::NTILE;+ } else {+ bptr = wptr->template WPtr<char>() + n_offset * KPad / 2 + k_offset * _GemmCore_T::NTILE / 2;+ }+ int constexpr ColSize = _GemmCore_T::NTILE * _GemmCore_T::PACK_ROW;+ for (int i = 0; i < n_size; i += _GemmCore_T::NTILE) {+ if (wptr->SDtype() == BTLA_DTYPE::F8_E8M0) {+ assert(wptr->mDType == BTLA_DTYPE::F8_E4M3 || wptr->mDType == BTLA_DTYPE::F8_E5M2);+ auto sptr = wptr->template SPtr<utils::f8>() + n_offset + i;+ kernel::wrapper::DecompressKBlockF8FP<_GemmCore_T::PACK_ROW>::template forward<ISA_T>(+ reinterpret_cast<utils::f8*>(bptr) + i * KPad, *dstptr + i * k_size, k_size / _GemmCore_T::PACK_ROW,+ ColSize, ColSize, ColSize, sptr, k_offset / _GemmCore_T::PACK_ROW, wptr->mBlockSize / _GemmCore_T::PACK_ROW,+ NPad, wptr->mDType);+ } else if (wptr->SDtype() == BTLA_DTYPE::F32) {+ auto sptr = wptr->template SPtr<float>() + n_offset + i;+ auto f4ptr = reinterpret_cast<utils::f4x2*>(bptr + i * KPad / 2);+ auto fp_ptr = *dstptr + i * k_size;+ if (wptr->mDType == BTLA_DTYPE::F8_E4M3 || wptr->mDType == BTLA_DTYPE::F8_E5M2) {+ kernel::wrapper::DecompressKBlockF8FP<_GemmCore_T::PACK_ROW>::template forward<ISA_T>(+ reinterpret_cast<utils::f8*>(bptr) + i * KPad, *dstptr + i * k_size, k_size / _GemmCore_T::PACK_ROW,+ ColSize, ColSize, ColSize, sptr, k_offset / _GemmCore_T::PACK_ROW,+ wptr->mBlockSize / _GemmCore_T::PACK_ROW, NPad, wptr->mDType);+ } else if (wptr->mDType == BTLA_DTYPE::F4_NF4) {+ kernel::wrapper::DecompressKBlockF4Fp<_T, _GemmCore_T::PACK_ROW>::template forward<ISA_T, float,+ BTLA_DTYPE::F4_NF4>(+ f4ptr, fp_ptr, k_size / _GemmCore_T::PACK_ROW, ColSize, ColSize, ColSize, sptr,+ k_offset / _GemmCore_T::PACK_ROW, wptr->mBlockSize / _GemmCore_T::PACK_ROW, NPad, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::F4_E2M1) {+ kernel::wrapper::DecompressKBlockF4Fp<_T, _GemmCore_T::PACK_ROW>::template forward<ISA_T, float,+ BTLA_DTYPE::F4_E2M1>(+ f4ptr, fp_ptr, k_size / _GemmCore_T::PACK_ROW, ColSize, ColSize, ColSize, sptr,+ k_offset / _GemmCore_T::PACK_ROW, wptr->mBlockSize / _GemmCore_T::PACK_ROW, NPad, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::F4_BNB) {+ kernel::wrapper::DecompressKBlockF4Fp<_T, _GemmCore_T::PACK_ROW>::template forward<ISA_T, float,+ BTLA_DTYPE::F4_BNB>(+ f4ptr, fp_ptr, k_size / _GemmCore_T::PACK_ROW, ColSize, ColSize, ColSize, sptr,+ k_offset / _GemmCore_T::PACK_ROW, wptr->mBlockSize / _GemmCore_T::PACK_ROW, NPad, tmpcache, cachesize);+ } else {+ assert(0);+ }+ } else if (wptr->SDtype() == BTLA_DTYPE::BF16) {+ auto sptr = wptr->template SPtr<utils::bf16>() + n_offset + i;+ auto f4ptr = reinterpret_cast<utils::f4x2*>(bptr + i * KPad / 2);+ auto fp_ptr = *dstptr + i * k_size;+ if (wptr->mDType == BTLA_DTYPE::F4_NF4) {+ kernel::wrapper::DecompressKBlockF4Fp<_T, _GemmCore_T::PACK_ROW>::template forward<ISA_T, utils::bf16,+ BTLA_DTYPE::F4_NF4>(+ f4ptr, fp_ptr, k_size / _GemmCore_T::PACK_ROW, ColSize, ColSize, ColSize, sptr,+ k_offset / _GemmCore_T::PACK_ROW, wptr->mBlockSize / _GemmCore_T::PACK_ROW, NPad, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::F4_E2M1) {+ kernel::wrapper::DecompressKBlockF4Fp<_T, _GemmCore_T::PACK_ROW>::template forward<ISA_T, utils::bf16,+ BTLA_DTYPE::F4_E2M1>(+ f4ptr, fp_ptr, k_size / _GemmCore_T::PACK_ROW, ColSize, ColSize, ColSize, sptr,+ k_offset / _GemmCore_T::PACK_ROW, wptr->mBlockSize / _GemmCore_T::PACK_ROW, NPad, tmpcache, cachesize);+ } else if (wptr->mDType == BTLA_DTYPE::F4_BNB) {+ kernel::wrapper::DecompressKBlockF4Fp<_T, _GemmCore_T::PACK_ROW>::template forward<ISA_T, utils::bf16,+ BTLA_DTYPE::F4_BNB>(+ f4ptr, fp_ptr, k_size / _GemmCore_T::PACK_ROW, ColSize, ColSize, ColSize, sptr,+ k_offset / _GemmCore_T::PACK_ROW, wptr->mBlockSize / _GemmCore_T::PACK_ROW, NPad, tmpcache, cachesize);+ } else {+ assert(0);+ }+ } else if (wptr->SDtype() == BTLA_DTYPE::DQ8_BNB) {+ auto f4ptr = reinterpret_cast<utils::f4x2*>(bptr + i * KPad / 2);+ auto fp_ptr = *dstptr + i * k_size;+ auto internal_n_offset = n_offset + i;+ auto internal_k_offset = k_offset / _GemmCore_T::PACK_ROW;+ auto internal_kblock = wptr->mBlockSize / _GemmCore_T::PACK_ROW;+ auto dq_offset_idx = static_cast<int>(wptr->mCorrection.mDQCorrectionBuf.mBufSize / sizeof(float) - 1);+ if (wptr->mDType == BTLA_DTYPE::F4_NF4) {+ kernel::wrapper::DecompressDqKBlockF4Fp<_T, _GemmCore_T::PACK_ROW>::template forward<ISA_T,+ BTLA_DTYPE::F4_NF4>(+ f4ptr, fp_ptr, k_size / _GemmCore_T::PACK_ROW, ColSize, ColSize, ColSize, wptr->template SPtr<uint8_t>(),+ wptr->template DQPtr<float>(), internal_k_offset, internal_n_offset, internal_kblock, wptr->mDqBlockSize,+ dq_offset_idx, NPad, wptr->mN, tmpcache, cachesize);+ } else {+ assert(0);+ }+ } else {+ assert(0);+ }+ }+ *dststep = k_size;+ return BTLA_CODE::Success;+ }++ protected:+ AUTOCALL void quantRowBlock(const float* srcptr, int8_t* dstptr, int row, int col, int ld_src, int ld_dst,+ float* scales, StorageWeight* ptr) {+ auto quant_dtype = ptr->mDType;+ if (quant_dtype == BTLA_DTYPE::F8_E4M3) {+ kernel::wrapper::QuantizeF8RowBlock<BTLA_DTYPE::F8_E4M3>::forward_auto(srcptr, dstptr, row, col, ld_src, ld_dst,+ scales, ptr->mBlockSize, ptr->SDtype());+ } else if (quant_dtype == BTLA_DTYPE::F8_E5M2) {+ kernel::wrapper::QuantizeF8RowBlock<BTLA_DTYPE::F8_E5M2>::forward_auto(srcptr, dstptr, row, col, ld_src, ld_dst,+ scales, ptr->mBlockSize, ptr->SDtype());+ } else if (quant_dtype == BTLA_DTYPE::F4_BNB) {+ kernel::wrapper::QuantizeF4RowBlock<BTLA_DTYPE::F4_BNB>::forward_auto(srcptr, dstptr, row, col, ld_src, ld_dst,+ scales, ptr->mBlockSize);+ } else if (quant_dtype == BTLA_DTYPE::F4_E2M1) {+ kernel::wrapper::QuantizeF4RowBlock<BTLA_DTYPE::F4_E2M1>::forward_auto(srcptr, dstptr, row, col, ld_src, ld_dst,+ scales, ptr->mBlockSize);+ } else if (quant_dtype == BTLA_DTYPE::F4_NF4) {+ kernel::wrapper::QuantizeF4RowBlock<BTLA_DTYPE::F4_NF4>::forward_auto(srcptr, dstptr, row, col, ld_src, ld_dst,+ scales, ptr->mBlockSize);+ } else {+ assert(0);+ }+ }+};+} // namespace gemm+} // namespace prologue_b+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_storage.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_storage.hnew file mode 100644index 000000000..6d6f0c149--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_storage.h@@ -0,0 +1,1347 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include "bestla.h"+#include "bestla_gemm.h"+#include "bestla_utils.h"++namespace bestla {+namespace storage {++constexpr size_t Alignment = 64;+class ISerialObject {+ protected:+ virtual size_t getSerializedSize() = 0;++ virtual void serializeToBuffer(int8_t*& wptr) = 0;++ virtual void deserializeBuffer(int8_t*& rptr, bool map_buf) = 0;+};++class ISerializable : public ISerialObject {+ public:+ virtual ~ISerializable() = default;++ virtual void assign(int8_t* buf) = 0;++ virtual void serialize(int8_t* wptr) = 0;++ virtual void deserialize(int8_t* rptr) = 0;+ size_t mSize = 0;++ protected:+ virtual size_t getSerializedSize() override {+ size_t totalsize = 0;+ totalsize += sizeof(mSize);+ return totalsize;+ }+ virtual void serializeToBuffer(int8_t*& wptr) override { utils::serialize(wptr, mSize); }+ virtual void deserializeBuffer(int8_t*& rptr, bool map_buf) override {+ if (!map_buf) {+ mSize = utils::deserialize<size_t>(rptr);+ } else {+ utils::serialize<size_t>(rptr, mSize);+ }+ }+};++class ISerialObjectV2 {+ public:+ virtual size_t get_serialized_size() { return get_misc_size() + get_buffer_size(); }+ virtual size_t get_misc_size() = 0;+ virtual size_t get_buffer_size() { return 0; };++ virtual void serialize_misc(int8_t*& wptr) = 0;+ virtual void deserialize_misc(int8_t*& wptr) = 0;+ virtual void set_buffer(int8_t*& bufptr){};+};++class ICollection {+ public:+ virtual ~ICollection() = default;+ BTLA_PROLOGUEB_IDS prologue_id_ = BTLA_PROLOGUEB_IDS::Undef;+ size_t misc_size_, buf_size_;+ static constexpr size_t get_misc_size() { return sizeof(prologue_id_) + sizeof(misc_size_) + sizeof(buf_size_); }++ virtual void set_buffers(int8_t* _buf) = 0;++ virtual void copy_buffers(int8_t* _dst_buf) = 0;++ virtual int8_t* serialize(int8_t* _ptr) {+ utils::serialize(_ptr, prologue_id_);+ utils::serialize(_ptr, misc_size_);+ utils::serialize(_ptr, buf_size_);+ return _ptr;+ }++ virtual int8_t* deserialize(int8_t* _ptr) {+ prologue_id_ = utils::deserialize<BTLA_PROLOGUEB_IDS>(_ptr);+ misc_size_ = utils::deserialize<size_t>(_ptr);+ buf_size_ = utils::deserialize<size_t>(_ptr);+ return _ptr;+ }+};++template <int ALIGN>+class ObjectAlignedBuffer : public ISerialObject {+ public:+ template <typename T>+ inline constexpr T* get() const {+ return reinterpret_cast<T*>(mBufPtr);+ }++ template <typename T>+ inline size_t size() {+ return mBufSize / sizeof(T);+ }++ void resize(size_t bytes) { mBufSize = bytes; }++ // ser+ int8_t* mBufPtr = nullptr;+ size_t mBufSize = 0;+ size_t mBufOffset = 0;++ size_t getSerializedSize() override {+ size_t totalsize = 0;+ totalsize += sizeof(mBufSize);+ totalsize += sizeof(mBufOffset);+ totalsize += mBufSize + ALIGN;+ return totalsize;+ }++ virtual void serializeToBuffer(int8_t*& wptr) override {+ utils::serialize(wptr, mBufSize);+ auto tmpptr = wptr + sizeof(mBufOffset);+ mBufOffset = utils::pointer_align<ALIGN>(tmpptr) - tmpptr;+ utils::serialize(wptr, mBufOffset);+ wptr += mBufOffset;+ if (wptr != mBufPtr) {+ std::memcpy(wptr, mBufPtr, mBufSize);+ }+ wptr += mBufSize;+ }+ virtual void deserializeBuffer(int8_t*& rptr, bool map_buf) override {+ if (!map_buf) {+ mBufSize = utils::deserialize<size_t>(rptr);+ mBufOffset = utils::deserialize<size_t>(rptr);+ } else {+ utils::serialize<size_t>(rptr, mBufSize);+ auto tmpptr = rptr + sizeof(mBufOffset);+ mBufOffset = utils::pointer_align<ALIGN>(tmpptr) - tmpptr;+ utils::serialize(rptr, mBufOffset);+ }+ rptr += mBufOffset;+ mBufPtr = rptr;+ rptr += mBufSize;+ }+};++class ObjectBuffer : public ISerialObjectV2 {+ public:+ // ser+ int8_t* buf_ptr_ = nullptr;+ size_t buf_size_ = 0;++ size_t get_misc_size() override {+ size_t totalsize = 0;+ totalsize += sizeof(buf_size_);+ return totalsize;+ }++ void serialize_misc(int8_t*& wptr) override { utils::serialize(wptr, buf_size_); }+ void deserialize_misc(int8_t*& rptr) override { buf_size_ = utils::deserialize<size_t>(rptr); }++ size_t get_buffer_size() override { return buf_size_; }++ void set_buffer(int8_t*& bufptr) override {+ buf_ptr_ = bufptr;+ bufptr += buf_size_;+ }+};++template <int ALIGN>+class ObjectOptionalBuffer : public ObjectAlignedBuffer<ALIGN> {+ public:+ void resize(size_t bytes) {+ ObjectAlignedBuffer<ALIGN>::resize(bytes);+ mNotEmpty = bytes > 0;+ }++ // ser+ bool mNotEmpty{false};++ virtual size_t getSerializedSize() override {+ size_t totalsize = 0;+ totalsize += sizeof(mNotEmpty);+ if (mNotEmpty) {+ totalsize += ObjectAlignedBuffer<ALIGN>::getSerializedSize();+ }+ return totalsize;+ }+ virtual void serializeToBuffer(int8_t*& wptr) override {+ utils::serialize(wptr, mNotEmpty);+ if (mNotEmpty) {+ ObjectAlignedBuffer<ALIGN>::serializeToBuffer(wptr);+ }+ }+ virtual void deserializeBuffer(int8_t*& rptr, bool map_buf) override {+ if (!map_buf) {+ mNotEmpty = utils::deserialize<bool>(rptr);+ } else {+ utils::serialize<bool>(rptr, mNotEmpty);+ }+ if (mNotEmpty) {+ ObjectAlignedBuffer<ALIGN>::deserializeBuffer(rptr, map_buf);+ }+ }+};++namespace gemm {+enum class Layout : uint32_t {+ PLAIN = 0, // torch default N*K or M*K+ T = PLAIN,+ NT,+ NPack,+};++class ObjectQuantCorrection : public ISerialObject {+ // ser+ public:+ size_t mCSize = 0;+ int mCStep = 0;+ BTLA_DTYPE mScaT = BTLA_DTYPE::F32, mZpT = BTLA_DTYPE::F32;+ ObjectAlignedBuffer<Alignment> mScaleBuf;+ ObjectOptionalBuffer<Alignment> mZpBuf, mRedBuf;+ ObjectOptionalBuffer<Alignment> mDQCorrectionBuf;++ // non-ser+ public:+ int mScaEleSize = 0, mZpEleSize = 0;++ size_t resize(int Rows, int Step, BTLA_DTYPE scalet, BTLA_DTYPE zpt, bool _is_asym, bool _has_reduce) {+ mScaT = scalet;+ mZpT = zpt;+ updateSize();+ mCStep = Step;+ mCSize = static_cast<size_t>(Rows) * Step;+ mScaleBuf.resize(mCSize * mScaEleSize);+ if (_is_asym) {+ mZpBuf.resize(mCSize * mZpEleSize);+ }+ if (_has_reduce) {+ mRedBuf.resize(mCSize * sizeof(int));+ }+ return getSerializedSize();+ }++ virtual size_t getSerializedSize() override {+ size_t totalsize = getMiscSize();+ totalsize += mScaleBuf.getSerializedSize();+ totalsize += mZpBuf.getSerializedSize();+ totalsize += mRedBuf.getSerializedSize();+ totalsize += mDQCorrectionBuf.getSerializedSize();+ return totalsize;+ }+ virtual void serializeToBuffer(int8_t*& wptr) override {+ utils::serialize(wptr, mScaT);+ utils::serialize(wptr, mZpT);+ utils::serialize(wptr, mCStep);+ utils::serialize(wptr, mCSize);+ mScaleBuf.serializeToBuffer(wptr);+ mZpBuf.serializeToBuffer(wptr);+ mRedBuf.serializeToBuffer(wptr);+ mDQCorrectionBuf.serializeToBuffer(wptr);+ }+ virtual void deserializeBuffer(int8_t*& rptr, bool locate_buf) override {+ if (!locate_buf) {+ mScaT = utils::deserialize<BTLA_DTYPE>(rptr);+ mZpT = utils::deserialize<BTLA_DTYPE>(rptr);+ updateSize();+ mCStep = utils::deserialize<int>(rptr);+ mCSize = utils::deserialize<size_t>(rptr);+ } else {+ utils::serialize<BTLA_DTYPE>(rptr, mScaT);+ utils::serialize<BTLA_DTYPE>(rptr, mZpT);+ utils::serialize<int>(rptr, mCStep);+ utils::serialize<size_t>(rptr, mCSize);+ }+ mScaleBuf.deserializeBuffer(rptr, locate_buf);+ mZpBuf.deserializeBuffer(rptr, locate_buf);+ mRedBuf.deserializeBuffer(rptr, locate_buf);+ mDQCorrectionBuf.deserializeBuffer(rptr, locate_buf);+ }+ void enable_double_quant(size_t scale_size, BTLA_DTYPE stype) {+ if (stype == BTLA_DTYPE::DQ8_BNB) {+ auto super_scale_size = scale_size * sizeof(float);+ auto super_zp_size = sizeof(float);+ mDQCorrectionBuf.resize(super_scale_size + super_zp_size);+ } else {+ assert(0);+ }+ };++ protected:+ inline void updateSize() {+ mScaEleSize = int(utils::bestla_dtype_bytes(mScaT));+ mZpEleSize = int(utils::bestla_dtype_bytes(mZpT));+ }++ inline constexpr size_t getMiscSize() {+ size_t totalsize = 0;+ totalsize += sizeof(mScaT);+ totalsize += sizeof(mZpT);+ totalsize += sizeof(mCStep);+ totalsize += sizeof(mCSize);+ return totalsize;+ }+};++class IWeightBase : public storage::ISerializable {+ public:+ BTLA_PROLOGUEB_IDS mPrologueID = BTLA_PROLOGUEB_IDS::Undef;+ uint64_t mCoreId = 0;+ BTLA_DTYPE mDType = BTLA_DTYPE::F32;+ Layout mLayout = Layout::PLAIN;+ int mNPad = 0, mKPad = 0;+ int mN = 0, mK = 0;++ IWeightBase(uint64_t _id) { mCoreId = _id; }++ // bytes offset to mPrologueID+ static constexpr inline size_t offset() { return sizeof(mSize); }++ protected:+ void resize(int NPad, int KPad, int N, int K, BTLA_DTYPE dtype) {+ mNPad = NPad;+ mKPad = KPad;+ mN = N;+ mK = K;+ mDType = dtype;+ }++ virtual size_t getSerializedSize() { return ISerializable::getSerializedSize() + getMiscSize(); }++ virtual void serializeToBuffer(int8_t*& wptr) {+ ISerializable::serializeToBuffer(wptr);+ utils::serialize(wptr, mPrologueID);+ utils::serialize(wptr, mCoreId);+ utils::serialize(wptr, mNPad);+ utils::serialize(wptr, mKPad);+ utils::serialize(wptr, mN);+ utils::serialize(wptr, mK);+ utils::serialize(wptr, mDType);+ utils::serialize(wptr, mLayout);+ }++ virtual void deserializeBuffer(int8_t*& rptr, bool map_buf) {+ ISerializable::deserializeBuffer(rptr, map_buf);+ if (!map_buf) {+ mPrologueID = utils::deserialize<BTLA_PROLOGUEB_IDS>(rptr);+ mCoreId = utils::deserialize<uint64_t>(rptr);+ mNPad = utils::deserialize<int>(rptr);+ mKPad = utils::deserialize<int>(rptr);+ mN = utils::deserialize<int>(rptr);+ mK = utils::deserialize<int>(rptr);+ mDType = utils::deserialize<BTLA_DTYPE>(rptr);+ mLayout = utils::deserialize<Layout>(rptr);+ } else {+ utils::serialize<BTLA_PROLOGUEB_IDS>(rptr, mPrologueID);+ utils::serialize<uint64_t>(rptr, mCoreId);+ utils::serialize<int>(rptr, mNPad);+ utils::serialize<int>(rptr, mKPad);+ utils::serialize<int>(rptr, mN);+ utils::serialize<int>(rptr, mK);+ utils::serialize<BTLA_DTYPE>(rptr, mDType);+ utils::serialize<Layout>(rptr, mLayout);+ }+ }++ inline constexpr size_t getMiscSize() {+ size_t totalsize = 0;+ totalsize += sizeof(mPrologueID);+ totalsize += sizeof(mCoreId);+ totalsize += sizeof(mNPad);+ totalsize += sizeof(mKPad);+ totalsize += sizeof(mN);+ totalsize += sizeof(mK);+ totalsize += sizeof(mDType);+ totalsize += sizeof(mLayout);+ return totalsize;+ }+};++class ObjectWeightInfo : public storage::ISerialObjectV2 {+ public:+ uint64_t core_id_ = 0;+ BTLA_DTYPE dtype_ = BTLA_DTYPE::F32;+ Layout layout_ = Layout::PLAIN;+ int npad_ = 0, kpad_ = 0;+ int n_ = 0, k_ = 0;++ ObjectWeightInfo(uint64_t _id) { core_id_ = _id; }++ void resize(int NPad, int KPad, int N, int K, BTLA_DTYPE dtype, Layout _layout) {+ npad_ = NPad;+ kpad_ = KPad;+ n_ = N;+ k_ = K;+ dtype_ = dtype;+ layout_ = _layout;+ }++ size_t get_misc_size() override {+ size_t totalsize = 0;+ totalsize += sizeof(core_id_);+ totalsize += sizeof(npad_);+ totalsize += sizeof(kpad_);+ totalsize += sizeof(n_);+ totalsize += sizeof(k_);+ totalsize += sizeof(dtype_);+ totalsize += sizeof(layout_);+ return totalsize;+ }++ void serialize_misc(int8_t*& wptr) override {+ utils::serialize(wptr, core_id_);+ utils::serialize(wptr, npad_);+ utils::serialize(wptr, kpad_);+ utils::serialize(wptr, n_);+ utils::serialize(wptr, k_);+ utils::serialize(wptr, dtype_);+ utils::serialize(wptr, layout_);+ }++ void deserialize_misc(int8_t*& rptr) override {+ core_id_ = utils::deserialize<uint64_t>(rptr);+ npad_ = utils::deserialize<int>(rptr);+ kpad_ = utils::deserialize<int>(rptr);+ n_ = utils::deserialize<int>(rptr);+ k_ = utils::deserialize<int>(rptr);+ dtype_ = utils::deserialize<BTLA_DTYPE>(rptr);+ layout_ = utils::deserialize<Layout>(rptr);+ }+};++class IWeightKBlockBase : public IWeightBase {+ public:+ int mBlockSize = 1;+ int mDqBlockSize = 0;+ IWeightKBlockBase(uint64_t _id) : IWeightBase(_id) {}+ void resize(int NPad, int KPad, int Block, int N, int K, BTLA_DTYPE dtype) {+ IWeightBase::resize(NPad, KPad, N, K, dtype);+ mBlockSize = Block;+ }++ protected:+ virtual size_t getSerializedSize() {+ size_t totalsize = IWeightBase::getSerializedSize() + getMiscSize();+ return totalsize;+ }++ virtual void serializeToBuffer(int8_t*& wptr) {+ IWeightBase::serializeToBuffer(wptr);+ utils::serialize(wptr, mBlockSize);+ utils::serialize(wptr, mDqBlockSize);+ }++ virtual void deserializeBuffer(int8_t*& rptr, bool map_buf) {+ IWeightBase::deserializeBuffer(rptr, map_buf);+ if (!map_buf) {+ mBlockSize = utils::deserialize<int>(rptr);+ mDqBlockSize = utils::deserialize<int>(rptr);+ } else {+ utils::serialize(rptr, mBlockSize);+ utils::serialize(rptr, mDqBlockSize);+ }+ }++ inline constexpr size_t getMiscSize() {+ size_t totalsize = sizeof(mBlockSize);+ totalsize += sizeof(mDqBlockSize);+ return totalsize;+ }+};++class IActivationBase : public storage::ISerializable {+ public:+ BTLA_PROLOGUEB_IDS mPrologueID = BTLA_PROLOGUEB_IDS::Undef;+ uint64_t mCoreId = 0;+ BTLA_DTYPE mDType = BTLA_DTYPE::F32;+ Layout mLayout = Layout::PLAIN;+ int mMPad = 0, mKPad = 0;+ int mM = 0, mK = 0;++ IActivationBase(uint64_t _id) { mCoreId = _id; }++ // bytes offset to mPrologueID+ static constexpr inline size_t offset() { return sizeof(mSize); }++ protected:+ void resize(int NPad, int KPad, int N, int K, BTLA_DTYPE dtype) {+ mMPad = NPad;+ mKPad = KPad;+ mM = N;+ mK = K;+ mDType = dtype;+ }++ virtual size_t getSerializedSize() { return ISerializable::getSerializedSize() + getMiscSize(); }++ virtual void serializeToBuffer(int8_t*& wptr) {+ ISerializable::serializeToBuffer(wptr);+ utils::serialize(wptr, mPrologueID);+ utils::serialize(wptr, mCoreId);+ utils::serialize(wptr, mMPad);+ utils::serialize(wptr, mKPad);+ utils::serialize(wptr, mM);+ utils::serialize(wptr, mK);+ utils::serialize(wptr, mDType);+ utils::serialize(wptr, mLayout);+ }++ virtual void deserializeBuffer(int8_t*& rptr, bool map_buf) {+ ISerializable::deserializeBuffer(rptr, map_buf);+ if (!map_buf) {+ mPrologueID = utils::deserialize<BTLA_PROLOGUEB_IDS>(rptr);+ mCoreId = utils::deserialize<uint64_t>(rptr);+ mMPad = utils::deserialize<int>(rptr);+ mKPad = utils::deserialize<int>(rptr);+ mM = utils::deserialize<int>(rptr);+ mK = utils::deserialize<int>(rptr);+ mDType = utils::deserialize<BTLA_DTYPE>(rptr);+ mLayout = utils::deserialize<Layout>(rptr);+ } else {+ utils::serialize<BTLA_PROLOGUEB_IDS>(rptr, mPrologueID);+ utils::serialize<uint64_t>(rptr, mCoreId);+ utils::serialize<int>(rptr, mMPad);+ utils::serialize<int>(rptr, mKPad);+ utils::serialize<int>(rptr, mM);+ utils::serialize<int>(rptr, mK);+ utils::serialize<BTLA_DTYPE>(rptr, mDType);+ utils::serialize<Layout>(rptr, mLayout);+ }+ }++ inline constexpr size_t getMiscSize() {+ size_t totalsize = 0;+ totalsize += sizeof(mPrologueID);+ totalsize += sizeof(mCoreId);+ totalsize += sizeof(mMPad);+ totalsize += sizeof(mKPad);+ totalsize += sizeof(mM);+ totalsize += sizeof(mK);+ totalsize += sizeof(mDType);+ totalsize += sizeof(mLayout);+ return totalsize;+ }+};++class IActivationKBlockBase : public IActivationBase {+ public:+ int mBlockSize = 1;+ IActivationKBlockBase(uint64_t _id) : IActivationBase(_id) {}+ void resize(int MPad, int KPad, int Block, int N, int K, BTLA_DTYPE dtype) {+ IActivationBase::resize(MPad, KPad, N, K, dtype);+ mBlockSize = Block;+ }++ protected:+ virtual size_t getSerializedSize() {+ size_t totalsize = IActivationBase::getSerializedSize() + getMiscSize();+ return totalsize;+ }++ virtual void serializeToBuffer(int8_t*& wptr) {+ IActivationBase::serializeToBuffer(wptr);+ utils::serialize(wptr, mBlockSize);+ }++ virtual void deserializeBuffer(int8_t*& rptr, bool map_buf) {+ IActivationBase::deserializeBuffer(rptr, map_buf);+ if (!map_buf) {+ mBlockSize = utils::deserialize<int>(rptr);+ } else {+ utils::serialize(rptr, mBlockSize);+ }+ }++ inline constexpr size_t getMiscSize() {+ size_t totalsize = sizeof(mBlockSize);+ return totalsize;+ }+};++class StoragePackedWeight : public IWeightBase {+ public:+ ObjectAlignedBuffer<Alignment> mWBuf;+ StoragePackedWeight(uint64_t _id) : IWeightBase(_id) { mPrologueID = BTLA_PROLOGUEB_IDS::WeightPack; }++ size_t resize(int NPad, int KPad, int N, int K, BTLA_DTYPE dtype, Layout _layout) {+ IWeightBase::resize(NPad, KPad, N, K, dtype);+ auto bsize = static_cast<size_t>(NPad) * KPad * utils::bestla_dtype_bytes(dtype);+ mWBuf.resize(bsize);+ mSize = IWeightBase::getSerializedSize() + mWBuf.getSerializedSize();+ mSize = utils::padto(mSize, Alignment);+ mLayout = _layout;+ return mSize;+ }++ template <typename T>+ inline constexpr T* WPtr() const {+ return mWBuf.get<T>();+ }++ virtual void assign(int8_t* buf) override {+ IWeightBase::deserializeBuffer(buf, true);+ mWBuf.deserializeBuffer(buf, true);+ }++ virtual void serialize(int8_t* wptr) {+ IWeightBase::serializeToBuffer(wptr);+ mWBuf.serializeToBuffer(wptr);+ }++ virtual void deserialize(int8_t* rptr) override {+ IWeightBase::deserializeBuffer(rptr, false);+ mWBuf.deserializeBuffer(rptr, false);+ }+};++class StorageWeight : public ICollection {+ public:+ ObjectWeightInfo info_;+ ObjectBuffer buf_;++ StorageWeight(uint64_t _id = 0) : info_(_id) {+ prologue_id_ = BTLA_PROLOGUEB_IDS::WeightPack;+ update_size();+ }++ void set_plain_data(int _n, int _k, int _ld, BTLA_DTYPE dtype, void* _ptr) {+ info_.resize(_n, _ld, _n, _k, dtype, gemm::Layout::PLAIN);+ buf_.buf_size_ = (size_t)_n * _ld * ele_size();+ update_size();+ set_buffers((int8_t*)_ptr);+ }++ void set_nt_data(int _n, int _k, int _ld, BTLA_DTYPE dtype, void* _ptr) {+ info_.resize(_ld, _k, _n, _k, dtype, gemm::Layout::NT);+ buf_.buf_size_ = (size_t)_k * _ld * ele_size();+ update_size();+ set_buffers((int8_t*)_ptr);+ }++ void set_buffers(int8_t* _buf) override { buf_.set_buffer(_buf); }++ void copy_buffers(int8_t* _dst_buf) override { std::memcpy(_dst_buf, buf_.buf_ptr_, buf_size_); }++ int8_t* serialize(int8_t* _ptr) {+ auto raw_ptr = _ptr;+ ICollection::serialize(_ptr);+ _ptr += get_misc_size();+ info_.serialize_misc(_ptr);+ buf_.serialize_misc(_ptr);+ return raw_ptr + misc_size_;+ }++ int8_t* deserialize(int8_t* _ptr) {+ auto raw_ptr = _ptr;+ ICollection::deserialize(_ptr);+ _ptr += get_misc_size();+ info_.deserialize_misc(_ptr);+ buf_.deserialize_misc(_ptr);+ return raw_ptr + misc_size_;+ }++ inline constexpr size_t ele_size() const { return bestla::utils::bestla_dtype_bytes(info_.dtype_); }++ size_t resize(int NPad, int KPad, int N, int K, BTLA_DTYPE dtype, Layout _layout) {+ info_.resize(NPad, KPad, N, K, dtype, _layout);+ auto bsize = static_cast<size_t>(NPad) * KPad * utils::bestla_dtype_bytes(dtype);+ buf_.buf_size_ = bsize;+ update_size();+ return buf_size_;+ }++ void update_size() {+ buf_size_ = info_.get_buffer_size() + buf_.get_buffer_size();+ misc_size_ = utils::padto(get_misc_size() + info_.get_misc_size() + buf_.get_misc_size(), Alignment);+ }++ size_t total_size() { return buf_size_ + misc_size_; }++ inline constexpr int8_t* ptr() const { return buf_.buf_ptr_; }+};++class ObjectCorrection : public ISerialObjectV2 {+ // ser+ public:+ size_t size_ = 0;+ int step_ = 0;+ BTLA_DTYPE sdtype_ = BTLA_DTYPE::F32, zdtype_ = BTLA_DTYPE::S8;+ ObjectBuffer scale_buf_, zp_buf_, red_buf_, dq_buf_;++ // non-ser+ public:+ inline int8_t* sptr() const { return scale_buf_.buf_size_ ? scale_buf_.buf_ptr_ : nullptr; }+ inline int8_t* zptr() const { return zp_buf_.buf_size_ ? zp_buf_.buf_ptr_ : nullptr; }+ inline int8_t* rptr() const { return red_buf_.buf_size_ ? red_buf_.buf_ptr_ : nullptr; }++ void resize(int Rows, int Step, BTLA_DTYPE scalet, BTLA_DTYPE zpt, bool _is_asym, bool _has_reduce) {+ sdtype_ = scalet;+ zdtype_ = zpt;+ step_ = Step;+ size_ = static_cast<size_t>(Rows) * Step;+ scale_buf_.buf_size_ = size_ * utils::bestla_dtype_bytes(sdtype_);+ if (_is_asym) {+ zp_buf_.buf_size_ = size_ * utils::bestla_dtype_bytes(zdtype_);+ }+ if (_has_reduce) {+ red_buf_.buf_size_ = size_ * sizeof(int);+ }+ }++ size_t get_misc_size() override {+ size_t totalsize = 0;+ totalsize += sizeof(size_);+ totalsize += sizeof(step_);+ totalsize += sizeof(sdtype_);+ totalsize += sizeof(zdtype_);+ totalsize += scale_buf_.get_misc_size();+ totalsize += zp_buf_.get_misc_size();+ totalsize += red_buf_.get_misc_size();+ totalsize += dq_buf_.get_misc_size();+ return totalsize;+ }++ void serialize_misc(int8_t*& wptr) override {+ utils::serialize(wptr, size_);+ utils::serialize(wptr, step_);+ utils::serialize(wptr, sdtype_);+ utils::serialize(wptr, zdtype_);+ scale_buf_.serialize_misc(wptr);+ zp_buf_.serialize_misc(wptr);+ red_buf_.serialize_misc(wptr);+ dq_buf_.serialize_misc(wptr);+ }++ void deserialize_misc(int8_t*& rptr) override {+ size_ = utils::deserialize<size_t>(rptr);+ step_ = utils::deserialize<int>(rptr);+ sdtype_ = utils::deserialize<BTLA_DTYPE>(rptr);+ zdtype_ = utils::deserialize<BTLA_DTYPE>(rptr);+ scale_buf_.deserialize_misc(rptr);+ zp_buf_.deserialize_misc(rptr);+ red_buf_.deserialize_misc(rptr);+ dq_buf_.deserialize_misc(rptr);+ }++ size_t get_buffer_size() override {+ return scale_buf_.get_buffer_size() + zp_buf_.get_buffer_size() + red_buf_.get_buffer_size() ++ dq_buf_.get_buffer_size();+ }++ void set_buffer(int8_t*& bufptr) override {+ scale_buf_.set_buffer(bufptr);+ zp_buf_.set_buffer(bufptr);+ red_buf_.set_buffer(bufptr);+ dq_buf_.set_buffer(bufptr);+ }++ void copy_buffers(int8_t*& _dst_buf) {+ if (scale_buf_.buf_size_) {+ std::memcpy(_dst_buf, scale_buf_.buf_ptr_, scale_buf_.buf_size_);+ _dst_buf += scale_buf_.buf_size_;+ }+ if (zp_buf_.buf_size_) {+ std::memcpy(_dst_buf, zp_buf_.buf_ptr_, zp_buf_.buf_size_);+ _dst_buf += zp_buf_.buf_size_;+ }+ if (red_buf_.buf_size_) {+ std::memcpy(_dst_buf, red_buf_.buf_ptr_, red_buf_.buf_size_);+ _dst_buf += red_buf_.buf_size_;+ }+ if (dq_buf_.buf_size_) {+ std::memcpy(_dst_buf, dq_buf_.buf_ptr_, dq_buf_.buf_size_);+ _dst_buf += dq_buf_.buf_size_;+ }+ }++ void enable_double_quant(size_t scale_size, BTLA_DTYPE stype) {+ if (stype == BTLA_DTYPE::DQ8_BNB) {+ auto super_scale_size = scale_size * sizeof(float);+ auto super_zp_size = sizeof(float);+ dq_buf_.buf_size_ = super_scale_size + super_zp_size;+ } else {+ assert(0);+ }+ };+};+class StorageWeightNInt : public ICollection {+ public:+ ObjectWeightInfo info_;+ int block_ = -1, n_block_ = 0;+ ObjectBuffer buf_;+ ObjectCorrection corr_;++ StorageWeightNInt(uint64_t _id = 0) : info_(_id) {+ prologue_id_ = BTLA_PROLOGUEB_IDS::WeightKBlockNInteger;+ update_size();+ }++ void t_from(const StorageWeightNInt& src) {+ block_ = src.block_;+ n_block_ = src.n_block_;+ info_ = src.info_;+ info_.kpad_ = info_.k_;+ info_.npad_ = info_.n_;+ info_.layout_ = gemm::Layout::T;+ auto bytes = nbit_size(static_cast<size_t>(info_.n_) * info_.k_, info_.dtype_);+ buf_.buf_size_ = bytes;+ corr_.resize(info_.n_, n_block_, src.corr_.sdtype_, src.corr_.zdtype_, src.corr_.zptr() != nullptr,+ src.corr_.rptr() != nullptr);+ update_size();+ }++ void set_plain_data(int _n, int _k, int _ld, BTLA_DTYPE dtype, void* _ptr) {+ info_.resize(_n, _ld, _n, _k, dtype, gemm::Layout::PLAIN);+ buf_.buf_size_ = (size_t)_n * _ld * ele_size();+ update_size();+ set_buffers((int8_t*)_ptr);+ }++ void set_nt_data(int _n, int _k, int _ld, BTLA_DTYPE dtype, void* _ptr) {+ info_.resize(_ld, _k, _n, _k, dtype, gemm::Layout::NT);+ buf_.buf_size_ = (size_t)_k * _ld * ele_size();+ update_size();+ set_buffers((int8_t*)_ptr);+ }++ void set_buffers(int8_t* _buf) override {+ buf_.set_buffer(_buf);+ corr_.set_buffer(_buf);+ }++ void copy_buffers(int8_t* _dst_buf) override {+ std::memcpy(_dst_buf, buf_.buf_ptr_, buf_.buf_size_);+ _dst_buf += buf_.buf_size_;+ corr_.copy_buffers(_dst_buf);+ }++ int8_t* serialize(int8_t* _ptr) {+ auto raw_ptr = _ptr;+ ICollection::serialize(_ptr);+ _ptr += get_misc_size();+ info_.serialize_misc(_ptr);+ utils::serialize(_ptr, block_);+ utils::serialize(_ptr, n_block_);+ buf_.serialize_misc(_ptr);+ corr_.serialize_misc(_ptr);+ return raw_ptr + misc_size_;+ }++ int8_t* deserialize(int8_t* _ptr) {+ auto raw_ptr = _ptr;+ ICollection::deserialize(_ptr);+ _ptr += get_misc_size();+ info_.deserialize_misc(_ptr);+ block_ = utils::deserialize<int>(_ptr);+ n_block_ = utils::deserialize<int>(_ptr);+ buf_.deserialize_misc(_ptr);+ corr_.deserialize_misc(_ptr);+ return raw_ptr + misc_size_;+ }++ inline constexpr size_t ele_size() const { return bestla::utils::bestla_dtype_bytes(info_.dtype_); }++ static size_t nbit_size(size_t elecount, BTLA_DTYPE dtype) {+ auto bitsize = elecount * utils::bestla_dtype_bits(dtype);+ auto bytes = utils::updiv(bitsize, 8); // add 3bits, 5btis, 7bits size calculation here+ if (dtype == BTLA_DTYPE::S3_CLIP)+ bytes = utils::updiv(elecount * 2, 8) + utils::updiv(elecount * 1, 8);+ else if (dtype == BTLA_DTYPE::S5_CLIP)+ bytes = utils::updiv(elecount * 4, 8) + utils::updiv(elecount * 1, 8);+ else if (dtype == BTLA_DTYPE::S6_CLIP)+ bytes = utils::updiv(elecount * 4, 8) + utils::updiv(elecount * 2, 8);+ else if (dtype == BTLA_DTYPE::S7_CLIP)+ bytes = utils::updiv(elecount * 4, 8) + utils::updiv(elecount * 2, 8) + utils::updiv(elecount * 1, 8);+ return bytes;+ }++ size_t resize(int NPad, int KPad, int Block, int N, int K, BTLA_DTYPE dtype, Layout _layout, BTLA_DTYPE scalet,+ bool _is_asym) {+ info_.resize(NPad, KPad, N, K, dtype, _layout);+ block_ = Block;+ auto bytes = nbit_size(static_cast<size_t>(NPad) * KPad, dtype);+ buf_.buf_size_ = bytes;+ n_block_ = utils::updiv(KPad, Block);+ auto gemm_comp = bestla::gemm::CoreAttr::get_comp(info_.core_id_);+ auto is_cint = bestla::gemm::CompTypeHelper::is_integer(gemm_comp);+ BTLA_DTYPE zpt = BTLA_DTYPE::S8;+ corr_.resize(n_block_, NPad, scalet, zpt, _is_asym, is_cint);+ if (scalet == BTLA_DTYPE::DQ8_BNB) {+ corr_.enable_double_quant(utils::updiv(n_block_ * N, Block), scalet);+ }+ update_size();+ return buf_size_;+ }++ void update_size() {+ buf_size_ = info_.get_buffer_size() + buf_.get_buffer_size() + corr_.get_buffer_size();+ misc_size_ =+ utils::padto(get_misc_size() + info_.get_misc_size() + buf_.get_misc_size() + corr_.get_misc_size(), Alignment);+ }++ size_t total_size() { return buf_size_ + misc_size_; }++ inline constexpr int8_t* ptr() const { return buf_.buf_ptr_; }+};++class StorageReduce : public ISerializable {+ public:+ using CorrectionType = ObjectQuantCorrection;+ int m = 0, k = 0, lda = 0, kblock = 1;+ ObjectAlignedBuffer<Alignment> mRedBuf;+ size_t resize(int _m, int _k, int _kblock, BTLA_DTYPE redt) {+ kblock = _kblock;+ m = _m;+ k = _k;+ lda = utils::updiv(_k, _kblock);+ size_t bufsize = static_cast<size_t>(m) * lda * utils::bestla_dtype_bytes(redt);+ mRedBuf.resize(bufsize);+ mSize = getSerializedSize();+ mSize = utils::padto(mSize, Alignment);+ return mSize;+ }+ template <typename QT_T>+ inline QT_T* RPtr() {+ return mRedBuf.get<QT_T>();+ }++ virtual void assign(int8_t* buf) override {+ ISerializable::deserializeBuffer(buf, true);+ deserializeBuffer(buf, true);+ mRedBuf.deserializeBuffer(buf, true);+ }++ virtual void serialize(int8_t* wptr) {+ ISerializable::serializeToBuffer(wptr);+ serializeToBuffer(wptr);+ mRedBuf.serializeToBuffer(wptr);+ }++ virtual void deserialize(int8_t* rptr) override {+ ISerializable::deserializeBuffer(rptr, false);+ deserializeBuffer(rptr, false);+ mRedBuf.deserializeBuffer(rptr, false);+ }++ protected:+ virtual size_t getSerializedSize() {+ return ISerializable::getSerializedSize() + getMiscSize() + mRedBuf.getSerializedSize();+ }++ virtual void serializeToBuffer(int8_t*& wptr) {+ utils::serialize(wptr, m);+ utils::serialize(wptr, k);+ utils::serialize(wptr, lda);+ utils::serialize(wptr, kblock);+ }++ virtual void deserializeBuffer(int8_t*& rptr, bool map_buf) {+ if (!map_buf) {+ m = utils::deserialize<int>(rptr);+ lda = utils::deserialize<int>(rptr);+ kblock = utils::deserialize<int>(rptr);+ } else {+ utils::serialize(rptr, m);+ utils::serialize(rptr, k);+ utils::serialize(rptr, lda);+ utils::serialize(rptr, kblock);+ }+ }++ inline constexpr size_t getMiscSize() {+ size_t totalsize = 0;+ totalsize += sizeof(m);+ totalsize += sizeof(k);+ totalsize += sizeof(lda);+ totalsize += sizeof(kblock);+ return totalsize;+ }+};++class StorageReorderActivation : public IActivationKBlockBase {+ public:+ ObjectAlignedBuffer<Alignment> mABuf;+ StorageReorderActivation(uint64_t _id) : IActivationKBlockBase(_id) { mPrologueID = BTLA_PROLOGUEB_IDS::WeightPack; }++ size_t resize(int MPad, int KPad, int M, int K, int KBlock, BTLA_DTYPE dtype) {+ IActivationKBlockBase::resize(MPad, KPad, KBlock, M, K, dtype);+ auto bsize = static_cast<size_t>(MPad) * KPad * utils::bestla_dtype_bytes(dtype);+ mABuf.resize(bsize);+ mSize = IActivationKBlockBase::getSerializedSize() + mABuf.getSerializedSize();+ mSize = utils::padto(mSize, Alignment);+ return mSize;+ }++ template <typename T>+ inline constexpr T* APtr() const {+ return mABuf.get<T>();+ }++ virtual void assign(int8_t* buf) override {+ IActivationKBlockBase::deserializeBuffer(buf, true);+ mABuf.deserializeBuffer(buf, true);+ }++ virtual void serialize(int8_t* wptr) {+ IActivationKBlockBase::serializeToBuffer(wptr);+ mABuf.serializeToBuffer(wptr);+ }++ virtual void deserialize(int8_t* rptr) override {+ IActivationKBlockBase::deserializeBuffer(rptr, false);+ mABuf.deserializeBuffer(rptr, false);+ }+};++class StorageQuantActivation : public IActivationKBlockBase {+ public:+ using CorrectionType = ObjectQuantCorrection;+ CorrectionType mCorrection;+ ObjectAlignedBuffer<Alignment> mQBuf;+ StorageQuantActivation(uint64_t _id = 0) : IActivationKBlockBase(_id) {+ mPrologueID = BTLA_PROLOGUEB_IDS::WeightPack;+ }++ size_t resize(int _mpad, int _kpad, int _m, int _k, int _kblock, BTLA_DTYPE buft, BTLA_DTYPE scalet, BTLA_DTYPE zpt,+ bool is_asym, bool has_reduce) {+ IActivationKBlockBase::resize(_mpad, _kpad, _kblock, _m, _k, buft);+ mCorrection.resize(_mpad, utils::updiv(_kpad, _kblock), scalet, zpt, is_asym, has_reduce);+ size_t bufsize = static_cast<size_t>(_mpad) * _kpad * utils::bestla_dtype_bytes(buft);+ mQBuf.resize(bufsize);+ mSize = getSerializedSize();+ mSize = utils::padto(mSize, Alignment);+ return mSize;+ }++ template <typename QT_T>+ inline constexpr QT_T* APtr() {+ return mQBuf.get<QT_T>();+ }++ template <typename T>+ inline constexpr size_t ASize() {+ return mQBuf.size<T>();+ }++ template <typename QT_T>+ inline constexpr QT_T* ZPtr() {+ return mCorrection.mZpBuf.get<QT_T>();+ }++ template <typename QT_T>+ inline constexpr QT_T* SPtr() {+ return mCorrection.mScaleBuf.get<QT_T>();+ }++ template <typename QT_T>+ inline constexpr QT_T* RPtr() {+ return mCorrection.mRedBuf.get<QT_T>();+ }++ inline constexpr BTLA_DTYPE ZDtype() { return mCorrection.mZpT; }+ inline constexpr BTLA_DTYPE SDtype() { return mCorrection.mScaT; }+ inline constexpr bool IsAsym() { return mCorrection.mZpBuf.mNotEmpty; }+ inline constexpr bool HasReduce() { return mCorrection.mRedBuf.mNotEmpty; }+ inline constexpr size_t CSize() { return mCorrection.mCSize; }+ inline constexpr int CStep() { return mCorrection.mCStep; }++ virtual void assign(int8_t* buf) override {+ IActivationKBlockBase::deserializeBuffer(buf, true);+ deserializeBuffer(buf, true);+ }++ virtual void serialize(int8_t* wptr) {+ IActivationKBlockBase::serializeToBuffer(wptr);+ serializeToBuffer(wptr);+ }++ virtual void deserialize(int8_t* rptr) override {+ IActivationKBlockBase::deserializeBuffer(rptr, false);+ deserializeBuffer(rptr, false);+ }++ protected:+ virtual size_t getSerializedSize() {+ return ISerializable::getSerializedSize() + getMiscSize() + mQBuf.getSerializedSize() ++ mCorrection.getSerializedSize();+ }++ virtual void serializeToBuffer(int8_t*& wptr) {+ mQBuf.serializeToBuffer(wptr);+ mCorrection.serializeToBuffer(wptr);+ }++ virtual void deserializeBuffer(int8_t*& rptr, bool map_buf) {+ mQBuf.deserializeBuffer(rptr, map_buf);+ mCorrection.deserializeBuffer(rptr, map_buf);+ }++ inline constexpr size_t getMiscSize() {+ size_t totalsize = 0;+ return totalsize;+ }+};++class StorageWeightKBlockNInteger : public IWeightKBlockBase {+ public:+ using InfoType = IWeightKBlockBase;+ using QWeightType = ObjectAlignedBuffer<Alignment>;+ using CorrectionType = ObjectQuantCorrection;+ QWeightType mQBuf;+ CorrectionType mCorrection;+ ObjectOptionalBuffer<Alignment> mShuffleIndices;+ StorageWeightKBlockNInteger(uint64_t _type) : IWeightKBlockBase(_type) {+ mPrologueID = BTLA_PROLOGUEB_IDS::WeightKBlockNInteger;+ }++ StorageWeightKBlockNInteger toTrans() {+ StorageWeightKBlockNInteger trans(-1);+ trans.mK = mK;+ trans.mN = mN;+ trans.mNPad = mNPad;+ trans.mKPad = mKPad;+ trans.mBlockSize = mBlockSize;+ trans.mDType = mDType;+ trans.mQBuf.resize(mQBuf.size<int8_t>());+ int nk_scale = utils::updiv(mKPad, mBlockSize);+ trans.mCorrection.resize(mNPad, nk_scale, mCorrection.mScaT, mCorrection.mZpT, mCorrection.mZpBuf.size<int>() > 0,+ mCorrection.mRedBuf.size<int>() > 0);+ trans.update_size();+ return trans;+ }++ size_t resize(int NPad, int KPad, int Block, int N, int K, BTLA_DTYPE qtype, BTLA_DTYPE scalet, bool IsAsym) {+ BTLA_DTYPE zpt = BTLA_DTYPE::S8;+ InfoType::resize(NPad, KPad, Block, N, K, qtype);+ auto bits = utils::bestla_dtype_bits(qtype);+ auto elesize = static_cast<size_t>(NPad) * KPad;+ auto bytes = utils::updiv(elesize * bits, 8); // add 3bits, 5btis, 7bits size calculation here+ if (qtype == BTLA_DTYPE::S3_CLIP)+ bytes =+ utils::updiv(static_cast<size_t>(KPad) * NPad * 2, 8) + utils::updiv(static_cast<size_t>(KPad) * NPad * 1, 8);+ else if (qtype == BTLA_DTYPE::S5_CLIP)+ bytes =+ utils::updiv(static_cast<size_t>(KPad) * NPad * 4, 8) + utils::updiv(static_cast<size_t>(KPad) * NPad * 1, 8);+ else if (qtype == BTLA_DTYPE::S6_CLIP)+ bytes =+ utils::updiv(static_cast<size_t>(KPad) * NPad * 4, 8) + utils::updiv(static_cast<size_t>(KPad) * NPad * 2, 8);+ else if (qtype == BTLA_DTYPE::S7_CLIP)+ bytes = utils::updiv(static_cast<size_t>(KPad) * NPad * 4, 8) ++ utils::updiv(static_cast<size_t>(KPad) * NPad * 2, 8) ++ utils::updiv(static_cast<size_t>(KPad) * NPad * 1, 8);+ mQBuf.resize(bytes);+ int nk_scale = utils::updiv(KPad, Block);+ auto gemm_comp = bestla::gemm::CoreAttr::get_comp(mCoreId);+ auto is_cint = bestla::gemm::CompTypeHelper::is_integer(gemm_comp);+ mCorrection.resize(nk_scale, NPad, scalet, zpt, IsAsym, is_cint);+ if (scalet == BTLA_DTYPE::DQ8_BNB) initDoubleQuantBlkSize(Block, nk_scale, IsAsym, N, scalet);+ update_size();+ return mSize;+ }++ void initDoubleQuantBlkSize(int dq_blksize, int nk_scale, bool asym, int N, BTLA_DTYPE stype) {+ mDqBlockSize = dq_blksize;+ if (asym || dq_blksize % 8 != 0) assert(0);+ mCorrection.enable_double_quant(utils::updiv(nk_scale * N, dq_blksize), stype);+ }++ void enable_shuffle() {+ auto indicessize = mK * sizeof(int);+ mShuffleIndices.resize(indicessize);+ update_size();+ }++ inline constexpr BTLA_DTYPE ZDtype() { return mCorrection.mZpT; }+ inline constexpr BTLA_DTYPE SDtype() { return mCorrection.mScaT; }+ inline constexpr bool IsAsym() { return mCorrection.mZpBuf.mNotEmpty; }+ inline constexpr bool HasReduce() { return mCorrection.mRedBuf.mNotEmpty; }+ inline constexpr bool IsDoubleQuant() { return mCorrection.mDQCorrectionBuf.mNotEmpty; }+ inline constexpr size_t CSize() { return mCorrection.mCSize; }+ inline constexpr int CStep() { return mCorrection.mCStep; }++ template <typename T>+ inline constexpr size_t WSize() {+ return mQBuf.size<T>();+ }++ template <typename T>+ inline constexpr T* WPtr() const {+ return mQBuf.get<T>();+ }++ template <typename T>+ inline constexpr T* SPtr() {+ return mCorrection.mScaleBuf.get<T>();+ }++ template <typename T>+ inline constexpr T* ZPtr() {+ return mCorrection.mZpBuf.get<T>();+ }++ template <typename T>+ inline constexpr T* RPtr() {+ return mCorrection.mRedBuf.get<T>();+ }++ template <typename T>+ inline constexpr T* DQPtr() {+ return mCorrection.mDQCorrectionBuf.get<T>();+ }++ inline constexpr int* ShfIndice() { return mShuffleIndices.get<int>(); }++ void update_size() {+ mSize = InfoType::getSerializedSize() + mQBuf.getSerializedSize() + mCorrection.getSerializedSize() ++ mShuffleIndices.getSerializedSize();+ mSize = utils::padto(mSize, Alignment);+ }++ virtual void assign(int8_t* buf) override {+ InfoType::deserializeBuffer(buf, true);+ mQBuf.deserializeBuffer(buf, true);+ mCorrection.deserializeBuffer(buf, true);+ mShuffleIndices.deserializeBuffer(buf, true);+ }++ virtual void serialize(int8_t* wptr) {+ InfoType::serializeToBuffer(wptr);+ mQBuf.serializeToBuffer(wptr);+ mCorrection.serializeToBuffer(wptr);+ mShuffleIndices.serializeToBuffer(wptr);+ }++ virtual void deserialize(int8_t* rptr) override {+ InfoType::deserializeBuffer(rptr, false);+ mQBuf.deserializeBuffer(rptr, false);+ mCorrection.deserializeBuffer(rptr, false);+ mShuffleIndices.deserializeBuffer(rptr, false);+ }+};++class StorageWeightKBlockNFloat : public StorageWeightKBlockNInteger {+ public:+ StorageWeightKBlockNFloat(uint64_t _type) : StorageWeightKBlockNInteger(_type) {+ mPrologueID = BTLA_PROLOGUEB_IDS::WeightKBlockNFloat;+ }++ size_t resize(int NPad, int KPad, int Block, int N, int K, BTLA_DTYPE ftype, BTLA_DTYPE scalet) {+ StorageWeightKBlockNInteger::InfoType::resize(NPad, KPad, Block, N, K, ftype);+ auto bits = utils::bestla_dtype_bits(ftype);+ auto elesize = static_cast<size_t>(NPad) * KPad;+ auto bytes = utils::updiv(elesize * bits, 8); // add fp6 size calculation here+ StorageWeightKBlockNInteger::mQBuf.resize(bytes);+ int nk_scale = utils::updiv(KPad, Block);+ StorageWeightKBlockNInteger::mCorrection.resize(nk_scale, NPad, scalet, BTLA_DTYPE::EleBitsUndef, false, false);+ if (scalet == BTLA_DTYPE::DQ8_BNB) initDoubleQuantBlkSize(Block, nk_scale, false, N, scalet);+ update_size();+ mSize = StorageWeightKBlockNInteger::InfoType::getSerializedSize() ++ StorageWeightKBlockNInteger::mQBuf.getSerializedSize() ++ StorageWeightKBlockNInteger::mCorrection.getSerializedSize();+ mSize = utils::padto(mSize, Alignment);+ return mSize;+ }+};++class PackedWeightParser {+ public:+ static gemm::IWeightBase* deserialBuffer(const void* serialized_buf) {+ if (serialized_buf == nullptr) {+ return nullptr;+ }+ auto tmpptr = const_cast<void*>(serialized_buf);+ auto rptr = reinterpret_cast<int8_t*>(tmpptr);+ rptr += IWeightBase::offset();+ int mProID = utils::deserialize<int>(rptr);+ IWeightBase* ptr = nullptr;+ if (mProID >= int(BTLA_PROLOGUEB_IDS::Begin) && mProID < int(BTLA_PROLOGUEB_IDS::End)) {+ rptr = reinterpret_cast<int8_t*>(tmpptr);+ auto type = static_cast<BTLA_PROLOGUEB_IDS>(mProID);+ switch (type) {+ case BTLA_PROLOGUEB_IDS::WeightPack:+ ptr = new gemm::StoragePackedWeight(0);+ break;+ case BTLA_PROLOGUEB_IDS::WeightKBlockNInteger:+ ptr = new gemm::StorageWeightKBlockNInteger(0);+ break;+ case BTLA_PROLOGUEB_IDS::WeightKBlockNFloat:+ ptr = new gemm::StorageWeightKBlockNFloat(0);+ break;+ default:+ break;+ }+ if (ptr) {+ ptr->deserialize(rptr);+ }+ }+ return ptr;+ }+};++} // namespace gemm+class CollectionParser {+ public:+ static inline bool valid_prologue(BTLA_PROLOGUEB_IDS pro_id) {+ return pro_id >= BTLA_PROLOGUEB_IDS::Begin && pro_id < BTLA_PROLOGUEB_IDS::End;+ }+ static BTLA_PROLOGUEB_IDS parse_prologue_id(const void* serialized_buf) {+ auto rptr = (int8_t*)(serialized_buf);+ return utils::deserialize<BTLA_PROLOGUEB_IDS>(rptr);+ }++ static ICollection* parse_buffer(const void* serialized_buf) {+ if (serialized_buf == nullptr) {+ return nullptr;+ }+ auto tmpptr = const_cast<void*>(serialized_buf);+ auto rptr = reinterpret_cast<int8_t*>(tmpptr);+ auto pro_id = utils::deserialize<BTLA_PROLOGUEB_IDS>(rptr);+ ICollection* ptr = nullptr;+ if (pro_id >= BTLA_PROLOGUEB_IDS::Begin && pro_id < BTLA_PROLOGUEB_IDS::End) {+ rptr = reinterpret_cast<int8_t*>(tmpptr);+ switch (pro_id) {+ case BTLA_PROLOGUEB_IDS::WeightPack:+ ptr = new gemm::StorageWeight;+ break;+ case BTLA_PROLOGUEB_IDS::WeightKBlockNInteger:+ ptr = new gemm::StorageWeightNInt;+ break;+ default:+ break;+ }+ if (ptr) {+ auto data_ptr = ptr->deserialize(rptr);+ ptr->set_buffers(data_ptr);+ }+ }+ return ptr;+ }+};++} // namespace storage+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_utils.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_utils.hnew file mode 100644index 000000000..d2ff0d8c8--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_utils.h@@ -0,0 +1,870 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include <type_traits>++#if defined(BTLA_USE_OPENMP)+#define BTLA_OPENMP 1+#else+#define BTLA_OPENMP 0+#endif++#define FP32_BF16_FAST 0++#if BTLA_OPENMP+#include <omp.h>+#endif++#include <algorithm>+#include <chrono>+#include <cmath>+#include <cstring>+#include <functional>+#include <cassert>+#include <vector>+#include <cstdio>+#include <string>+#include <set>+#ifdef _WIN32+#include <cstdlib>+#else+#include <err.h>+#include <errno.h>+#include <sys/mman.h>+#include <sys/signal.h>+#include <sys/syscall.h>+#include <unistd.h>+#include <stdlib.h>++#define fatal_error(msg, ...) err(1, "[FAIL]\t" msg, ##__VA_ARGS__)+#define XFEATURE_XTILECFG 17+#define XFEATURE_XTILEDATA 18+#define XFEATURE_MASK_XTILECFG (1 << XFEATURE_XTILECFG)+#define XFEATURE_MASK_XTILEDATA (1 << XFEATURE_XTILEDATA)+#define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILECFG | XFEATURE_MASK_XTILEDATA)++#define ARCH_GET_XCOMP_PERM 0x1022+#define ARCH_REQ_XCOMP_PERM 0x1023++#endif+#include "bestla.h"++// As long as the compiler supports the ISA, we will enable it.+// Only the ISA you use in your project will be compiled.+#if defined(_MSC_VER) && defined(__INTEL_LLVM_COMPILER)+#define CompileAVX512F() defined(__AVX512F__)+#define CompileAVX512VNNI() defined(__AVX512VNNI__)+#define CompileAVX2() defined(__AVX2__) && defined(__F16C__) && defined(__FMA__)+#define CompileAVXVNNI() defined(__AVXVNNI__)+#define CompileAMX() defined(__AMX_TILE__)+#define CompileBF16() defined(__AVX512BF16__)+#define CompileFP16() defined(__AVX512FP16__)+#define CompileAMXBF16() (CompileAMX())+#define CompileAMXINT8() (CompileAMX())+#else+#define CompileAVX512F() BTLA_AVX512_FOUND+#define CompileAVX512VNNI() BTLA_AVX512_VNNI_FOUND+#define CompileAVX2() BTLA_AVX2_FOUND+#define CompileAVXVNNI() BTLA_AVX_VNNI_FOUND+#define CompileBF16() BTLA_AVX512_BF16_FOUND+#define CompileFP16() BTLA_AVX512_FP16_FOUND+#define CompileAMXBF16() BTLA_AMX_BF16_FOUND+#define CompileAMXFP16() BTLA_AMX_FP16_FOUND+#define CompileAMXINT8() BTLA_AMX_INT8_FOUND+#define CompileAMX() BTLA_AMX_BF16_FOUND+#endif++// called by launcher, time critical functions+#define TLACALL \+ template <BTLA_ISA ISA_T> \+ static inline++#define ISACALL \+ template <BTLA_ISA ISA_T> \+ static++// runtime auto-dispatch ISA, not time critical functions+#define AUTOCALL static++namespace bestla {+namespace utils {++template <typename T2, typename T1>+inline const T2 bit_cast(T1 i) {+ static_assert(sizeof(T1) == sizeof(T2), "Bit-casting must preserve size.");+ T2 o;+ memcpy(&o, &i, sizeof(T2));+ return o;+}++template <typename T>+inline uint32_t bitand_u32(const T& src, const T& src1) {+ return uint32_t(src) & uint32_t(src1);+}++struct bf16 {+ uint16_t x;+ union bf16f32 {+ float f32;+ unsigned int u;+ uint16_t bf16[2];+ };+ bf16() : x(0) {}++ explicit bf16(float vf32) { fromfloat(vf32); }++ float tofloat() const {+ bf16f32 tmp = {0.f};+ tmp.bf16[1] = x;+ return tmp.f32;+ }++ bf16& operator=(float val) {+ fromfloat(val);+ return *this;+ }++ operator float() const { return tofloat(); }++ static bf16 from_bin(const uint16_t x) {+ bf16 res;+ res.x = x;+ return res;+ }++ void fromfloat(float _v) {+ bf16f32 tmp = {0.f};+ tmp.f32 = _v;+ // See document of VCVTNEPS2BF16 in Intel® 64 and IA-32 Architectures Software Developer’s Manual Volume 2+ const auto lsb = tmp.bf16[1] & 1;+ tmp.u += 0x7fff + lsb;+ x = tmp.bf16[1];+ }+};++struct f8 {+ int8_t x;+ explicit f8(int8_t v) { x = v; }+ f8& operator=(int8_t v) {+ x = v;+ return *this;+ }++ inline float tofloat() const {+ int32_t r = x + 127;+ uint32_t tmp = bit_cast<uint32_t, int32_t>(r & 0xff);+ tmp <<= 23;+ return bit_cast<float, uint32_t>(tmp);+ }++ inline float mul(float src) const {+ auto scale = tofloat();+ return src * scale;+ }+};++struct f8e4m3 {+ int8_t x;+ explicit f8e4m3(int8_t v) { x = v; }+ f8e4m3& operator=(int8_t v) {+ x = v;+ return *this;+ }++ inline float tofloat() const {+ uint8_t value = bit_cast<uint8_t, int8_t>(x);+ uint32_t sign_revert = value;+ uint32_t e_revert = value;+ uint32_t mantissa_revert = value;+ sign_revert <<= 24;+ sign_revert &= 0x80000000;+ int constexpr ebits = 4;+ auto mantissabit = 7 - ebits;+ e_revert &= 0x7f;+ e_revert >>= mantissabit;+ e_revert = e_revert + 1 + 127 - static_cast<uint32_t>(std::pow(2, ebits - 1));+ e_revert <<= 23;+ mantissa_revert <<= (23 - mantissabit);+ mantissa_revert &= 0x007fffff;+ uint32_t tmp = sign_revert | e_revert | mantissa_revert;+ return bit_cast<float, uint32_t>(tmp);+ }++ operator float() const { return tofloat(); }++ inline float mul(float src) const {+ auto scale = tofloat();+ return src * scale;+ }+};++struct fp16 {+ uint16_t x;++ fp16() { x = 0; }+ explicit fp16(float val) { (*this) = val; }+ explicit fp16(bf16 val) { (*this) = static_cast<float>(val); }++ fp16& operator=(float val) {+ // round-to-nearest-even: add last bit after truncated mantissa+ const uint32_t b = bit_cast<uint32_t>(val) + 0x00001000;+ const uint32_t e = (b & 0x7F800000) >> 23; // exponent+ // mantissa; in line below: 0x007FF000 = 0x00800000-0x00001000 = decimal indicator flag - initial rounding+ const uint32_t m = b & 0x007FFFFF;+ // sign : normalized : denormalized : saturate++ this->x = static_cast<uint16_t>((b & 0x80000000) >> 16 | (e > 112) * ((((e - 112) << 10) & 0x7C00) | m >> 13) |+ ((e < 113) & (e > 101)) * ((((0x007FF000 + m) >> (125 - e)) + 1) >> 1) |+ (e > 143) * 0x7FFF);+ return *this;+ }+ explicit operator float() const {+ // IEEE-754 16-bit floating-point format (without infinity): 1-5-10, exp-15,+ // +-131008.0, +-6.1035156E-5, +-5.9604645E-8, 3.311 digits+ const uint32_t e = (x & 0x7C00) >> 10; // exponent+ const uint32_t m = (x & 0x03FF) << 13; // mantissa+ // evil log2 bit hack to count leading zeros in denormalized format+ const uint32_t v = bit_cast<uint32_t>(static_cast<float>(m)) >> 23;+ // sign : normalized : denormalized+ return bit_cast<float>((x & 0x8000) << 16 | (e != 0) * ((e + 112) << 23 | m) |+ ((e == 0) & (m != 0)) * ((v - 37) << 23 | ((m << (150 - v)) & 0x007FE000)));+ }+ explicit operator bf16() const {+ // Extract the exponent, and mantissa from the fp16 value.+ int exponent = x >> 10 & 0x1f;+ int mantissa = x & 0x3ff;++ // If the exponent is 0, the bf16 value is 0.+ if (exponent == 0) {+ return bf16();+ }+ // If the exponent is 31, the bf16 value is the sign bit plus 0x7fff.+ else if (exponent == 31) {+ bf16 res{};+ return bf16::from_bin(x | 0x7fff);+ }+ // Otherwise, the bf16 value is the sign bit plus the exponent minus 15,+ // followed by the mantissa.+ else {+ int sign = x & 0x8000;+ return bf16::from_bin(static_cast<uint16_t>(sign | (exponent + 128 - 16) << 7 | mantissa >> 3));+ }+ }+};++struct bit2x4 {+ uint8_t a : 2;+ uint8_t b : 2;+ uint8_t c : 2;+ uint8_t d : 2;+};++struct bit1x8 {+ uint8_t a : 1;+ uint8_t b : 1;+ uint8_t c : 1;+ uint8_t d : 1;+ uint8_t e : 1;+ uint8_t f : 1;+ uint8_t g : 1;+ uint8_t h : 1;+};++struct bit4x2 {+ uint8_t x : 4;+ uint8_t y : 4;+ bit4x2(int8_t v) : x(v), y(v) {}+ bit4x2() : x(0), y(0) {}+};++struct int4x2 : bit4x2 {+ int4x2(int8_t v) : bit4x2(v) {}+ int4x2() : bit4x2() {}+ static int8_t convert(int8_t src) {+ int32_t dst = src;+ dst = dst > 7 ? 7 : dst;+ dst = dst < -8 ? -8 : dst;+ return static_cast<int8_t>(dst);+ }+};++struct f4x2 : bit4x2 {+ f4x2(int8_t v) : bit4x2(v) {}+ f4x2() : bit4x2() {}+};++struct GemmProblem {+ int n;+ int dims[8]; // batch, m, n, k, kblock, ... reserved+ GemmProblem(int _n, int* _dims) {+ n = _n;+ for (size_t i = 0; i < n; i++) {+ dims[i] = _dims[i];+ }+ }+ GemmProblem(int batch, int _m, int _n, int _k) {+ n = 4;+ dims[0] = batch;+ dims[1] = _m;+ dims[2] = _n;+ dims[3] = _k;+ }+ GemmProblem(int batch, int _m, int _n, int _k, int _kblock) {+ n = 5;+ dims[0] = batch;+ dims[1] = _m;+ dims[2] = _n;+ dims[3] = _k;+ dims[4] = _kblock;+ }+};++template <typename ScaleT>+struct GemvParamB {+ uint8_t *b4ptr = 0, *b2ptr = 0, *b1ptr = 0;+ ScaleT* sptr = 0;+ int8_t* zpptr = 0;+ int nbits = 0;+ int ldzp = 0;+ int kpad = 0;+};++struct GemvParamA {+ uint8_t* aptr = 0;+ float* sptr = 0;+ uint8_t* zpptr = 0;+ int lda = 0;+ int ldzp = 0;+};++template <typename T>+inline constexpr BTLA_DTYPE bestla_dtype = std::is_same_v<T, double> ? BTLA_DTYPE::F64+ : std::is_same_v<T, float> ? BTLA_DTYPE::F32+ : std::is_same_v<T, utils::bf16> ? BTLA_DTYPE::BF16+ : std::is_same_v<T, utils::fp16> ? BTLA_DTYPE::F16+ : std::is_same_v<T, int8_t> ? BTLA_DTYPE::S8+ : std::is_same_v<T, uint8_t> ? BTLA_DTYPE::U8+ : std::is_same_v<T, int> ? BTLA_DTYPE::S32+ : std::is_same_v<T, f8> ? BTLA_DTYPE::F8_E8M0+ : (assert(0), BTLA_DTYPE::F32);+template <typename T>+inline constexpr const char* type_str = std::is_same_v<T, double> ? "double"+ : std::is_same_v<T, float> ? "float"+ : std::is_same_v<T, bf16> ? "bf16"+ : std::is_same_v<T, fp16> ? "fp16"+ : std::is_same_v<T, int8_t> ? "int8_t"+ : std::is_same_v<T, uint8_t> ? "uint8_t"+ : std::is_same_v<T, f8> ? "f8" // TODO(zhe): more f8 cases?+ : (assert(0), "undef");++inline const char* bestla_dtype_str(BTLA_DTYPE dtype) {+ switch (dtype) {+ case BTLA_DTYPE::F64:+ return "float64";+ case BTLA_DTYPE::F32:+ return "float32";+ case BTLA_DTYPE::F16:+ return "float16";+ case BTLA_DTYPE::BF16:+ return "bfloat16";+ case BTLA_DTYPE::F8_E4M3:+ return "fp8_e4m3";+ case BTLA_DTYPE::F8_E5M2:+ return "fp8_e5m2";+ case BTLA_DTYPE::F8_E3M4:+ return "fp8_e3m4";+ case BTLA_DTYPE::F8_E8M0:+ return "fp8_e8m0";+ case BTLA_DTYPE::S8:+ return "signed_int8";+ case BTLA_DTYPE::U8:+ return "unsigned_int8";+ case BTLA_DTYPE::S7_CLIP:+ return "int7_clip";+ case BTLA_DTYPE::S6_CLIP:+ return "int6_clip";+ case BTLA_DTYPE::S5_CLIP:+ return "int5_clip";+ case BTLA_DTYPE::S4_CLIP:+ return "int4_clip";+ case BTLA_DTYPE::S3_CLIP:+ return "int3_clip";+ case BTLA_DTYPE::S2_CLIP:+ return "int2_clip";+ case BTLA_DTYPE::S1_CLIP:+ return "int1_clip";+ case BTLA_DTYPE::F4_E2M1:+ return "fp4_e2m1";+ case BTLA_DTYPE::F4_BNB:+ return "fp4_bitsandbytes";+ case BTLA_DTYPE::F4_NF4:+ return "fp4_nf4";+ case BTLA_DTYPE::S32:+ return "signed_int32";+ case BTLA_DTYPE::U32:+ return "unsigned_int32";+ case BTLA_DTYPE::DQ8_BNB:+ return "dq8_bnb";+ default:+ return "ErrType";+ }+}++template <BTLA_DTYPE DT>+inline constexpr const char* dtype_str() {+ return bestla_dtype_str(DT);+}++inline constexpr uint32_t bestla_dtype_get_mask_val(const BTLA_DTYPE& t, const BTLA_DTYPE& mask,+ const BTLA_DTYPE& shift) {+ return (static_cast<uint32_t>(t) & static_cast<uint32_t>(mask)) >> static_cast<uint32_t>(shift);+}++inline constexpr size_t bestla_dtype_bits(const BTLA_DTYPE t) {+ return bestla_dtype_get_mask_val(t, BTLA_DTYPE::EleBitsMask, BTLA_DTYPE::EleBitsShift);+}++inline constexpr size_t bestla_dtype_bytes(const BTLA_DTYPE t) { return bestla_dtype_bits(t) >> 3; }++inline constexpr size_t bestla_dtype_type(const BTLA_DTYPE t) {+ return bestla_dtype_get_mask_val(t, BTLA_DTYPE::TypeMask, BTLA_DTYPE::TypeShift);+}++inline int bestla_dtype_get_f8_ebits(const BTLA_DTYPE t) {+ int ret = -1;+ switch (t) {+ case BTLA_DTYPE::F8_E4M3:+ ret = 4;+ break;+ case BTLA_DTYPE::F8_E5M2:+ ret = 5;+ break;+ default:+ assert(0);+ }+ return ret;+}++inline int bestla_dtype_get_f8_quant_mbits(const BTLA_DTYPE t) {+ int ret = -1;+ switch (t) {+ case BTLA_DTYPE::F8_E4M3:+ ret = 5;+ break;+ case BTLA_DTYPE::F8_E5M2:+ ret = 4;+ break;+ default:+ assert(0);+ }+ return ret;+}++inline float get_mxfp_maxnorm(const BTLA_DTYPE t, int ebits, int mantissa_bits) {+ auto emax = std::pow(2, ebits - 1);+ if (t == BTLA_DTYPE::F8_E5M2) emax -= 1;+ auto max_norm = std::pow(2, emax);+ if (t != BTLA_DTYPE::F8_E4M3) {+ max_norm *= ((std::pow(2, mantissa_bits - 1) - 1) / std::pow(2, mantissa_bits - 2));+ } else {+ max_norm *= 1.75;+ }+ return static_cast<float>(max_norm);+}++#ifndef _WIN32+static void request_perm_xtile_data() {+ unsigned long bitmask;+ long rc;++ rc = syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_PERM, XFEATURE_XTILEDATA);+ if (rc) fatal_error("XTILE_DATA request failed: %ld", rc);++ rc = syscall(SYS_arch_prctl, ARCH_GET_XCOMP_PERM, &bitmask);+ if (rc) fatal_error("prctl(ARCH_GET_XCOMP_PERM) error: %ld", rc);+#ifndef NDEBUG+ if (bitmask & XFEATURE_MASK_XTILE) printf("ARCH_REQ_XCOMP_PERM XTILE_DATA successful.\n");+#endif+}+#else+static void request_perm_xtile_data() {}+#endif++template <BTLA_ISA ISA_T>+class isa_base {+ public:+ static bool constexpr avx = ISA_T >= BTLA_ISA::AVX;+ static bool constexpr avx2 = ISA_T >= BTLA_ISA::AVX2;+ static bool constexpr avx512f = ISA_T >= BTLA_ISA::AVX512F;+ static bool constexpr avx512_vnni = ISA_T >= BTLA_ISA::AVX512_VNNI;+ static bool constexpr avx512_bf16 = ISA_T >= BTLA_ISA::AVX512_BF16;+ static bool constexpr avx512_fp16 = ISA_T >= BTLA_ISA::AVX512_FP16;+ static bool constexpr amx_bf16 = ISA_T >= BTLA_ISA::AMX_BF16;+ static bool constexpr amx_int8 = ISA_T >= BTLA_ISA::AMX_INT8;+ static bool constexpr amx_fp16 = ISA_T >= BTLA_ISA::AMX_FP16;+};++static inline int padto_le(int src, int padding) { return src / padding * padding; }++static inline int64_t padto_le(int64_t src, int64_t padding) { return src / padding * padding; }++static inline size_t padto_le(size_t src, int padding) { return src / size_t(padding) * size_t(padding); }++static inline int updiv(int a, int b) { return (a + b - 1) / b; }++static inline size_t updiv(size_t a, int b) { return (a + b - 1) / b; }++static inline int downdiv(int a, int b) { return a / b; }++static inline int remainsize(int pos, int size, int N) { return pos + N <= size ? N : size - pos; }+static inline size_t remainsize(size_t pos, size_t size, size_t N) { return pos + N <= size ? N : size - pos; }+static inline size_t remainsize(size_t pos, int size, size_t N) { return pos + N <= size ? N : size - pos; }++template <typename _SRCT, typename _DSTT>+static inline _DSTT cast(_SRCT _src) {+ return static_cast<_DSTT>(_src);+}++template <>+int8_t cast(float _src) {+ _src = roundf(_src);+ _src = std::min(_src, 127.f);+ _src = std::max(_src, -128.f);+ return static_cast<int8_t>(_src);+}++template <>+uint8_t cast(float _src) {+ _src += 0.5f;+ _src = std::min(_src, 255.f);+ _src = std::max(_src, 0.f);+ return static_cast<uint8_t>(_src);+}++template <>+int cast(float _src) {+ return int(roundf(_src));+}++template <>+float cast(bf16 _src) {+ return _src.tofloat();+}++template <>+bf16 cast(float _src) {+ bf16 tmp;+ tmp.fromfloat(_src);+ return tmp;+}++template <>+float cast(f8e4m3 _src) {+ return _src.tofloat();+}++template <typename _T>+void serialize(int8_t*& buf, _T _val) {+ *reinterpret_cast<_T*>(buf) = _val;+ buf += sizeof(_T);+}++template <typename _T>+_T deserialize(int8_t*& buf) {+ auto val = *reinterpret_cast<_T*>(buf);+ buf += sizeof(_T);+ return val;+}++template <typename T>+static inline T padto(T a, int b) {+ return (a + b - 1) / b * b;+}++template <int _Alignment, typename _T>+static inline _T* pointer_align(_T* src) {+ auto uptr = reinterpret_cast<uint64_t>(src);+ return reinterpret_cast<_T*>((uptr + _Alignment - 1) / _Alignment * _Alignment);+}++template <typename _T>+static inline _T* cpu_pointer_align(_T* src) {+ return pointer_align<64, _T>(src);+}++template <typename _T>+static inline _T* amalloc(size_t _size, size_t _alignment = 64) {+ if (_size == 0) {+ return nullptr;+ }+ auto psize = padto(_size * sizeof(_T), static_cast<int>(_alignment));+#ifdef _WIN32+ return reinterpret_cast<_T*>(_aligned_malloc(psize, _alignment));+#else+ return reinterpret_cast<_T*>(aligned_alloc(_alignment, psize));+#endif+}++static inline void afree(void* ptr) {+ if (ptr == nullptr) {+ return;+ }+#ifdef _WIN32+ _aligned_free(ptr);+#else+ free(ptr);+#endif+}++template <typename _T, int _Alignment = 64>+class aligned_vector {+ public:+ aligned_vector() : mRawsize(0), mPtr(nullptr), mAlignedsize(0) {}+ aligned_vector(size_t _size) { resize(_size); }+ aligned_vector(size_t _size, _T _val) {+ resize(_size);+ std::fill_n(mVec.begin(), mVec.size(), _val);+ }+ size_t size() { return mRawsize; }+ void resize(size_t size) {+ mRawsize = size;+ mAlignedsize = (mRawsize + _Alignment - 1) / _Alignment * _Alignment + _Alignment;+ if (size) {+ mVec.resize(mAlignedsize);+ auto uptr = reinterpret_cast<uint64_t>(mVec.data());+ mPtr = reinterpret_cast<_T*>((uptr + _Alignment - 1) / _Alignment * _Alignment);+ } else {+ mPtr = nullptr;+ }+ }++ template <typename T>+ aligned_vector<T> to() {+ aligned_vector<T> _new(size());+ for (size_t i = 0; i < size(); i++) {+ _new[i] = (T)mPtr[i];+ }+ return _new;+ }++ _T* data() const { return mPtr; }+ _T& operator[](size_t _n) noexcept { return mPtr[_n]; }++ protected:+ size_t mAlignedsize, mRawsize;+ std::vector<_T> mVec;+ _T* mPtr;+};++template <typename _T, int _Alignment = 64>+using avector = aligned_vector<_T, _Alignment>;++using milliseconds = std::chrono::milliseconds;+using nanoseconds = std::chrono::nanoseconds;+using microseconds = std::chrono::microseconds;+template <typename _DUR = std::chrono::milliseconds>+class timer {+ public:+ using sclock_t = std::chrono::steady_clock;+ using stime_point_t = std::chrono::time_point<sclock_t>;++ timer() { clear(); }++ void start() { startT = sclock_t::now(); }++ void clear() { startT = stime_point_t::min(); }++ bool null_state() { return startT == stime_point_t::min(); }++ float stop() { return static_cast<float>(std::chrono::duration_cast<_DUR>(sclock_t::now() - startT).count()); }++ stime_point_t startT;+};++template <typename T>+class minmax_statistics {+ public:+ minmax_statistics() { clear(); }++ void clear() {+ min_val = std::numeric_limits<T>::max();+ max_val = std::numeric_limits<T>::min();+ avg_val = 0;+ count = 0;+ }++ void add(T _val) {+ min_val = min_val > _val ? _val : min_val;+ max_val = max_val < _val ? _val : max_val;+ count += 1;+ avg_val = (avg_val * (count - 1) + _val) / count;+ }++ T min_val, max_val, avg_val;+ size_t count;+};++template <int _PRINT_CYCLE_MS = 100, typename _PRECISION = microseconds, typename _LOG_PRECISION = milliseconds>+class timer_statistics_logger {+ public:+ typedef timer<milliseconds> log_timer_t;+ timer_statistics_logger() {+ clear();+ log_ratio = static_cast<float>(std::chrono::duration_cast<_PRECISION>(_LOG_PRECISION(1)).count());+ }++ void clear() {+ statis.clear();+ logtm.clear();+ }++ void start() {+ if (logtm.null_state()) {+ logtm.start();+ }+ tm.start();+ }++ bool stop() {+ auto elapsed = tm.stop();+ statis.add(elapsed);+ if (logtm.stop() >= _PRINT_CYCLE_MS) {+ record();+ clear();+ logtm.start();+ return true;+ }+ return false;+ }++ bool add(float time) {+ statis.add(time);+ if (logtm.stop() >= _PRINT_CYCLE_MS) {+ record();+ clear();+ logtm.start();+ return true;+ }+ return false;+ }++ const char* get_log_str() {+ sprintf(str, "Min:%.4f, Max:%.4f, Average:%.4f", min_val, max_val, avg_val);+ return str;+ }+ float min_val, max_val, avg_val;++ void record() {+ if (statis.count) {+ min_val = statis.min_val / log_ratio;+ max_val = statis.max_val / log_ratio;+ avg_val = statis.avg_val / log_ratio;+ }+ }++ private:+ float log_ratio;+ char str[256];+ timer<_PRECISION> tm;+ minmax_statistics<float> statis;+ timer<milliseconds> logtm;+};++inline bool isFastExp() {+ static std::set<std::string> valid_env_vars = {"1", "TRUE", "ON", "On", "True"};+ if (getenv("BTLA_FAST_EXP") != nullptr) {+ if (valid_env_vars.count(getenv("BTLA_FAST_EXP")) != 0) {+ return true;+ } else {+ return false;+ }+ } else {+ return false;+ }+}+} // namespace utils++static float fp4_bnb_dequant_fp32_LUT alignas(64)[] = {+ 0.00000000f, 5.208333333e-03f, 0.66666667f, 1.00000000f, 0.33333333f,+ 0.50000000f, 0.16666667f, 0.25000000f, -1.f * 0.00000000f, -1.f * 5.208333333e-03f,+ -1.f * 0.66666667f, -1.f * 1.00000000f, -1.f * 0.33333333f, -1.f * 0.50000000f, -1.f * 0.16666667f,+ -1.f * 0.25000000f};++static float fp4_e2m1_dequant_fp32_LUT alignas(64)[] = {+ 0.f,+ 0.010416666666666666f,+ 0.16666666666666666f,+ 0.25f,+ 0.333333333333333f,+ 0.5f,+ 0.6666666666666f,+ 1.f,+ -1.f * 0.f,+ -1.f * 0.010416666666666666f,+ -1.f * 0.16666666666666666f,+ -1.f * 0.25f,+ -1.f * 0.333333333333333f,+ -1.f * 0.5f,+ -1.f * 0.6666666666666f,+ -1.f * 1.f,+};++static float nf4_dequant_fp32_LUT alignas(64)[] = {0.f,+ -0.6961928009986877f,+ -0.5250730514526367f,+ -0.39491748809814453f,+ -0.28444138169288635f,+ -0.18477343022823334f,+ -0.09105003625154495f,+ -1.f,+ 0.07958029955625534f,+ 0.16093020141124725f,+ 0.24611230194568634f,+ 0.33791524171829224f,+ 0.44070982933044434f,+ 0.5626170039176941f,+ 0.7229568362236023f,+ 1.0f};++// 8bit dynamic-tree-quantization map from bitsandbytes double-quant implementation.+// For more details pls refer+// (8-Bit Approximations for Parallelism in Deep Learning)[https://arxiv.org/abs/1511.04561]+static float dq8_bnb_LUT alignas(64)[] = {+ -0.99297f, -0.97891f, -0.96484f, -0.95078f, -0.93672f, -0.92266f, -0.90859f, -0.89453f, -0.88047f, -0.86641f,+ -0.85234f, -0.83828f, -0.82422f, -0.81016f, -0.79609f, -0.78203f, -0.76797f, -0.75391f, -0.73984f, -0.72578f,+ -0.71172f, -0.69766f, -0.68359f, -0.66953f, -0.65547f, -0.64141f, -0.62734f, -0.61328f, -0.59922f, -0.58516f,+ -0.57109f, -0.55703f, -0.54297f, -0.52891f, -0.51484f, -0.50078f, -0.48672f, -0.47266f, -0.45859f, -0.44453f,+ -0.43047f, -0.41641f, -0.40234f, -0.38828f, -0.37422f, -0.36016f, -0.34609f, -0.33203f, -0.31797f, -0.30391f,+ -0.28984f, -0.27578f, -0.26172f, -0.24766f, -0.23359f, -0.21953f, -0.20547f, -0.19141f, -0.17734f, -0.16328f,+ -0.14922f, -0.13516f, -0.12109f, -0.10703f, -0.09859f, -0.09578f, -0.09297f, -0.09016f, -0.08734f, -0.08453f,+ -0.08172f, -0.07891f, -0.07609f, -0.07328f, -0.07047f, -0.06766f, -0.06484f, -0.06203f, -0.05922f, -0.05641f,+ -0.05359f, -0.05078f, -0.04797f, -0.04516f, -0.04234f, -0.03953f, -0.03672f, -0.03391f, -0.03109f, -0.02828f,+ -0.02547f, -0.02266f, -0.01984f, -0.01703f, -0.01422f, -0.01141f, -0.00972f, -0.00916f, -0.00859f, -0.00803f,+ -0.00747f, -0.00691f, -0.00634f, -0.00578f, -0.00522f, -0.00466f, -0.00409f, -0.00353f, -0.00297f, -0.00241f,+ -0.00184f, -0.00128f, -0.00094f, -0.00083f, -0.00072f, -0.00061f, -0.00049f, -0.00038f, -0.00027f, -0.00016f,+ -0.00009f, -0.00007f, -0.00004f, -0.00002f, -0.00001f, -0.00000f, -0.00000f, 0.00000f, 0.00000f, 0.00000f,+ 0.00001f, 0.00002f, 0.00004f, 0.00007f, 0.00009f, 0.00016f, 0.00027f, 0.00038f, 0.00049f, 0.00061f,+ 0.00072f, 0.00083f, 0.00094f, 0.00128f, 0.00184f, 0.00241f, 0.00297f, 0.00353f, 0.00409f, 0.00466f,+ 0.00522f, 0.00578f, 0.00634f, 0.00691f, 0.00747f, 0.00803f, 0.00859f, 0.00916f, 0.00972f, 0.01141f,+ 0.01422f, 0.01703f, 0.01984f, 0.02266f, 0.02547f, 0.02828f, 0.03109f, 0.03391f, 0.03672f, 0.03953f,+ 0.04234f, 0.04516f, 0.04797f, 0.05078f, 0.05359f, 0.05641f, 0.05922f, 0.06203f, 0.06484f, 0.06766f,+ 0.07047f, 0.07328f, 0.07609f, 0.07891f, 0.08172f, 0.08453f, 0.08734f, 0.09016f, 0.09297f, 0.09578f,+ 0.09859f, 0.10703f, 0.12109f, 0.13516f, 0.14922f, 0.16328f, 0.17734f, 0.19141f, 0.20547f, 0.21953f,+ 0.23359f, 0.24766f, 0.26172f, 0.27578f, 0.28984f, 0.30391f, 0.31797f, 0.33203f, 0.34609f, 0.36016f,+ 0.37422f, 0.38828f, 0.40234f, 0.41641f, 0.43047f, 0.44453f, 0.45859f, 0.47266f, 0.48672f, 0.50078f,+ 0.51484f, 0.52891f, 0.54297f, 0.55703f, 0.57109f, 0.58516f, 0.59922f, 0.61328f, 0.62734f, 0.64141f,+ 0.65547f, 0.66953f, 0.68359f, 0.69766f, 0.71172f, 0.72578f, 0.73984f, 0.75391f, 0.76797f, 0.78203f,+ 0.79609f, 0.81016f, 0.82422f, 0.83828f, 0.85234f, 0.86641f, 0.88047f, 0.89453f, 0.90859f, 0.92266f,+ 0.93672f, 0.95078f, 0.96484f, 0.97891f, 0.99297f, 1.00000f};+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_wrapper.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_wrapper.hnew file mode 100644index 000000000..e3a7ec1cb--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/bestla_wrapper.h@@ -0,0 +1,1442 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include <thread>++#include "bestla_epilogue.h"+#include "bestla_gemm.h"+#include "bestla_gemm_kblock.h"+#include "bestla_prologue_a.h"+#include "bestla_prologue_b.h"+#include "bestla_utils.h"++namespace bestla {+namespace wrapper {+namespace gemv_nbits {+class S7 {+ public:+ static int constexpr NBits = 7;+ template <typename ScaleT>+ static inline utils::GemvParamB<ScaleT> createB(storage::gemm::StorageWeightKBlockNInteger* packedW) {+ auto isasym = packedW->IsAsym();+ auto bzptr = packedW->template ZPtr<int8_t>();+ int ld_scaleb = packedW->CStep();+ auto bwptr = packedW->template WPtr<uint8_t>();+ auto bit2_offset = packedW->mNPad * packedW->mKPad / 2;+ auto bit1_offset = packedW->mNPad * packedW->mKPad / 2 + packedW->mNPad * packedW->mKPad / 4;+ utils::GemvParamB<ScaleT> paramB{+ bwptr, bwptr + bit2_offset, bwptr + bit1_offset, packedW->template SPtr<ScaleT>(), isasym ? bzptr : nullptr,+ NBits, ld_scaleb, packedW->mKPad};+ return paramB;+ }+ template <typename ScaleT>+ static void updateBNStep(utils::GemvParamB<ScaleT>& paramB, int n_offset) {+ paramB.b4ptr += n_offset * paramB.kpad / 2;+ paramB.b2ptr += n_offset * paramB.kpad / 4;+ paramB.b1ptr += n_offset * paramB.kpad / 8;+ paramB.sptr += n_offset;+ if (paramB.zpptr) {+ paramB.zpptr += n_offset;+ }+ }+};++class S6 {+ public:+ static int constexpr NBits = 6;+ template <typename ScaleT>+ static inline utils::GemvParamB<ScaleT> createB(storage::gemm::StorageWeightKBlockNInteger* packedW) {+ auto isasym = packedW->IsAsym();+ auto bzptr = packedW->template ZPtr<int8_t>();+ int ld_scaleb = packedW->CStep();+ auto bwptr = packedW->template WPtr<uint8_t>();+ auto bit2_offset = packedW->mNPad * packedW->mKPad / 2;+ utils::GemvParamB<ScaleT> paramB{+ bwptr, bwptr + bit2_offset, nullptr, packedW->template SPtr<ScaleT>(), isasym ? bzptr : nullptr,+ NBits, ld_scaleb, packedW->mKPad};+ return paramB;+ }+ template <typename ScaleT>+ static void updateBNStep(utils::GemvParamB<ScaleT>& paramB, int n_offset) {+ paramB.b4ptr += n_offset * paramB.kpad / 2;+ paramB.b2ptr += n_offset * paramB.kpad / 4;+ paramB.sptr += n_offset;+ if (paramB.zpptr) {+ paramB.zpptr += n_offset;+ }+ }+};++class S5 {+ public:+ static int constexpr NBits = 5;+ template <typename ScaleT>+ static inline utils::GemvParamB<ScaleT> createB(storage::gemm::StorageWeightKBlockNInteger* packedW) {+ auto isasym = packedW->IsAsym();+ auto bzptr = packedW->template ZPtr<int8_t>();+ int ld_scaleb = packedW->CStep();+ auto bwptr = packedW->template WPtr<uint8_t>();+ auto bit1_offset = packedW->mNPad * packedW->mKPad / 2;+ utils::GemvParamB<ScaleT> paramB{+ bwptr, nullptr, bwptr + bit1_offset, packedW->template SPtr<ScaleT>(), isasym ? bzptr : nullptr,+ NBits, ld_scaleb, packedW->mKPad};+ return paramB;+ }+ template <typename ScaleT>+ static void updateBNStep(utils::GemvParamB<ScaleT>& paramB, int n_offset) {+ paramB.b4ptr += n_offset * paramB.kpad / 2;+ paramB.b1ptr += n_offset * paramB.kpad / 8;+ paramB.sptr += n_offset;+ if (paramB.zpptr) {+ paramB.zpptr += n_offset;+ }+ }+};++class S4 {+ public:+ static int constexpr NBits = 4;+ template <typename ScaleT>+ static inline utils::GemvParamB<ScaleT> createB(storage::gemm::StorageWeightKBlockNInteger* packedW) {+ auto isasym = packedW->IsAsym();+ auto bzptr = packedW->template ZPtr<int8_t>();+ int ld_scaleb = packedW->CStep();+ utils::GemvParamB<ScaleT> paramB{+ packedW->template WPtr<uint8_t>(), nullptr, nullptr, packedW->template SPtr<ScaleT>(),+ isasym ? bzptr : nullptr, NBits, ld_scaleb, packedW->mKPad};+ return paramB;+ }+ template <typename ScaleT>+ static void updateBNStep(utils::GemvParamB<ScaleT>& paramB, int n_offset) {+ paramB.b4ptr += n_offset * paramB.kpad / 2;+ paramB.sptr += n_offset;+ if (paramB.zpptr) {+ paramB.zpptr += n_offset;+ }+ }+};++class S3 {+ public:+ static int constexpr NBits = 3;+ template <typename ScaleT>+ static inline utils::GemvParamB<ScaleT> createB(storage::gemm::StorageWeightKBlockNInteger* packedW) {+ auto isasym = packedW->IsAsym();+ auto bzptr = packedW->template ZPtr<int8_t>();+ int ld_scaleb = packedW->CStep();+ auto bwptr = packedW->template WPtr<uint8_t>();+ auto bit1_offset = packedW->mNPad * packedW->mKPad / 4;+ utils::GemvParamB<ScaleT> paramB{+ nullptr, bwptr, bwptr + bit1_offset, packedW->template SPtr<ScaleT>(), isasym ? bzptr : nullptr,+ NBits, ld_scaleb, packedW->mKPad};+ return paramB;+ }+ template <typename ScaleT>+ static void updateBNStep(utils::GemvParamB<ScaleT>& paramB, int n_offset) {+ paramB.b2ptr += n_offset * paramB.kpad / 4;+ paramB.b1ptr += n_offset * paramB.kpad / 8;+ paramB.sptr += n_offset;+ if (paramB.zpptr) {+ paramB.zpptr += n_offset;+ }+ }+};++class S2 {+ public:+ static int constexpr NBits = 2;+ template <typename ScaleT>+ static inline utils::GemvParamB<ScaleT> createB(storage::gemm::StorageWeightKBlockNInteger* packedW) {+ auto isasym = packedW->IsAsym();+ auto bzptr = packedW->template ZPtr<int8_t>();+ int ld_scaleb = packedW->CStep();+ auto bwptr = packedW->template WPtr<uint8_t>();+ utils::GemvParamB<ScaleT> paramB{+ nullptr, bwptr, nullptr, packedW->template SPtr<ScaleT>(), isasym ? bzptr : nullptr,+ NBits, ld_scaleb, packedW->mKPad};+ return paramB;+ }+ template <typename ScaleT>+ static void updateBNStep(utils::GemvParamB<ScaleT>& paramB, int n_offset) {+ paramB.b2ptr += n_offset * paramB.kpad / 4;+ paramB.sptr += n_offset;+ if (paramB.zpptr) {+ paramB.zpptr += n_offset;+ }+ }+};++class S1 {+ public:+ static int constexpr NBits = 1;+ template <typename ScaleT>+ static inline utils::GemvParamB<ScaleT> createB(storage::gemm::StorageWeightKBlockNInteger* packedW) {+ auto isasym = packedW->IsAsym();+ auto bzptr = packedW->template ZPtr<int8_t>();+ int ld_scaleb = packedW->CStep();+ auto bwptr = packedW->template WPtr<uint8_t>();+ utils::GemvParamB<ScaleT> paramB{+ nullptr, nullptr, bwptr, packedW->template SPtr<ScaleT>(), isasym ? bzptr : nullptr,+ NBits, ld_scaleb, packedW->mKPad};+ return paramB;+ }+ template <typename ScaleT>+ static void updateBNStep(utils::GemvParamB<ScaleT>& paramB, int n_offset) {+ paramB.b1ptr += n_offset * paramB.kpad / 8;+ paramB.sptr += n_offset;+ if (paramB.zpptr) {+ paramB.zpptr += n_offset;+ }+ }+};++class NBitsHelper {+ public:+ template <typename ScaleT>+ static inline utils::GemvParamB<ScaleT> createB(storage::gemm::StorageWeightKBlockNInteger* packedW) {+ if (packedW->mDType == BTLA_DTYPE::S4_CLIP) {+ return S4::createB<ScaleT>(packedW);+ }+ if (packedW->mDType == BTLA_DTYPE::S3_CLIP) {+ return S3::createB<ScaleT>(packedW);+ }+ if (packedW->mDType == BTLA_DTYPE::S5_CLIP) {+ return S5::createB<ScaleT>(packedW);+ }+ if (packedW->mDType == BTLA_DTYPE::S2_CLIP) {+ return S2::createB<ScaleT>(packedW);+ }+ if (packedW->mDType == BTLA_DTYPE::S6_CLIP) {+ return S6::createB<ScaleT>(packedW);+ }+ if (packedW->mDType == BTLA_DTYPE::S7_CLIP) {+ return S7::createB<ScaleT>(packedW);+ }+ if (packedW->mDType == BTLA_DTYPE::S1_CLIP) {+ return S1::createB<ScaleT>(packedW);+ }+ assert(0);+ return utils::GemvParamB<ScaleT>();+ }+ template <typename ScaleT>+ static void updateBNStep(utils::GemvParamB<ScaleT>& paramB, int n_offset) {+ if (paramB.nbits == 4) {+ return S4::updateBNStep<ScaleT>(paramB, n_offset);+ }+ if (paramB.nbits == 3) {+ return S3::updateBNStep<ScaleT>(paramB, n_offset);+ }+ if (paramB.nbits == 5) {+ return S5::updateBNStep<ScaleT>(paramB, n_offset);+ }+ if (paramB.nbits == 2) {+ return S2::updateBNStep<ScaleT>(paramB, n_offset);+ }+ if (paramB.nbits == 6) {+ return S6::updateBNStep<ScaleT>(paramB, n_offset);+ }+ if (paramB.nbits == 7) {+ return S7::updateBNStep<ScaleT>(paramB, n_offset);+ }+ if (paramB.nbits == 1) {+ return S1::updateBNStep<ScaleT>(paramB, n_offset);+ }+ assert(0);+ }+};++class GemvWeight {+ public:+ template <typename ScaleT>+ static inline utils::GemvParamB<ScaleT> createB(const storage::gemm::StorageWeightNInt* packedW) {+ auto bzptr = packedW->corr_.zptr();+ int ld_scaleb = packedW->corr_.step_;+ auto bwptr = packedW->buf_.buf_ptr_;+ auto nbits = (int)utils::bestla_dtype_bits(packedW->info_.dtype_);+ utils::GemvParamB<ScaleT> paramB{nullptr, nullptr, nullptr, (ScaleT*)packedW->corr_.scale_buf_.buf_ptr_,+ bzptr, nbits, ld_scaleb, packedW->info_.kpad_};+ auto bit2_offset = packedW->info_.npad_ * packedW->info_.kpad_ / 2;+ auto bit1_offset =+ packedW->info_.npad_ * packedW->info_.kpad_ / 2 + packedW->info_.npad_ * packedW->info_.kpad_ / 4;+ if (nbits & 0x4) {+ paramB.b4ptr = (uint8_t*)bwptr;+ bwptr += packedW->info_.npad_ * packedW->info_.kpad_ / 2;+ }+ if (nbits & 0x2) {+ paramB.b2ptr = (uint8_t*)bwptr;+ bwptr += packedW->info_.npad_ * packedW->info_.kpad_ / 4;+ }+ if (nbits & 0x1) {+ paramB.b1ptr = (uint8_t*)bwptr;+ }+ return paramB;+ }+ template <typename ScaleT>+ static void updateBNStep(utils::GemvParamB<ScaleT>& paramB, int n_offset) {+ if (paramB.b4ptr) paramB.b4ptr += n_offset * paramB.kpad / 2;+ if (paramB.b2ptr) paramB.b2ptr += n_offset * paramB.kpad / 4;+ if (paramB.b1ptr) paramB.b1ptr += n_offset * paramB.kpad / 8;+ paramB.sptr += n_offset;+ if (paramB.zpptr) paramB.zpptr += n_offset;+ }+};++} // namespace gemv_nbits++namespace gemm {+template <class _GemmCore_T, template <class _T> class _PrologueA_T, template <class _T> class _PrologueB_T,+ class _Epilogue_T>+class LauncherBase {+ public:+ using GemmCore = _GemmCore_T;+ static constexpr BTLA_ISA ISA = GemmCore::ISA;+ using PrologueA = _PrologueA_T<GemmCore>;+ using PrologueB = _PrologueB_T<GemmCore>;+ using Epilogue = _Epilogue_T;+ using AType = typename GemmCore::AType;+ using AParam = typename PrologueA::Param;+ using BType = typename GemmCore::BType;+ using BParam = typename PrologueB::Param;+ using CType = typename GemmCore::CType;+ using EpiParam = typename Epilogue::Param;+ struct Param {+ const utils::GemmProblem problem;+ const AParam paramA;+ const BParam paramB;+ const EpiParam paramC;+ };++ class GEMVWrapper {+ public:+ static constexpr bool support() {+ if constexpr (!std::is_same_v<PrologueB, prologue_b::gemm::WeightKBlockNInteger<_GemmCore_T>>) {+ return false;+ }+ if constexpr (!std::is_same_v<PrologueA, prologue_a::gemm::ShuffleActivationKBlockBaseF32<_GemmCore_T>> &&+ !std::is_same_v<PrologueA, prologue_a::gemm::ActivationKBlockBaseF32<_GemmCore_T>> &&+ !std::is_same_v<PrologueA, prologue_a::gemm::ActivationF32KBlockQuantize<_GemmCore_T>> &&+ !std::is_same_v<PrologueA, prologue_a::gemm::ShuffleActivationKBlockQuantizeF32<_GemmCore_T>> &&+ !std::is_same_v<PrologueA, prologue_a::gemm::ActivationBase<_GemmCore_T>>) {+ return false;+ }++ if constexpr (GemmCore::ISA == BTLA_ISA::AVX2) {+#if CompileAVX2()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_FP32) {+ static_assert(GemmCore::PACK_ROW == 1);+ return true;+ }+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_INT32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+#endif+ }+ if constexpr (GemmCore::ISA == BTLA_ISA::AVX512_VNNI || GemmCore::ISA == BTLA_ISA::AMX_INT8) {+#if CompileAVX512VNNI()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_INT32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+#endif+ }+ if constexpr (GemmCore::ISA == BTLA_ISA::AVX_VNNI) {+#if CompileAVXVNNI()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_INT32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+#endif+ }+ if constexpr (GemmCore::ISA == BTLA_ISA::AVX512F) {+#if CompileAVX512F()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_FP32) {+ static_assert(GemmCore::PACK_ROW == 1);+ return true;+ }+#endif+ }+ if constexpr (GemmCore::ISA == BTLA_ISA::AVX512BW) {+#if CompileAVX512F()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_INT32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+#endif+ }+ return false;+ }+ static bool constexpr Reg32 = GemmCore::ISA >= BTLA_ISA::AVX512F;+ static int constexpr MaxGemvM = Reg32 ? 4 : 4;+ static bool implemented(const Param& _param) {+ bool impl = true;+ impl &= _param.paramB.packedW->mDType == BTLA_DTYPE::S4_CLIP ||+ _param.paramB.packedW->mDType == BTLA_DTYPE::S3_CLIP ||+ _param.paramB.packedW->mDType == BTLA_DTYPE::S7_CLIP ||+ _param.paramB.packedW->mDType == BTLA_DTYPE::S6_CLIP ||+ _param.paramB.packedW->mDType == BTLA_DTYPE::S5_CLIP ||+ _param.paramB.packedW->mDType == BTLA_DTYPE::S1_CLIP ||+ _param.paramB.packedW->mDType == BTLA_DTYPE::S2_CLIP;+ if constexpr (support()) {+ impl &= _param.paramB.packedW->mCorrection.mScaT == BTLA_DTYPE::F32 ||+ _param.paramB.packedW->mCorrection.mScaT == BTLA_DTYPE::F16 ||+ _param.paramB.packedW->mCorrection.mScaT == BTLA_DTYPE::BF16;+ }++ impl &= _param.problem.dims[1] <= MaxGemvM;+ return impl;+ }++ template <typename ScaleT, int MTILE>+ static void gemv_kblock(const Param& _param, const parallel::gemm::ThreadProblemBase& _config) {+ if constexpr (support()) {+ auto constexpr TmpSize = 16 * 1024LL;+ auto constexpr CSize = 8 * 1024LL;+ auto StackTmp_ = alloca(TmpSize + CSize);+ auto StackTmp = utils::cpu_pointer_align<void>(StackTmp_);+ auto tmpc_ptr = reinterpret_cast<float*>((char*)StackTmp + TmpSize);+ static_assert(CSize >= (MTILE * GemmCore::NTILE * sizeof(float)));+ utils::GemvParamB<ScaleT> paramB = gemv_nbits::NBitsHelper::template createB<ScaleT>(_param.paramB.packedW);+ int m = _param.problem.dims[1];+ int n = _param.problem.dims[2];+ int k = _param.problem.dims[3];+ int kblocksize = _param.problem.dims[4];+ gemv_nbits::NBitsHelper::template updateBNStep<ScaleT>(paramB, _config.loc[1]);+ int size_padded = utils::padto_le(_config.size[1], GemmCore::NTILE);+ int in = 0;+ for (; in < size_padded; in += GemmCore::NTILE) {+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_INT32) {+ utils::GemvParamA paramA{+ _param.paramA.quan->template APtr<uint8_t>(), _param.paramA.quan->template SPtr<float>(),+ _param.paramA.quan->template ZPtr<uint8_t>(), _param.paramA.quan->mKPad, _param.paramA.quan->CStep()};+ kernel::wrapper::GEMVWoqNBits::forward_u8s8_fp32<ISA, ScaleT, GemmCore::NTILE, MTILE>(+ paramA, paramB, tmpc_ptr, GemmCore::NTILE, k, kblocksize, StackTmp, TmpSize);+ Epilogue::Fp32Epi::template forward<ISA>(tmpc_ptr, GemmCore::NTILE, 0, _config.loc[1] + in, MTILE,+ GemmCore::NTILE, _param.paramC.param2, StackTmp, TmpSize);+ } else {+ const float* Aptr = _param.paramA.A;+ if constexpr (std::is_same_v<PrologueA, prologue_a::gemm::ShuffleActivationKBlockBaseF32<_GemmCore_T>>) {+ if (_param.paramA.reordered && _param.paramA.reordered->template APtr<float>()) {+ Aptr = _param.paramA.reordered->template APtr<float>();+ }+ }+ kernel::wrapper::GEMVWoqNBits::forward_fp32_fp32<ISA, ScaleT, GemmCore::NTILE, MTILE>(+ Aptr, _param.paramA.lda, paramB, tmpc_ptr, GemmCore::NTILE, k, kblocksize, StackTmp, TmpSize);+ Epilogue::template forward<ISA>(tmpc_ptr, GemmCore::NTILE, 0, _config.loc[1] + in, MTILE, GemmCore::NTILE,+ _param.paramC, StackTmp, TmpSize);+ }+ gemv_nbits::NBitsHelper::template updateBNStep<ScaleT>(paramB, GemmCore::NTILE);+ }+ if (size_padded != _config.size[1]) {+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_INT32) {+ utils::GemvParamA paramA{+ _param.paramA.quan->template APtr<uint8_t>(), _param.paramA.quan->template SPtr<float>(),+ _param.paramA.quan->template ZPtr<uint8_t>(), _param.paramA.quan->mKPad, _param.paramA.quan->CStep()};+ kernel::wrapper::GEMVWoqNBits::forward_u8s8_fp32<ISA, ScaleT, GemmCore::NTILE, MTILE>(+ paramA, paramB, tmpc_ptr, GemmCore::NTILE, k, kblocksize, StackTmp, TmpSize);+ Epilogue::Fp32Epi::template forward<ISA>(tmpc_ptr, GemmCore::NTILE, 0, _config.loc[1] + in, MTILE,+ (_config.size[1] - in), _param.paramC.param2, StackTmp, TmpSize);+ } else {+ const float* Aptr = _param.paramA.A;+ if constexpr (std::is_same_v<PrologueA, prologue_a::gemm::ShuffleActivationKBlockBaseF32<_GemmCore_T>>) {+ if (_param.paramA.reordered && _param.paramA.reordered->template APtr<float>()) {+ Aptr = _param.paramA.reordered->template APtr<float>();+ }+ }+ kernel::wrapper::GEMVWoqNBits::forward_fp32_fp32<ISA, ScaleT, GemmCore::NTILE, MTILE>(+ Aptr, _param.paramA.lda, paramB, tmpc_ptr, GemmCore::NTILE, k, kblocksize, StackTmp, TmpSize);+ Epilogue::template forward<ISA>(tmpc_ptr, GemmCore::NTILE, 0, _config.loc[1] + in, MTILE,+ (_config.size[1] - in), _param.paramC, StackTmp, TmpSize);+ }+ }+ }+ }++ static void gemv(const Param& _param, const parallel::gemm::ThreadProblemBase& _config) {+ if constexpr (support()) {+ assert(_param.problem.dims[4] > 0);+ auto& m = _param.problem.dims[1];+ if (_param.paramB.packedW->SDtype() == BTLA_DTYPE::F32) {+ if (m == 1) gemv_kblock<float, 1>(_param, _config);+ if (m == 2) gemv_kblock<float, 2>(_param, _config);+ if (m == 3) gemv_kblock<float, 3>(_param, _config);+ if (m == 4) gemv_kblock<float, 4>(_param, _config);+ if constexpr (Reg32) {+ if (m == 5) gemv_kblock<float, 5>(_param, _config);+ if (m == 6) gemv_kblock<float, 6>(_param, _config);+ if (m == 7) gemv_kblock<float, 7>(_param, _config);+ if (m == 8) gemv_kblock<float, 8>(_param, _config);+ }+ } else if (_param.paramB.packedW->SDtype() == BTLA_DTYPE::BF16) {+ if (m == 1) gemv_kblock<utils::bf16, 1>(_param, _config);+ if (m == 2) gemv_kblock<utils::bf16, 2>(_param, _config);+ if (m == 3) gemv_kblock<utils::bf16, 3>(_param, _config);+ if (m == 4) gemv_kblock<utils::bf16, 4>(_param, _config);+ if constexpr (Reg32) {+ if (m == 5) gemv_kblock<utils::bf16, 5>(_param, _config);+ if (m == 6) gemv_kblock<utils::bf16, 6>(_param, _config);+ if (m == 7) gemv_kblock<utils::bf16, 7>(_param, _config);+ if (m == 8) gemv_kblock<utils::bf16, 8>(_param, _config);+ }+ } else if (_param.paramB.packedW->SDtype() == BTLA_DTYPE::F16) {+ if (m == 1) gemv_kblock<utils::fp16, 1>(_param, _config);+ if (m == 2) gemv_kblock<utils::fp16, 2>(_param, _config);+ if (m == 3) gemv_kblock<utils::fp16, 3>(_param, _config);+ if (m == 4) gemv_kblock<utils::fp16, 4>(_param, _config);+ if constexpr (Reg32) {+ if (m == 5) gemv_kblock<utils::fp16, 5>(_param, _config);+ if (m == 6) gemv_kblock<utils::fp16, 6>(_param, _config);+ if (m == 7) gemv_kblock<utils::fp16, 7>(_param, _config);+ if (m == 8) gemv_kblock<utils::fp16, 8>(_param, _config);+ }+ }+ }+ }+ };++ static void run(const Param& _param, const parallel::gemm::ThreadProblemBase& _config) {+ if (GEMVWrapper::support() && GEMVWrapper::implemented(_param)) {+ GEMVWrapper::gemv(_param, _config);+ } else {+ gemm(_param, _config);+ }+ bestla::kernel::wrapper::ZeroReg::forward();+ }++ protected:+ static void gemm(const Param& _param, const parallel::gemm::ThreadProblemBase& _config) {+ GemmCore::configure(_config.size[0], _config.size[1], _param.problem.dims[3]);+ auto StackTmp = alloca(_config.stacksize);+ auto tmpB = reinterpret_cast<BType*>(StackTmp);+ tmpB = utils::cpu_pointer_align(tmpB);+ auto tmpA = reinterpret_cast<AType*>(tmpB + static_cast<size_t>(_config.block[1]) * _config.block[2]);+ tmpA = utils::cpu_pointer_align(tmpA);+ auto tmpC = reinterpret_cast<CType*>(tmpA + static_cast<size_t>(GemmCore::MTILE) * _config.block[2]);+ tmpC = utils::cpu_pointer_align(tmpC);+ auto tmpCache = (void*)(tmpC + static_cast<size_t>(_config.block[0]) * _config.block[1]);+ tmpCache = utils::cpu_pointer_align(tmpCache);+ for (int itern = 0; itern < _config.size[1]; itern += _config.block[1]) {+ int n_remain = utils::remainsize(itern, _config.size[1], _config.block[1]);+ for (int iterm = 0; iterm < _config.size[0]; iterm += _config.block[0]) {+ int m_remain = utils::remainsize(iterm, _config.size[0], _config.block[0]);+ run_block(_param, _config, iterm, itern, m_remain, n_remain, tmpA, tmpB, tmpC, tmpCache);+ }+ }+ }++ static void run_block(const Param& _param, const parallel::gemm::ThreadProblemBase& _config, int blk_m, int blk_n,+ int blk_msize, int blk_nsize, AType* tmpA, BType* tmpB, CType* tmpC, void* tmpcache) {+ int n_padded = utils::padto(blk_nsize, GemmCore::NTILE);+ auto& K = _param.problem.dims[3];+ for (int iterk = 0; iterk < _param.problem.dims[3]; iterk += _config.block[2]) {+ int k_remain = utils::remainsize(iterk, K, _config.block[2]);+ int k_padded = utils::padto(k_remain, GemmCore::KTILE);+ int k_paddedle = utils::padto_le(k_remain, GemmCore::KTILE);+ auto bptr_cache = tmpB;+ int bcache_step = 0;+ PrologueB::template getWeight<ISA>(&bptr_cache, &bcache_step, k_padded, n_padded, iterk, _config.loc[1] + blk_n,+ _param.paramB, tmpcache, _config.tmpcachesize);+ int bcache_stride = bcache_step * sizeof(BType);+ for (int i = 0; i < blk_msize; i += GemmCore::MTILE) {+ int m_remain = utils::remainsize(i, blk_msize, GemmCore::MTILE);+ auto cptr_cache = tmpC + i * _config.block[1];+ int ccache_stride = _config.block[1] * sizeof(CType);+ if (k_paddedle) {+ AType* aptr_cache = tmpA;+ int acache_step = 0;+ PrologueA::template getActivation<ISA>(&aptr_cache, &acache_step, _param.paramA, m_remain, k_paddedle,+ (blk_m + i + _config.loc[0]), iterk, tmpcache, _config.tmpcachesize);+ GemmCore::forward(aptr_cache, bptr_cache, cptr_cache, m_remain, n_padded, k_paddedle,+ acache_step * sizeof(AType), bcache_stride, ccache_stride, iterk, tmpcache,+ _config.tmpcachesize);+ }+ int k_tail = k_remain - k_paddedle;+ if (k_tail) {+ AType* aptr_cache = tmpA;+ int acache_step = 0;+ PrologueA::template getActivation<ISA>(&aptr_cache, &acache_step, _param.paramA, m_remain, k_tail,+ (blk_m + i + _config.loc[0]), iterk + k_paddedle, tmpcache,+ _config.tmpcachesize);+ GemmCore::forward(aptr_cache, bptr_cache + k_paddedle * GemmCore::NTILE, cptr_cache, m_remain, n_padded,+ GemmCore::KTILE, acache_step * sizeof(AType), bcache_stride, ccache_stride,+ iterk + k_paddedle, tmpcache, _config.tmpcachesize);+ }+ }+ }+ Epilogue::template forward<ISA>(tmpC, _config.block[1], (_config.loc[0] + blk_m), _config.loc[1] + blk_n, blk_msize,+ blk_nsize, _param.paramC, tmpcache, _config.tmpcachesize);+ }+};++template <class _GemmCore_T, template <class _T> class _PrologueA_T, template <class _T> class _PrologueB_T,+ class _Epilogue_T>+class LauncherIntKBlock {+ public:+ using GemmCore = _GemmCore_T;+ static constexpr BTLA_ISA ISA = GemmCore::ISA;+ using PrologueA = _PrologueA_T<GemmCore>;+ using PrologueB = _PrologueB_T<GemmCore>;+ using Epilogue = _Epilogue_T;+ using AType = typename GemmCore::AType;+ using AParam = typename PrologueA::Param;+ using BType = typename GemmCore::BType;+ using BParam = typename PrologueB::Param;+ using CType = typename GemmCore::CType;+ using EpiParam = typename Epilogue::Param;+ using AccType = float;+ struct Param {+ const utils::GemmProblem problem;+ const AParam paramA;+ const BParam paramB;+ const EpiParam paramC;+ };++ class GEMVWrapper {+ public:+ static constexpr bool support() {+ if constexpr (!std::is_same_v<PrologueB, prologue_b::gemm::WeightKBlockNInteger<_GemmCore_T>>) {+ return false;+ }+ if constexpr (!std::is_same_v<PrologueA, prologue_a::gemm::ActivationF32KBlockQuantize<_GemmCore_T>> &&+ !std::is_same_v<PrologueA, prologue_a::gemm::ShuffleActivationKBlockQuantizeF32<_GemmCore_T>>) {+ return false;+ }+ if constexpr (GemmCore::ISA == BTLA_ISA::AVX_VNNI) {+#if CompileAVXVNNI()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_FP32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_SS_FP32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+#endif+ }+ if constexpr (GemmCore::ISA == BTLA_ISA::AVX2) {+#if CompileAVX2()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_FP32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_SS_FP32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+#endif+ }+ if constexpr (GemmCore::ISA == BTLA_ISA::AVX512BW) {+#if CompileAVX512F()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_FP32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+#endif+ }+ if constexpr (GemmCore::ISA == BTLA_ISA::AVX512_VNNI || GemmCore::ISA == BTLA_ISA::AMX_INT8) {+#if CompileAVX512VNNI()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_FP32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_SS_FP32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+#endif+ }+ return false;+ }+ static bool constexpr Reg32 = GemmCore::ISA >= BTLA_ISA::AVX512F;+ static int constexpr MaxGemvM = Reg32 ? 4 : 4;++ static bool implemented(const Param& _param) {+ bool impl = true;+ impl &= _param.paramB.packedW->mDType == BTLA_DTYPE::S4_CLIP ||+ _param.paramB.packedW->mDType == BTLA_DTYPE::S7_CLIP ||+ _param.paramB.packedW->mDType == BTLA_DTYPE::S6_CLIP ||+ _param.paramB.packedW->mDType == BTLA_DTYPE::S5_CLIP ||+ _param.paramB.packedW->mDType == BTLA_DTYPE::S3_CLIP ||+ _param.paramB.packedW->mDType == BTLA_DTYPE::S1_CLIP ||+ _param.paramB.packedW->mDType == BTLA_DTYPE::S2_CLIP;+ impl &= _param.paramB.packedW->mCorrection.mScaT == BTLA_DTYPE::F32 ||+ _param.paramB.packedW->mCorrection.mScaT == BTLA_DTYPE::F16 ||+ _param.paramB.packedW->mCorrection.mScaT == BTLA_DTYPE::BF16;+ impl &= _param.problem.dims[1] <= MaxGemvM;+ return impl;+ }++ template <typename ScaleT, int MTILE>+ static void gemv_kblock(const Param& _param, const parallel::gemm::ThreadProblemBase& _config) {+ if constexpr (support()) {+ auto constexpr TmpSize = 16 * 1024LL;+ auto constexpr CSize = 8 * 1024LL;+ static_assert(CSize >= (MTILE * GemmCore::NTILE * sizeof(float)));+ auto StackTmp_ = alloca(TmpSize + CSize);+ auto StackTmp = utils::cpu_pointer_align<void>(StackTmp_);+ auto tmpc_ptr = reinterpret_cast<CType*>((char*)StackTmp + TmpSize);+ utils::GemvParamB<ScaleT> paramB = gemv_nbits::NBitsHelper::template createB<ScaleT>(_param.paramB.packedW);+ utils::GemvParamA paramA{+ _param.paramA.quan->template APtr<uint8_t>(), _param.paramA.quan->template SPtr<float>(),+ _param.paramA.quan->template ZPtr<uint8_t>(), _param.paramA.quan->mKPad, _param.paramA.quan->CStep()};++ int m = _param.problem.dims[1];+ int n = _param.problem.dims[2];+ int k = _param.problem.dims[3];+ int kblocksize = _param.problem.dims[4];+ gemv_nbits::NBitsHelper::template updateBNStep<ScaleT>(paramB, _config.loc[1]);+ int size_padded = utils::padto_le(_config.size[1], GemmCore::NTILE);+ int in = 0;+ for (; in < size_padded; in += GemmCore::NTILE) {+ if constexpr (std::is_same_v<AType, uint8_t>) {+ kernel::wrapper::GEMVWoqNBits::forward_u8s8_fp32<ISA, ScaleT, GemmCore::NTILE, MTILE>(+ paramA, paramB, tmpc_ptr, GemmCore::NTILE, k, kblocksize, StackTmp, TmpSize);+ } else if constexpr (std::is_same_v<AType, int8_t>) {+ kernel::wrapper::GEMVWoqNBits::forward_s8s8_fp32<ISA, ScaleT, GemmCore::NTILE, MTILE>(+ paramA, paramB, tmpc_ptr, GemmCore::NTILE, k, kblocksize, StackTmp, TmpSize);+ }+ Epilogue::template forward<ISA>(tmpc_ptr, GemmCore::NTILE, 0, _config.loc[1] + in, MTILE, GemmCore::NTILE,+ _param.paramC, StackTmp, TmpSize);+ gemv_nbits::NBitsHelper::template updateBNStep<ScaleT>(paramB, GemmCore::NTILE);+ }+ if (size_padded != _config.size[1]) {+ if constexpr (std::is_same_v<AType, uint8_t>) {+ kernel::wrapper::GEMVWoqNBits::forward_u8s8_fp32<ISA, ScaleT, GemmCore::NTILE, MTILE>(+ paramA, paramB, tmpc_ptr, GemmCore::NTILE, k, kblocksize, StackTmp, TmpSize);+ } else if constexpr (std::is_same_v<AType, int8_t>) {+ kernel::wrapper::GEMVWoqNBits::forward_s8s8_fp32<ISA, ScaleT, GemmCore::NTILE, MTILE>(+ paramA, paramB, tmpc_ptr, GemmCore::NTILE, k, kblocksize, StackTmp, TmpSize);+ }+ Epilogue::template forward<ISA>(tmpc_ptr, GemmCore::NTILE, 0, _config.loc[1] + in, MTILE,+ (_config.size[1] - in), _param.paramC, StackTmp, TmpSize);+ }+ }+ }++ static void gemv(const Param& _param, const parallel::gemm::ThreadProblemBase& _config) {+ if constexpr (support()) {+ auto& m = _param.problem.dims[1];+ if (_param.paramB.packedW->SDtype() == BTLA_DTYPE::F32) {+ if (m == 1) gemv_kblock<float, 1>(_param, _config);+ if (m == 2) gemv_kblock<float, 2>(_param, _config);+ if (m == 3) gemv_kblock<float, 3>(_param, _config);+ if (m == 4) gemv_kblock<float, 4>(_param, _config);+ if constexpr (Reg32) {+ if (m == 5) gemv_kblock<float, 5>(_param, _config);+ if (m == 6) gemv_kblock<float, 6>(_param, _config);+ if (m == 7) gemv_kblock<float, 7>(_param, _config);+ if (m == 8) gemv_kblock<float, 8>(_param, _config);+ }+ } else if (_param.paramB.packedW->SDtype() == BTLA_DTYPE::BF16) {+ if (m == 1) gemv_kblock<utils::bf16, 1>(_param, _config);+ if (m == 2) gemv_kblock<utils::bf16, 2>(_param, _config);+ if (m == 3) gemv_kblock<utils::bf16, 3>(_param, _config);+ if (m == 4) gemv_kblock<utils::bf16, 4>(_param, _config);+ if constexpr (Reg32) {+ if (m == 5) gemv_kblock<utils::bf16, 5>(_param, _config);+ if (m == 6) gemv_kblock<utils::bf16, 6>(_param, _config);+ if (m == 7) gemv_kblock<utils::bf16, 7>(_param, _config);+ if (m == 8) gemv_kblock<utils::bf16, 8>(_param, _config);+ }+ } else if (_param.paramB.packedW->SDtype() == BTLA_DTYPE::F16) {+ if (m == 1) gemv_kblock<utils::fp16, 1>(_param, _config);+ if (m == 2) gemv_kblock<utils::fp16, 2>(_param, _config);+ if (m == 3) gemv_kblock<utils::fp16, 3>(_param, _config);+ if (m == 4) gemv_kblock<utils::fp16, 4>(_param, _config);+ if constexpr (Reg32) {+ if (m == 5) gemv_kblock<utils::fp16, 5>(_param, _config);+ if (m == 6) gemv_kblock<utils::fp16, 6>(_param, _config);+ if (m == 7) gemv_kblock<utils::fp16, 7>(_param, _config);+ if (m == 8) gemv_kblock<utils::fp16, 8>(_param, _config);+ }+ }+ }+ }+ };++ static void run(const Param& _param, const parallel::gemm::ThreadProblemBase& _config) {+ if (GEMVWrapper::support() && GEMVWrapper::implemented(_param)) {+ GEMVWrapper::gemv(_param, _config);+ } else {+ gemm(_param, _config);+ }+ bestla::kernel::wrapper::ZeroReg::forward();+ }++ protected:+ static void gemm(const Param& _param, const parallel::gemm::ThreadProblemBase& _config) {+ GemmCore::configure(_config.size[0], _config.size[1], _param.problem.dims[3]);+ auto StackTmp = alloca(_config.stacksize);+ auto tmpB = reinterpret_cast<BType*>(StackTmp);+ tmpB = utils::cpu_pointer_align(tmpB);+ auto tmpA = reinterpret_cast<AType*>(tmpB + static_cast<size_t>(_config.block[1]) * _config.block[2]);+ tmpA = utils::cpu_pointer_align(tmpA);+ auto tmpC = reinterpret_cast<AccType*>(tmpA + static_cast<size_t>(GemmCore::MTILE) * _config.block[2]);+ tmpC = utils::cpu_pointer_align(tmpC);+ auto tmpCache = reinterpret_cast<int8_t*>(tmpC + static_cast<size_t>(_config.block[0]) * _config.block[1]);+ tmpCache = utils::cpu_pointer_align(tmpCache);+ auto& KBlock = _param.problem.dims[4];+ assert(_config.block[2] >= KBlock);+ for (int itern = 0; itern < _config.size[1]; itern += _config.block[1]) {+ int n_remain = utils::remainsize(itern, _config.size[1], _config.block[1]);+ for (int iterm = 0; iterm < _config.size[0]; iterm += _config.block[0]) {+ int m_remain = utils::remainsize(iterm, _config.size[0], _config.block[0]);+ run_block(_param, _config, iterm, itern, m_remain, n_remain, tmpA, tmpB, tmpC, tmpCache);+ }+ }+ }++ // _config.block[2]%kblock==0+ // _config.block[2]>=kblock+ static void run_block(const Param& _param, const parallel::gemm::ThreadProblemBase& _config, int blk_m, int blk_n,+ int blk_msize, int blk_nsize, AType* tmpA, BType* tmpB, AccType* tmpC, int8_t* tmpcache) {+ int n_padded = utils::padto(blk_nsize, GemmCore::NTILE);+ auto& K = _param.problem.dims[3];+ auto& KBlock = _param.problem.dims[4];+ assert(_config.block[2] % KBlock == 0);+ assert(_config.block[2] % GemmCore::KTILE == 0);+ // GemmCore: int8+int8=int32=>dequant to fp32+ // accumulate to tmpC+ // extra parameters: zpA, scaleA, scaleB, reduceB+ // zpA scaleA: [MTILE,kblk_perstep]+ // scaleB reduceB: [kblk_perstep, NStep]+ int kblk_perstep = utils::updiv(_config.block[2], KBlock);+ int tmp_ldsb = _config.block[1];+ int tmp_ldsa = kblk_perstep;+ auto zpA = reinterpret_cast<AType*>(tmpcache);+ zpA = utils::cpu_pointer_align(zpA);+ auto scaleA = reinterpret_cast<float*>(zpA + GemmCore::MTILE * tmp_ldsa);+ scaleA = utils::cpu_pointer_align(scaleA);+ auto scaleB = reinterpret_cast<float*>(scaleA + GemmCore::MTILE * tmp_ldsa);+ scaleB = utils::cpu_pointer_align(scaleB);+ auto reduceB = reinterpret_cast<int32_t*>(scaleB + _config.block[1] * tmp_ldsa);+ reduceB = utils::cpu_pointer_align(reduceB);+ auto tmp_ = reinterpret_cast<int8_t*>(reduceB + _config.block[1] * tmp_ldsa);+ tmp_ = utils::cpu_pointer_align(tmp_);++ for (int iterk = 0; iterk < K; iterk += _config.block[2]) {+ int k_remain = utils::remainsize(iterk, K, _config.block[2]);+ int k_padded = utils::padto(k_remain, GemmCore::KTILE);+ auto bptr_cache = tmpB;+ int bcache_step = 0;+ int ldsb_cache = tmp_ldsb;+ auto scaleB_cache = scaleB;+ auto reduceB_cache = reduceB;+ PrologueB::template getWeight<ISA>(&bptr_cache, &bcache_step, k_padded, n_padded, iterk, _config.loc[1] + blk_n,+ _param.paramB, tmp_, _config.tmpcachesize);+ PrologueB::template getScale<ISA>(&scaleB_cache, &ldsb_cache, k_padded, n_padded, iterk, _config.loc[1] + blk_n,+ _param.paramB, tmp_, _config.tmpcachesize);+ PrologueB::template getReduce<ISA>(&reduceB_cache, &ldsb_cache, k_padded, n_padded, iterk, _config.loc[1] + blk_n,+ _param.paramB, tmp_, _config.tmpcachesize);+ int bcache_stride = bcache_step * sizeof(BType);+ for (int i = 0; i < blk_msize; i += GemmCore::MTILE) {+ int m_remain = utils::remainsize(i, blk_msize, GemmCore::MTILE);+ auto cptr_cache = tmpC + i * _config.block[1];+ int ccache_stride = _config.block[1] * sizeof(CType);+ auto aptr_cache = tmpA;+ int acache_step = k_padded;+ auto zpA_cache = zpA;+ auto scaleA_cache = scaleA;+ int ldsa_cache = tmp_ldsa;+ PrologueA::template getActivation<ISA>(&aptr_cache, &acache_step, _param.paramA, m_remain, k_padded,+ (blk_m + i + _config.loc[0]), iterk, tmp_, _config.tmpcachesize);+ PrologueA::template getZp<ISA>(&zpA_cache, &ldsa_cache, _param.paramA, m_remain, k_padded,+ (blk_m + i + _config.loc[0]), iterk, tmp_, _config.tmpcachesize);+ PrologueA::template getScale<ISA>(&scaleA_cache, &ldsa_cache, _param.paramA, m_remain, k_padded,+ (blk_m + i + _config.loc[0]), iterk, tmp_, _config.tmpcachesize);+ GemmCore::forward(aptr_cache, bptr_cache, cptr_cache, zpA_cache, scaleA_cache, ldsa_cache, scaleB_cache,+ reduceB_cache, ldsb_cache, m_remain, n_padded, k_padded, KBlock, acache_step * sizeof(AType),+ bcache_stride, ccache_stride, iterk, tmp_, _config.tmpcachesize);+ }+ }+ Epilogue::template forward<ISA>(tmpC, _config.block[1], (_config.loc[0] + blk_m), _config.loc[1] + blk_n, blk_msize,+ blk_nsize, _param.paramC, tmpcache, _config.tmpcachesize);+ }+};++template <class _GemmCore_T, template <class _T> class _PrologueA_T, template <class _T> class _PrologueB_T,+ class _Epilogue_T>+class LauncherBaseV2 {+ public:+ using GemmCore = _GemmCore_T;+ static constexpr BTLA_ISA ISA = GemmCore::ISA;+ using PrologueA = _PrologueA_T<GemmCore>;+ using PrologueB = _PrologueB_T<GemmCore>;+ using Epilogue = _Epilogue_T;+ using AType = typename GemmCore::AType;+ using AParam = typename PrologueA::Param;+ using BType = typename GemmCore::BType;+ using BParam = typename PrologueB::Param;+ using CType = typename GemmCore::CType;+ using EpiParam = typename Epilogue::Param;+ struct Param {+ const utils::GemmProblem problem;+ const AParam paramA;+ const BParam paramB;+ const EpiParam paramC;+ };++ class GEMVWrapper {+ public:+ static constexpr bool support() {+ if constexpr (!std::is_same_v<PrologueB, prologue_b::gemm::WeightKBlockNInt<_GemmCore_T>>) {+ return false;+ }+ if constexpr (!std::is_same_v<PrologueA, prologue_a::gemm::ShuffleActivationKBlockBaseF32<_GemmCore_T>> &&+ !std::is_same_v<PrologueA, prologue_a::gemm::ActivationKBlockBaseF32<_GemmCore_T>> &&+ !std::is_same_v<PrologueA, prologue_a::gemm::ActivationF32KBlockQuantize<_GemmCore_T>> &&+ !std::is_same_v<PrologueA, prologue_a::gemm::ShuffleActivationKBlockQuantizeF32<_GemmCore_T>> &&+ !std::is_same_v<PrologueA, prologue_a::gemm::ActivationBase<_GemmCore_T>>) {+ return false;+ }++ if constexpr (GemmCore::ISA == BTLA_ISA::AVX2) {+#if CompileAVX2()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_FP32) {+ static_assert(GemmCore::PACK_ROW == 1);+ return true;+ }+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_INT32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+#endif+ }+ if constexpr (GemmCore::ISA == BTLA_ISA::AVX512_VNNI || GemmCore::ISA == BTLA_ISA::AMX_INT8) {+#if CompileAVX512VNNI()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_INT32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+#endif+ }+ if constexpr (GemmCore::ISA == BTLA_ISA::AVX_VNNI) {+#if CompileAVXVNNI()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_INT32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+#endif+ }+ if constexpr (GemmCore::ISA == BTLA_ISA::AVX512F) {+#if CompileAVX512F()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_FP32) {+ static_assert(GemmCore::PACK_ROW == 1);+ return true;+ }+#endif+ }+ if constexpr (GemmCore::ISA == BTLA_ISA::AVX512BW) {+#if CompileAVX512F()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_INT32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+#endif+ }+ return false;+ }+ static bool constexpr Reg32 = GemmCore::ISA >= BTLA_ISA::AVX512F;+ static int constexpr MaxGemvM = Reg32 ? 4 : 4;+ static bool implemented(const Param& _param) {+ bool impl = true;+ impl &= _param.paramB.stor_->info_.dtype_ == BTLA_DTYPE::S4_CLIP ||+ _param.paramB.stor_->info_.dtype_ == BTLA_DTYPE::S3_CLIP ||+ _param.paramB.stor_->info_.dtype_ == BTLA_DTYPE::S7_CLIP ||+ _param.paramB.stor_->info_.dtype_ == BTLA_DTYPE::S6_CLIP ||+ _param.paramB.stor_->info_.dtype_ == BTLA_DTYPE::S5_CLIP ||+ _param.paramB.stor_->info_.dtype_ == BTLA_DTYPE::S1_CLIP ||+ _param.paramB.stor_->info_.dtype_ == BTLA_DTYPE::S2_CLIP;+ if constexpr (support()) {+ impl &= _param.paramB.stor_->corr_.sdtype_ == BTLA_DTYPE::F32 ||+ _param.paramB.stor_->corr_.sdtype_ == BTLA_DTYPE::F16 ||+ _param.paramB.stor_->corr_.sdtype_ == BTLA_DTYPE::BF16;+ }++ impl &= _param.problem.dims[1] <= MaxGemvM;+ return impl;+ }++ template <typename ScaleT, int MTILE>+ static void gemv_kblock(const Param& _param, const parallel::gemm::ThreadProblemBase& _config) {+ if constexpr (support()) {+ auto constexpr TmpSize = 16 * 1024LL;+ auto constexpr CSize = 8 * 1024LL;+ auto StackTmp_ = alloca(TmpSize + CSize);+ auto StackTmp = utils::cpu_pointer_align<void>(StackTmp_);+ auto tmpc_ptr = reinterpret_cast<float*>((char*)StackTmp + TmpSize);+ static_assert(CSize >= (MTILE * GemmCore::NTILE * sizeof(float)));+ utils::GemvParamB<ScaleT> paramB = gemv_nbits::GemvWeight::template createB<ScaleT>(_param.paramB.stor_);+ int m = _param.problem.dims[1];+ int n = _param.problem.dims[2];+ int k = _param.problem.dims[3];+ int kblocksize = _param.problem.dims[4];+ gemv_nbits::GemvWeight::template updateBNStep<ScaleT>(paramB, _config.loc[1]);+ int size_padded = utils::padto_le(_config.size[1], GemmCore::NTILE);+ int in = 0;+ for (; in < size_padded; in += GemmCore::NTILE) {+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_INT32) {+ utils::GemvParamA paramA{+ _param.paramA.quan->template APtr<uint8_t>(), _param.paramA.quan->template SPtr<float>(),+ _param.paramA.quan->template ZPtr<uint8_t>(), _param.paramA.quan->mKPad, _param.paramA.quan->CStep()};+ kernel::wrapper::GEMVWoqNBits::forward_u8s8_fp32<ISA, ScaleT, GemmCore::NTILE, MTILE>(+ paramA, paramB, tmpc_ptr, GemmCore::NTILE, k, kblocksize, StackTmp, TmpSize);+ Epilogue::Fp32Epi::template forward<ISA>(tmpc_ptr, GemmCore::NTILE, 0, _config.loc[1] + in, MTILE,+ GemmCore::NTILE, _param.paramC.param2, StackTmp, TmpSize);+ } else {+ const float* Aptr = _param.paramA.A;+ if constexpr (std::is_same_v<PrologueA, prologue_a::gemm::ShuffleActivationKBlockBaseF32<_GemmCore_T>>) {+ if (_param.paramA.reordered && _param.paramA.reordered->template APtr<float>()) {+ Aptr = _param.paramA.reordered->template APtr<float>();+ }+ }+ kernel::wrapper::GEMVWoqNBits::forward_fp32_fp32<ISA, ScaleT, GemmCore::NTILE, MTILE>(+ Aptr, _param.paramA.lda, paramB, tmpc_ptr, GemmCore::NTILE, k, kblocksize, StackTmp, TmpSize);+ Epilogue::template forward<ISA>(tmpc_ptr, GemmCore::NTILE, 0, _config.loc[1] + in, MTILE, GemmCore::NTILE,+ _param.paramC, StackTmp, TmpSize);+ }+ gemv_nbits::NBitsHelper::template updateBNStep<ScaleT>(paramB, GemmCore::NTILE);+ }+ if (size_padded != _config.size[1]) {+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_INT32) {+ utils::GemvParamA paramA{+ _param.paramA.quan->template APtr<uint8_t>(), _param.paramA.quan->template SPtr<float>(),+ _param.paramA.quan->template ZPtr<uint8_t>(), _param.paramA.quan->mKPad, _param.paramA.quan->CStep()};+ kernel::wrapper::GEMVWoqNBits::forward_u8s8_fp32<ISA, ScaleT, GemmCore::NTILE, MTILE>(+ paramA, paramB, tmpc_ptr, GemmCore::NTILE, k, kblocksize, StackTmp, TmpSize);+ Epilogue::Fp32Epi::template forward<ISA>(tmpc_ptr, GemmCore::NTILE, 0, _config.loc[1] + in, MTILE,+ (_config.size[1] - in), _param.paramC.param2, StackTmp, TmpSize);+ } else {+ const float* Aptr = _param.paramA.A;+ if constexpr (std::is_same_v<PrologueA, prologue_a::gemm::ShuffleActivationKBlockBaseF32<_GemmCore_T>>) {+ if (_param.paramA.reordered && _param.paramA.reordered->template APtr<float>()) {+ Aptr = _param.paramA.reordered->template APtr<float>();+ }+ }+ kernel::wrapper::GEMVWoqNBits::forward_fp32_fp32<ISA, ScaleT, GemmCore::NTILE, MTILE>(+ Aptr, _param.paramA.lda, paramB, tmpc_ptr, GemmCore::NTILE, k, kblocksize, StackTmp, TmpSize);+ Epilogue::template forward<ISA>(tmpc_ptr, GemmCore::NTILE, 0, _config.loc[1] + in, MTILE,+ (_config.size[1] - in), _param.paramC, StackTmp, TmpSize);+ }+ }+ }+ }++ static void gemv(const Param& _param, const parallel::gemm::ThreadProblemBase& _config) {+ if constexpr (support()) {+ assert(_param.problem.dims[4] > 0);+ auto& m = _param.problem.dims[1];+ if (_param.paramB.stor_->corr_.sdtype_ == BTLA_DTYPE::F32) {+ if (m == 1) gemv_kblock<float, 1>(_param, _config);+ if (m == 2) gemv_kblock<float, 2>(_param, _config);+ if (m == 3) gemv_kblock<float, 3>(_param, _config);+ if (m == 4) gemv_kblock<float, 4>(_param, _config);+ if constexpr (Reg32) {+ if (m == 5) gemv_kblock<float, 5>(_param, _config);+ if (m == 6) gemv_kblock<float, 6>(_param, _config);+ if (m == 7) gemv_kblock<float, 7>(_param, _config);+ if (m == 8) gemv_kblock<float, 8>(_param, _config);+ }+ } else if (_param.paramB.stor_->corr_.sdtype_ == BTLA_DTYPE::BF16) {+ if (m == 1) gemv_kblock<utils::bf16, 1>(_param, _config);+ if (m == 2) gemv_kblock<utils::bf16, 2>(_param, _config);+ if (m == 3) gemv_kblock<utils::bf16, 3>(_param, _config);+ if (m == 4) gemv_kblock<utils::bf16, 4>(_param, _config);+ if constexpr (Reg32) {+ if (m == 5) gemv_kblock<utils::bf16, 5>(_param, _config);+ if (m == 6) gemv_kblock<utils::bf16, 6>(_param, _config);+ if (m == 7) gemv_kblock<utils::bf16, 7>(_param, _config);+ if (m == 8) gemv_kblock<utils::bf16, 8>(_param, _config);+ }+ } else if (_param.paramB.stor_->corr_.sdtype_ == BTLA_DTYPE::F16) {+ if (m == 1) gemv_kblock<utils::fp16, 1>(_param, _config);+ if (m == 2) gemv_kblock<utils::fp16, 2>(_param, _config);+ if (m == 3) gemv_kblock<utils::fp16, 3>(_param, _config);+ if (m == 4) gemv_kblock<utils::fp16, 4>(_param, _config);+ if constexpr (Reg32) {+ if (m == 5) gemv_kblock<utils::fp16, 5>(_param, _config);+ if (m == 6) gemv_kblock<utils::fp16, 6>(_param, _config);+ if (m == 7) gemv_kblock<utils::fp16, 7>(_param, _config);+ if (m == 8) gemv_kblock<utils::fp16, 8>(_param, _config);+ }+ }+ }+ }+ };++ static void run(const Param& _param, const parallel::gemm::ThreadProblemBase& _config) {+ if (GEMVWrapper::support() && GEMVWrapper::implemented(_param)) {+ GEMVWrapper::gemv(_param, _config);+ } else {+ gemm(_param, _config);+ }+ bestla::kernel::wrapper::ZeroReg::forward();+ }++ protected:+ static void gemm(const Param& _param, const parallel::gemm::ThreadProblemBase& _config) {+ GemmCore::configure(_config.size[0], _config.size[1], _param.problem.dims[3]);+ auto StackTmp = alloca(_config.stacksize);+ auto tmpB = reinterpret_cast<BType*>(StackTmp);+ tmpB = utils::cpu_pointer_align(tmpB);+ auto tmpA = reinterpret_cast<AType*>(tmpB + static_cast<size_t>(_config.block[1]) * _config.block[2]);+ tmpA = utils::cpu_pointer_align(tmpA);+ auto tmpC = reinterpret_cast<CType*>(tmpA + static_cast<size_t>(GemmCore::MTILE) * _config.block[2]);+ tmpC = utils::cpu_pointer_align(tmpC);+ auto tmpCache = (void*)(tmpC + static_cast<size_t>(_config.block[0]) * _config.block[1]);+ tmpCache = utils::cpu_pointer_align(tmpCache);+ for (int itern = 0; itern < _config.size[1]; itern += _config.block[1]) {+ int n_remain = utils::remainsize(itern, _config.size[1], _config.block[1]);+ for (int iterm = 0; iterm < _config.size[0]; iterm += _config.block[0]) {+ int m_remain = utils::remainsize(iterm, _config.size[0], _config.block[0]);+ run_block(_param, _config, iterm, itern, m_remain, n_remain, tmpA, tmpB, tmpC, tmpCache);+ }+ }+ }++ static void run_block(const Param& _param, const parallel::gemm::ThreadProblemBase& _config, int blk_m, int blk_n,+ int blk_msize, int blk_nsize, AType* tmpA, BType* tmpB, CType* tmpC, void* tmpcache) {+ int n_padded = utils::padto(blk_nsize, GemmCore::NTILE);+ auto& K = _param.problem.dims[3];+ for (int iterk = 0; iterk < _param.problem.dims[3]; iterk += _config.block[2]) {+ int k_remain = utils::remainsize(iterk, K, _config.block[2]);+ int k_padded = utils::padto(k_remain, GemmCore::KTILE);+ int k_paddedle = utils::padto_le(k_remain, GemmCore::KTILE);+ auto bptr_cache = tmpB;+ int bcache_step = 0;+ PrologueB::template get_weight<ISA>(&bptr_cache, &bcache_step, k_padded, n_padded, iterk, _config.loc[1] + blk_n,+ _param.paramB, tmpcache, _config.tmpcachesize);+ int bcache_stride = bcache_step * sizeof(BType);+ for (int i = 0; i < blk_msize; i += GemmCore::MTILE) {+ int m_remain = utils::remainsize(i, blk_msize, GemmCore::MTILE);+ auto cptr_cache = tmpC + i * _config.block[1];+ int ccache_stride = _config.block[1] * sizeof(CType);+ if (k_paddedle) {+ AType* aptr_cache = tmpA;+ int acache_step = 0;+ PrologueA::template getActivation<ISA>(&aptr_cache, &acache_step, _param.paramA, m_remain, k_paddedle,+ (blk_m + i + _config.loc[0]), iterk, tmpcache, _config.tmpcachesize);+ GemmCore::forward(aptr_cache, bptr_cache, cptr_cache, m_remain, n_padded, k_paddedle,+ acache_step * sizeof(AType), bcache_stride, ccache_stride, iterk, tmpcache,+ _config.tmpcachesize);+ }+ int k_tail = k_remain - k_paddedle;+ if (k_tail) {+ AType* aptr_cache = tmpA;+ int acache_step = 0;+ PrologueA::template getActivation<ISA>(&aptr_cache, &acache_step, _param.paramA, m_remain, k_tail,+ (blk_m + i + _config.loc[0]), iterk + k_paddedle, tmpcache,+ _config.tmpcachesize);+ GemmCore::forward(aptr_cache, bptr_cache + k_paddedle * GemmCore::NTILE, cptr_cache, m_remain, n_padded,+ GemmCore::KTILE, acache_step * sizeof(AType), bcache_stride, ccache_stride,+ iterk + k_paddedle, tmpcache, _config.tmpcachesize);+ }+ }+ }+ Epilogue::template forward<ISA>(tmpC, _config.block[1], (_config.loc[0] + blk_m), _config.loc[1] + blk_n, blk_msize,+ blk_nsize, _param.paramC, tmpcache, _config.tmpcachesize);+ }+};++template <class _GemmCore_T, template <class _T> class _PrologueA_T, template <class _T> class _PrologueB_T,+ class _Epilogue_T>+class LauncherIntKBlockV2 {+ public:+ using GemmCore = _GemmCore_T;+ static constexpr BTLA_ISA ISA = GemmCore::ISA;+ using PrologueA = _PrologueA_T<GemmCore>;+ using PrologueB = _PrologueB_T<GemmCore>;+ using Epilogue = _Epilogue_T;+ using AType = typename GemmCore::AType;+ using AParam = typename PrologueA::Param;+ using BType = typename GemmCore::BType;+ using BParam = typename PrologueB::Param;+ using CType = typename GemmCore::CType;+ using EpiParam = typename Epilogue::Param;+ using AccType = float;+ struct Param {+ const utils::GemmProblem problem;+ const AParam paramA;+ const BParam paramB;+ const EpiParam paramC;+ };++ class GEMVWrapper {+ public:+ static constexpr bool support() {+ if constexpr (!std::is_same_v<PrologueB, prologue_b::gemm::WeightKBlockNInt<_GemmCore_T>>) {+ return false;+ }+ if constexpr (!std::is_same_v<PrologueA, prologue_a::gemm::ActivationF32KBlockQuantize<_GemmCore_T>> &&+ !std::is_same_v<PrologueA, prologue_a::gemm::ShuffleActivationKBlockQuantizeF32<_GemmCore_T>>) {+ return false;+ }+ if constexpr (GemmCore::ISA == BTLA_ISA::AVX_VNNI) {+#if CompileAVXVNNI()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_FP32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_SS_FP32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+#endif+ }+ if constexpr (GemmCore::ISA == BTLA_ISA::AVX2) {+#if CompileAVX2()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_FP32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_SS_FP32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+#endif+ }+ if constexpr (GemmCore::ISA == BTLA_ISA::AVX512BW) {+#if CompileAVX512F()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_FP32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+#endif+ }+ if constexpr (GemmCore::ISA == BTLA_ISA::AVX512_VNNI || GemmCore::ISA == BTLA_ISA::AMX_INT8) {+#if CompileAVX512VNNI()+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_US_FP32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+ if constexpr (GemmCore::COMP == bestla::gemm::CompType::COMP_INT8_SS_FP32) {+ static_assert(GemmCore::PACK_ROW == 4);+ return true;+ }+#endif+ }+ return false;+ }+ static bool constexpr Reg32 = GemmCore::ISA >= BTLA_ISA::AVX512F;+ static int constexpr MaxGemvM = Reg32 ? 4 : 4;++ static bool implemented(const Param& _param) {+ bool impl = true;+ impl &= _param.paramB.stor_->info_.dtype_ == BTLA_DTYPE::S4_CLIP ||+ _param.paramB.stor_->info_.dtype_ == BTLA_DTYPE::S3_CLIP ||+ _param.paramB.stor_->info_.dtype_ == BTLA_DTYPE::S7_CLIP ||+ _param.paramB.stor_->info_.dtype_ == BTLA_DTYPE::S6_CLIP ||+ _param.paramB.stor_->info_.dtype_ == BTLA_DTYPE::S5_CLIP ||+ _param.paramB.stor_->info_.dtype_ == BTLA_DTYPE::S1_CLIP ||+ _param.paramB.stor_->info_.dtype_ == BTLA_DTYPE::S2_CLIP;+ if constexpr (support()) {+ impl &= _param.paramB.stor_->corr_.sdtype_ == BTLA_DTYPE::F32 ||+ _param.paramB.stor_->corr_.sdtype_ == BTLA_DTYPE::F16 ||+ _param.paramB.stor_->corr_.sdtype_ == BTLA_DTYPE::BF16;+ }+ impl &= _param.problem.dims[1] <= MaxGemvM;+ return impl;+ }++ template <typename ScaleT, int MTILE>+ static void gemv_kblock(const Param& _param, const parallel::gemm::ThreadProblemBase& _config) {+ if constexpr (support()) {+ auto constexpr TmpSize = 16 * 1024LL;+ auto constexpr CSize = 8 * 1024LL;+ static_assert(CSize >= (MTILE * GemmCore::NTILE * sizeof(float)));+ auto StackTmp_ = alloca(TmpSize + CSize);+ auto StackTmp = utils::cpu_pointer_align<void>(StackTmp_);+ auto tmpc_ptr = reinterpret_cast<CType*>((char*)StackTmp + TmpSize);+ utils::GemvParamB<ScaleT> paramB = gemv_nbits::GemvWeight::template createB<ScaleT>(_param.paramB.stor_);+ utils::GemvParamA paramA{+ _param.paramA.quan->template APtr<uint8_t>(), _param.paramA.quan->template SPtr<float>(),+ _param.paramA.quan->template ZPtr<uint8_t>(), _param.paramA.quan->mKPad, _param.paramA.quan->CStep()};++ int m = _param.problem.dims[1];+ int n = _param.problem.dims[2];+ int k = _param.problem.dims[3];+ int kblocksize = _param.problem.dims[4];+ gemv_nbits::GemvWeight::template updateBNStep<ScaleT>(paramB, _config.loc[1]);+ int size_padded = utils::padto_le(_config.size[1], GemmCore::NTILE);+ int in = 0;+ for (; in < size_padded; in += GemmCore::NTILE) {+ if constexpr (std::is_same_v<AType, uint8_t>) {+ kernel::wrapper::GEMVWoqNBits::forward_u8s8_fp32<ISA, ScaleT, GemmCore::NTILE, MTILE>(+ paramA, paramB, tmpc_ptr, GemmCore::NTILE, k, kblocksize, StackTmp, TmpSize);+ } else if constexpr (std::is_same_v<AType, int8_t>) {+ kernel::wrapper::GEMVWoqNBits::forward_s8s8_fp32<ISA, ScaleT, GemmCore::NTILE, MTILE>(+ paramA, paramB, tmpc_ptr, GemmCore::NTILE, k, kblocksize, StackTmp, TmpSize);+ }+ Epilogue::template forward<ISA>(tmpc_ptr, GemmCore::NTILE, 0, _config.loc[1] + in, MTILE, GemmCore::NTILE,+ _param.paramC, StackTmp, TmpSize);+ gemv_nbits::NBitsHelper::template updateBNStep<ScaleT>(paramB, GemmCore::NTILE);+ }+ if (size_padded != _config.size[1]) {+ if constexpr (std::is_same_v<AType, uint8_t>) {+ kernel::wrapper::GEMVWoqNBits::forward_u8s8_fp32<ISA, ScaleT, GemmCore::NTILE, MTILE>(+ paramA, paramB, tmpc_ptr, GemmCore::NTILE, k, kblocksize, StackTmp, TmpSize);+ } else if constexpr (std::is_same_v<AType, int8_t>) {+ kernel::wrapper::GEMVWoqNBits::forward_s8s8_fp32<ISA, ScaleT, GemmCore::NTILE, MTILE>(+ paramA, paramB, tmpc_ptr, GemmCore::NTILE, k, kblocksize, StackTmp, TmpSize);+ }+ Epilogue::template forward<ISA>(tmpc_ptr, GemmCore::NTILE, 0, _config.loc[1] + in, MTILE,+ (_config.size[1] - in), _param.paramC, StackTmp, TmpSize);+ }+ }+ }++ static void gemv(const Param& _param, const parallel::gemm::ThreadProblemBase& _config) {+ if constexpr (support()) {+ auto& m = _param.problem.dims[1];+ if (_param.paramB.stor_->corr_.sdtype_ == BTLA_DTYPE::F32) {+ if (m == 1) gemv_kblock<float, 1>(_param, _config);+ if (m == 2) gemv_kblock<float, 2>(_param, _config);+ if (m == 3) gemv_kblock<float, 3>(_param, _config);+ if (m == 4) gemv_kblock<float, 4>(_param, _config);+ if constexpr (Reg32) {+ if (m == 5) gemv_kblock<float, 5>(_param, _config);+ if (m == 6) gemv_kblock<float, 6>(_param, _config);+ if (m == 7) gemv_kblock<float, 7>(_param, _config);+ if (m == 8) gemv_kblock<float, 8>(_param, _config);+ }+ } else if (_param.paramB.stor_->corr_.sdtype_ == BTLA_DTYPE::BF16) {+ if (m == 1) gemv_kblock<utils::bf16, 1>(_param, _config);+ if (m == 2) gemv_kblock<utils::bf16, 2>(_param, _config);+ if (m == 3) gemv_kblock<utils::bf16, 3>(_param, _config);+ if (m == 4) gemv_kblock<utils::bf16, 4>(_param, _config);+ if constexpr (Reg32) {+ if (m == 5) gemv_kblock<utils::bf16, 5>(_param, _config);+ if (m == 6) gemv_kblock<utils::bf16, 6>(_param, _config);+ if (m == 7) gemv_kblock<utils::bf16, 7>(_param, _config);+ if (m == 8) gemv_kblock<utils::bf16, 8>(_param, _config);+ }+ } else if (_param.paramB.stor_->corr_.sdtype_ == BTLA_DTYPE::F16) {+ if (m == 1) gemv_kblock<utils::fp16, 1>(_param, _config);+ if (m == 2) gemv_kblock<utils::fp16, 2>(_param, _config);+ if (m == 3) gemv_kblock<utils::fp16, 3>(_param, _config);+ if (m == 4) gemv_kblock<utils::fp16, 4>(_param, _config);+ if constexpr (Reg32) {+ if (m == 5) gemv_kblock<utils::fp16, 5>(_param, _config);+ if (m == 6) gemv_kblock<utils::fp16, 6>(_param, _config);+ if (m == 7) gemv_kblock<utils::fp16, 7>(_param, _config);+ if (m == 8) gemv_kblock<utils::fp16, 8>(_param, _config);+ }+ }+ }+ }+ };++ static void run(const Param& _param, const parallel::gemm::ThreadProblemBase& _config) {+ if (GEMVWrapper::support() && GEMVWrapper::implemented(_param)) {+ GEMVWrapper::gemv(_param, _config);+ } else {+ gemm(_param, _config);+ }+ bestla::kernel::wrapper::ZeroReg::forward();+ }++ protected:+ static void gemm(const Param& _param, const parallel::gemm::ThreadProblemBase& _config) {+ GemmCore::configure(_config.size[0], _config.size[1], _param.problem.dims[3]);+ auto StackTmp = alloca(_config.stacksize);+ auto tmpB = reinterpret_cast<BType*>(StackTmp);+ tmpB = utils::cpu_pointer_align(tmpB);+ auto tmpA = reinterpret_cast<AType*>(tmpB + static_cast<size_t>(_config.block[1]) * _config.block[2]);+ tmpA = utils::cpu_pointer_align(tmpA);+ auto tmpC = reinterpret_cast<AccType*>(tmpA + static_cast<size_t>(GemmCore::MTILE) * _config.block[2]);+ tmpC = utils::cpu_pointer_align(tmpC);+ auto tmpCache = reinterpret_cast<int8_t*>(tmpC + static_cast<size_t>(_config.block[0]) * _config.block[1]);+ tmpCache = utils::cpu_pointer_align(tmpCache);+ auto& KBlock = _param.problem.dims[4];+ assert(_config.block[2] >= KBlock);+ for (int itern = 0; itern < _config.size[1]; itern += _config.block[1]) {+ int n_remain = utils::remainsize(itern, _config.size[1], _config.block[1]);+ for (int iterm = 0; iterm < _config.size[0]; iterm += _config.block[0]) {+ int m_remain = utils::remainsize(iterm, _config.size[0], _config.block[0]);+ run_block(_param, _config, iterm, itern, m_remain, n_remain, tmpA, tmpB, tmpC, tmpCache);+ }+ }+ }++ // _config.block[2]%kblock==0+ // _config.block[2]>=kblock+ static void run_block(const Param& _param, const parallel::gemm::ThreadProblemBase& _config, int blk_m, int blk_n,+ int blk_msize, int blk_nsize, AType* tmpA, BType* tmpB, AccType* tmpC, int8_t* tmpcache) {+ int n_padded = utils::padto(blk_nsize, GemmCore::NTILE);+ auto& K = _param.problem.dims[3];+ auto& KBlock = _param.problem.dims[4];+ assert(_config.block[2] % KBlock == 0);+ assert(_config.block[2] % GemmCore::KTILE == 0);+ // GemmCore: int8+int8=int32=>dequant to fp32+ // accumulate to tmpC+ // extra parameters: zpA, scaleA, scaleB, reduceB+ // zpA scaleA: [MTILE,kblk_perstep]+ // scaleB reduceB: [kblk_perstep, NStep]+ int kblk_perstep = utils::updiv(_config.block[2], KBlock);+ int tmp_ldsb = _config.block[1];+ int tmp_ldsa = kblk_perstep;+ auto zpA = reinterpret_cast<AType*>(tmpcache);+ zpA = utils::cpu_pointer_align(zpA);+ auto scaleA = reinterpret_cast<float*>(zpA + GemmCore::MTILE * tmp_ldsa);+ scaleA = utils::cpu_pointer_align(scaleA);+ auto scaleB = reinterpret_cast<float*>(scaleA + GemmCore::MTILE * tmp_ldsa);+ scaleB = utils::cpu_pointer_align(scaleB);+ auto reduceB = reinterpret_cast<int32_t*>(scaleB + _config.block[1] * tmp_ldsa);+ reduceB = utils::cpu_pointer_align(reduceB);+ auto tmp_ = reinterpret_cast<int8_t*>(reduceB + _config.block[1] * tmp_ldsa);+ tmp_ = utils::cpu_pointer_align(tmp_);++ for (int iterk = 0; iterk < K; iterk += _config.block[2]) {+ int k_remain = utils::remainsize(iterk, K, _config.block[2]);+ int k_padded = utils::padto(k_remain, GemmCore::KTILE);+ auto bptr_cache = tmpB;+ int bcache_step = 0;+ int ldsb_cache = tmp_ldsb;+ auto scaleB_cache = scaleB;+ auto reduceB_cache = reduceB;+ PrologueB::template get_weight<ISA>(&bptr_cache, &bcache_step, k_padded, n_padded, iterk, _config.loc[1] + blk_n,+ _param.paramB, tmp_, _config.tmpcachesize);+ PrologueB::template get_scale<ISA>(&scaleB_cache, &ldsb_cache, k_padded, n_padded, iterk, _config.loc[1] + blk_n,+ _param.paramB, tmp_, _config.tmpcachesize);+ PrologueB::template get_reduce<ISA>(&reduceB_cache, &ldsb_cache, k_padded, n_padded, iterk,+ _config.loc[1] + blk_n, _param.paramB, tmp_, _config.tmpcachesize);+ int bcache_stride = bcache_step * sizeof(BType);+ for (int i = 0; i < blk_msize; i += GemmCore::MTILE) {+ int m_remain = utils::remainsize(i, blk_msize, GemmCore::MTILE);+ auto cptr_cache = tmpC + i * _config.block[1];+ int ccache_stride = _config.block[1] * sizeof(CType);+ auto aptr_cache = tmpA;+ int acache_step = k_padded;+ auto zpA_cache = zpA;+ auto scaleA_cache = scaleA;+ int ldsa_cache = tmp_ldsa;+ PrologueA::template getActivation<ISA>(&aptr_cache, &acache_step, _param.paramA, m_remain, k_padded,+ (blk_m + i + _config.loc[0]), iterk, tmp_, _config.tmpcachesize);+ PrologueA::template getZp<ISA>(&zpA_cache, &ldsa_cache, _param.paramA, m_remain, k_padded,+ (blk_m + i + _config.loc[0]), iterk, tmp_, _config.tmpcachesize);+ PrologueA::template getScale<ISA>(&scaleA_cache, &ldsa_cache, _param.paramA, m_remain, k_padded,+ (blk_m + i + _config.loc[0]), iterk, tmp_, _config.tmpcachesize);+ GemmCore::forward(aptr_cache, bptr_cache, cptr_cache, zpA_cache, scaleA_cache, ldsa_cache, scaleB_cache,+ reduceB_cache, ldsb_cache, m_remain, n_padded, k_padded, KBlock, acache_step * sizeof(AType),+ bcache_stride, ccache_stride, iterk, tmp_, _config.tmpcachesize);+ }+ }+ Epilogue::template forward<ISA>(tmpC, _config.block[1], (_config.loc[0] + blk_m), _config.loc[1] + blk_n, blk_msize,+ blk_nsize, _param.paramC, tmpcache, _config.tmpcachesize);+ }+};++} // namespace gemm+} // namespace wrapper+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx2.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx2.hnew file mode 100644index 000000000..56f1595a3--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx2.h@@ -0,0 +1,5418 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include "bestla.h"+#include "bestla_utils.h"+#include "kernel_jit.h"+#include "kernel_ref.h"++namespace bestla {+namespace kernel {+namespace avx2 {+#if CompileAVX2()+#if defined(__INTEL_LLVM_COMPILER)+#pragma clang attribute push(__attribute__((target("avx2,fma,f16c"))), apply_to = function)+#elif defined(__GNUC__)+#pragma GCC push_options+#pragma GCC target("avx2", "fma", "f16c")+#endif++static inline void zero_reg() { _mm256_zeroupper(); }++static inline __m256i unpack_4bits(void* srcptr, __m256i mask) {+ auto raw_data = _mm_loadu_si128(reinterpret_cast<__m128i*>(srcptr));+ auto ymm0 = _mm256_cvtepu8_epi16(raw_data);+ auto ymm1 = _mm256_slli_epi16(ymm0, 4);+ ymm0 = _mm256_or_si256(ymm0, ymm1);+ ymm0 = _mm256_and_si256(ymm0, mask);+ return ymm0;+}++static inline __m256i unpack_2bits(utils::bit2x4* ptr, const __m256i& vshift_y, const __m256i& vmask0_y,+ const __m256i& vsfhl_mask_y, const __m256i& vorder_y) {+ auto vraw_x = _mm_loadl_epi64((const __m128i*)ptr);+ auto vsrc_y = _mm256_broadcastq_epi64(vraw_x);+ auto vordered_y = _mm256_permutevar8x32_epi32(vsrc_y, vorder_y);+ auto vs_y = _mm256_srlv_epi32(vordered_y, vshift_y);+ auto v2_y = _mm256_and_si256(vs_y, vmask0_y);+ auto vout_y = _mm256_shuffle_epi8(v2_y, vsfhl_mask_y);+ return vout_y;+}++static inline __m256i unpack_1bits(utils::bit1x8* ptr, const __m256i& bit1Shift_1, const __m256i& bit1Mask,+ const __m256i& bit1Shift_2, const __m256i& highMask) {+ auto bit1x32 = _mm256_set1_epi32(*(int*)ptr);+ bit1x32 = _mm256_srlv_epi32(bit1x32, bit1Shift_1);+ bit1x32 = _mm256_and_si256(bit1x32, bit1Mask);+ bit1x32 = _mm256_mullo_epi32(bit1x32, bit1Shift_2);+ bit1x32 = _mm256_and_si256(highMask, bit1x32);+ return bit1x32;+}++inline __m256 ymm_cvt_bf16_fp32(__m128i vbf16) {+ auto vf32 = _mm256_cvtepu16_epi32(vbf16);+ return _mm256_castsi256_ps(_mm256_slli_epi32(vf32, 16));+}++inline __m256 ymm_cvt_fp16_fp32(__m128i vfp16) { return _mm256_cvtph_ps(vfp16); }++inline __m128i ymm_cvt_fp32_fp16(__m256 vfp32) { return _mm256_cvtps_ph(vfp32, _MM_FROUND_TO_NEAREST_INT); }++inline __m128i ymm_cvtepi32_epi16(__m256i src) {+ const auto shuffle_mask_32_to_16 = _mm256_set_epi8(13, 12, 9, 8, 5, 4, 1, 0, 13, 12, 9, 8, 5, 4, 1, 0, 13, 12, 9, 8,+ 5, 4, 1, 0, 13, 12, 9, 8, 5, 4, 1, 0);+ __m256i trunc_elements = _mm256_shuffle_epi8(src, shuffle_mask_32_to_16);+ __m256i ordered = _mm256_permute4x64_epi64(trunc_elements, 0x58);+ __m128i result = _mm256_castsi256_si128(ordered);+ return result;+}++static const uint8_t avx2_bf16_convert_maigc_num[32] = {+ 0x02, 0x03, 0x06, 0x07, 0x0a, 0x0b, 0x0e, 0x0f, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,+ 0x02, 0x03, 0x06, 0x07, 0x0a, 0x0b, 0x0e, 0x0f, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};++static inline __m128i cvt_fp32_to_bf16(const __m256 src) {+ const auto bf16_and_helper = _mm256_set1_epi32(0X00000001);+ const auto bf16_add_helper = _mm256_set1_epi32(0x00007FFF);+ auto shuffle_v = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(avx2_bf16_convert_maigc_num));+ auto round_bias = _mm256_castps_si256(src);+ round_bias = _mm256_and_si256(bf16_and_helper, _mm256_srli_si256(round_bias, 2));+ round_bias = _mm256_add_epi32(round_bias, bf16_add_helper);+ auto round_fp32_v = _mm256_add_epi32(_mm256_castps_si256(src), round_bias);+ __m256i trunc_elements = _mm256_shuffle_epi8(round_fp32_v, shuffle_v);+ __m256i ordered = _mm256_permute4x64_epi64(trunc_elements, 0x58);+ return _mm256_castsi256_si128(ordered);+}++inline __m128i ymm_cvt_fp32_bf16(const __m256& vfp32) {+#if FP32_BF16_FAST+ return ymm_cvtepi32_epi16(_mm256_bsrli_epi128(_mm256_castps_si256(vfp32), 2));+#else+ return cvt_fp32_to_bf16(vfp32);+#endif+}++static inline __m256i load_s8_s32(int8_t* srcptr) {+ auto xmm = _mm_loadl_epi64(reinterpret_cast<__m128i*>(srcptr));+ auto ymm = _mm256_cvtepi8_epi32(xmm);+ return ymm;+}++static inline __m256 load_bf16_fp32(const utils::bf16* srcptr) {+ auto tmp = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcptr));+ auto vf32 = ymm_cvt_bf16_fp32(tmp);+ return vf32;+}++static inline __m256 load_fp16_fp32(const utils::fp16* srcptr) {+ auto tmp = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcptr));+ auto vf32 = ymm_cvt_fp16_fp32(tmp);+ return vf32;+}++template <typename T>+static inline __m256 load_T_fp32(const T* srcptr) {+ __m256 vtmp;+ if constexpr (std::is_same_v<T, float>) {+ vtmp = _mm256_loadu_ps(srcptr);+ } else if constexpr (std::is_same_v<T, utils::bf16>) {+ vtmp = load_bf16_fp32(srcptr);+ } else if constexpr (std::is_same_v<T, utils::fp16>) {+ vtmp = load_fp16_fp32(srcptr);+ } else {+ static_assert(std::is_same_v<T, float> || std::is_same_v<T, utils::bf16> || std::is_same_v<T, utils::fp16>);+ }+ return vtmp;+}++static inline __m256 load_s8_fp32(int8_t* srcptr) {+ auto src_y = load_s8_s32(srcptr);+ auto dst_y = _mm256_cvtepi32_ps(src_y);+ return dst_y;+}++template <typename T>+static inline void store_fp32_T(const __m256& src_y, T* dstptr) {+ if constexpr (std::is_same_v<T, utils::bf16>) {+ auto xmm = ymm_cvt_fp32_bf16(src_y);+ _mm_storeu_si128(reinterpret_cast<__m128i*>(dstptr), xmm);+ } else if constexpr (std::is_same_v<T, float>) {+ _mm256_storeu_ps(dstptr, src_y);+ } else if constexpr (std::is_same_v<T, utils::fp16>) {+ auto xmm = ymm_cvt_fp32_fp16(src_y);+ _mm_storeu_si128(reinterpret_cast<__m128i*>(dstptr), xmm);+ } else {+ assert(0);+ }+}++template <typename T>+static inline void convert_s8_fp_v8(T* dstptr, int8_t* srcptr) {+ auto src_fp_y = load_s8_fp32(srcptr);+ store_fp32_T(src_fp_y, dstptr);+}++template <bool IsAsym = false>+static inline __m256 dequant_s8_fp(int8_t* srcptr, __m256 vscales, __m256i vzps = __m256i()) {+ auto src_s32_y = load_s8_s32(srcptr);+ if constexpr (IsAsym) src_s32_y = _mm256_sub_epi32(src_s32_y, vzps);+ auto src_fp_y = _mm256_cvtepi32_ps(src_s32_y);+ src_fp_y = _mm256_mul_ps(src_fp_y, vscales);+ return src_fp_y;+}++template <int N, bool IsAsym, typename DstT>+static inline void dequant_s8_N_avx2(DstT* dstptr, int8_t* srcptr, __m256* vscales, __m256i* vzps = nullptr) {+ static_assert(N % 8 == 0);+ int constexpr VLoop = N / 8;+ for (int iv = 0; iv < VLoop; iv += 1) {+ __m256 dq_f32_y;+ if constexpr (IsAsym) {+ dq_f32_y = dequant_s8_fp<IsAsym>(srcptr, vscales[iv], vzps[iv]);+ } else {+ dq_f32_y = dequant_s8_fp<IsAsym>(srcptr, vscales[iv]);+ }+ store_fp32_T(dq_f32_y, dstptr + iv * 8);+ }+}++static inline __m256i load_zp_epi8_broadcast_epi16_v16(int8_t* zpptr, const __m256i& vindex) {+ auto v_zp_x = _mm_loadu_si128((const __m128i*)zpptr);+ auto v_zp_y = _mm256_cvtepi8_epi16(v_zp_x);+ auto v_zp_y_cast = _mm256_shuffle_epi8(v_zp_y, vindex);+ return v_zp_y_cast;+}++static inline __m256i load_zp_epi8_broadcast_epi16(int8_t* zpptr, const __m256i& vindex) {+ auto v_zp_x = _mm_loadu_si128((const __m128i*)zpptr);+ auto v_zp_y = _mm256_cvtepi8_epi16(v_zp_x);+ auto v_zp_y_cast = _mm256_shuffle_epi8(v_zp_y, vindex);+ return v_zp_y_cast;+}++static inline __m256i load_zp_epi8_broadcast_epi32(int8_t* zpptr, const __m256i& vindex) {+ auto v_zp_x = _mm_loadl_epi64((const __m128i*)zpptr);+ auto v_zp_y = _mm256_cvtepi8_epi32(v_zp_x);+ auto v_zp_y_cast = _mm256_shuffle_epi8(v_zp_y, vindex);+ return v_zp_y_cast;+}++// vout= {vsrc.f32[0],vsrc.f32[0],...,vsrc.f32[4],vsrc.f32[4]}+template <bool LowBits>+static inline __m256 broadcast_ps_1_2(__m256 vsrc_y, const __m256i& vshuf_index_y) {+ __m256 tmp;+ if constexpr (LowBits) {+ tmp = _mm256_permute2f128_ps(vsrc_y, vsrc_y, 0);+ } else {+ tmp = _mm256_permute2f128_ps(vsrc_y, vsrc_y, 17);+ }+ auto tmpi = _mm256_castps_si256(tmp);++ auto out = _mm256_shuffle_epi8(tmpi, vshuf_index_y);+ return _mm256_castsi256_ps(out);+}++template <bool LowBits>+static inline __m256i broadcast_epi32_1_2(__m256i vsrc_y, const __m256i& vshuf_index_y) {+ return _mm256_castps_si256(broadcast_ps_1_2<LowBits>(_mm256_castsi256_ps(vsrc_y), vshuf_index_y));+}++inline BTLA_CODE dq8_get_fp_scale(uint8_t* src, float* dst, int row, int col, int scale_offset, int dq_blk,+ int dq_offset_idx, float* dq_scale, int src_stride, int dst_stride, bool zeropadding,+ int mN) {+ auto head_proc_num = utils::updiv(scale_offset, 8) * 8 - scale_offset;+ auto ymm_dq_offset = _mm256_set1_ps(dq_scale[dq_offset_idx]);++ auto get_fp_scale_ref = [&](int proc_src_num, int scale_offset, uint8_t* src, float* dst) {+ auto dq_s_idx = scale_offset / dq_blk;+ for (int j = 0; j < col; j++) dst[j] = dq8_bnb_LUT[src[j]] * dq_scale[dq_s_idx] + dq_scale[dq_offset_idx];+ };++ auto get_fp_scale_avx2 = [&](int scale_offset, uint8_t* src, float* dst) {+ auto dq_s_idx = scale_offset / dq_blk;+ auto ymm_dq_scale = _mm256_set1_ps(dq_scale[dq_s_idx]);+ float tmp[8];+ for (int i = 0; i < 8; i++) tmp[i] = dq8_bnb_LUT[src[i]];+ __m256 fp32_dq_ymm = _mm256_loadu_ps(tmp);+ auto fymm = _mm256_mul_ps(fp32_dq_ymm, ymm_dq_scale);+ fymm = _mm256_add_ps(fymm, ymm_dq_offset);+ _mm256_storeu_ps(dst, fymm);+ };++ for (int i = 0; i < row; i++) {+ if (head_proc_num > col) {+ get_fp_scale_ref(col, scale_offset + i * mN, src + i * src_stride, dst + i * dst_stride);+ } else {+ get_fp_scale_ref(head_proc_num, scale_offset + i * mN, src + i * src_stride, dst + i * dst_stride);+ auto scale_offset_iter = scale_offset + i * mN + head_proc_num;+ uint8_t* src_iter_ptr = src + head_proc_num;+ float* dst_iter_ptr = dst + head_proc_num;+ auto body_loop = (col - head_proc_num) / 8;+ auto tail_proc_num = (col - head_proc_num) % 8;+ int ii = 0;+ for (; ii < body_loop; ii++) {+ get_fp_scale_avx2(scale_offset_iter + ii * 8, src_iter_ptr + i * src_stride + ii * 8,+ dst_iter_ptr + i * dst_stride + ii * 8);+ }+ if (tail_proc_num > 0) {+ get_fp_scale_ref(tail_proc_num, scale_offset_iter + ii * 8, src_iter_ptr + i * src_stride + ii * 8,+ dst_iter_ptr + i * dst_stride + ii * 8);+ }+ }+ }+ if (zeropadding) assert(0);+ return BTLA_CODE::Success;+}++static inline BTLA_CODE alphabeta_f32_f32(const float alpha, const float* srcptr, const int srcstep, const float beta,+ const float* src1ptr, const int src1step, float* dstptr, const int dststep,+ const int M, const int N) {+ int constexpr Vlen = 8;+ auto vN = utils::padto_le(N, Vlen);+ auto valpha = _mm256_set1_ps(alpha);+ auto vbeta = _mm256_set1_ps(beta);++ for (int i = 0; i < M; i++) {+ int j = 0;+ if (beta != 0.f) {+ for (; j < vN; j += Vlen) {+ auto vsrc = _mm256_loadu_ps(srcptr + i * srcstep + j);+ auto vsrc1 = _mm256_loadu_ps(src1ptr + i * src1step + j);+ auto vdst = _mm256_mul_ps(valpha, vsrc);+ vdst = _mm256_fmadd_ps(vbeta, vsrc1, vdst);+ _mm256_storeu_ps(dstptr + i * dststep + j, vdst);+ }+ for (; j < N; j += 1) {+ dstptr[i * dststep + j] = alpha * srcptr[i * srcstep + j] + beta * src1ptr[i * src1step + j];+ }+ } else {+ for (; j < vN; j += Vlen) {+ auto vsrc = _mm256_loadu_ps(srcptr + i * srcstep + j);+ auto vdst = _mm256_mul_ps(valpha, vsrc);+ _mm256_storeu_ps(dstptr + i * dststep + j, vdst);+ }+ for (; j < N; j += 1) {+ dstptr[i * dststep + j] = alpha * srcptr[i * srcstep + j];+ }+ }+ }+ return BTLA_CODE::Success;+}++template <typename SCAB_T, int NTILE>+static inline BTLA_CODE dequant_s32_fp32_N(const int32_t* srcptr, const int srcstep, float* dstptr, const int dststep,+ const int row, const float* scaleA, const int ldsa, const SCAB_T* scaleB) {+ int constexpr VLen = 8;+ static_assert(NTILE % VLen == 0);+ int constexpr NRegs = NTILE / VLen;+ __m256 vwscale[NRegs];+ for (int ic = 0; ic < NRegs; ic++) {+ vwscale[ic] = load_T_fp32(scaleB + ic * VLen);+ }+ for (int irow = 0; irow < row; irow++) {+ auto scale = scaleA[irow * ldsa];+ auto valpha = _mm256_set1_ps(scale);+ for (int ic = 0; ic < NRegs; ic++) {+ auto vscale = _mm256_mul_ps(valpha, vwscale[ic]);+ auto vsrcd = _mm256_loadu_si256((const __m256i*)(srcptr + irow * srcstep + ic * VLen));+ auto vsrc = _mm256_cvtepi32_ps(vsrcd);+ vsrc = _mm256_mul_ps(vsrc, vscale);+ _mm256_storeu_ps(dstptr + irow * dststep + ic * VLen, vsrc);+ }+ }+ return BTLA_CODE::Success;+}++template <typename SCAB_T>+static inline BTLA_CODE dequant_s32_fp32(const int32_t* srcptr, const int srcstep, float* dstptr, const int dststep,+ const int row, const int col, const float* scaleA, const int ldsa,+ const SCAB_T* scaleB) {+ int col24 = utils::padto_le(col, 24);+ int col16 = utils::padto_le(col, 16);+ int col8 = utils::padto_le(col, 8);+ int icol = 0;+ for (; icol < col24; icol += 24) {+ dequant_s32_fp32_N<SCAB_T, 24>(srcptr + icol, srcstep, dstptr + icol, dststep, row, scaleA, ldsa, scaleB + icol);+ }+ if (icol + 16 <= col16) {+ for (; icol < col16; icol += 16) {+ dequant_s32_fp32_N<SCAB_T, 16>(srcptr + icol, srcstep, dstptr + icol, dststep, row, scaleA, ldsa, scaleB + icol);+ }+ }+ if (icol + 8 <= col8) {+ for (; icol < col8; icol += 8) {+ dequant_s32_fp32_N<SCAB_T, 8>(srcptr + icol, srcstep, dstptr + icol, dststep, row, scaleA, ldsa, scaleB + icol);+ }+ }+ for (; icol < col; icol += 1) {+ for (int irow = 0; irow < row; irow++) {+ auto scale = scaleA[irow * ldsa];+ dstptr[irow * dststep + icol] = scale * float(scaleB[icol]) * srcptr[irow * srcstep + icol];+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE remove_act_zeropoint_bias(float* accptr, int ldacc, int row, int col, uint8_t* zps,+ float* scales, int lds, const float* reduce) {+ int constexpr VLen = 8;+ auto col8 = utils::padto_le(col, VLen);+ for (int i = 0; i < row; i++) {+ auto zpf = static_cast<float>(zps[i * lds]) * scales[i * lds];+ int j = 0;+ auto vzp = _mm256_set1_ps(-zpf);+ for (; j < col8; j += VLen) {+ auto vreduce = _mm256_loadu_ps(reduce + j);+ auto vacc = _mm256_loadu_ps(&accptr[i * ldacc + j]);+ vacc = _mm256_fmadd_ps(vzp, vreduce, vacc);+ _mm256_storeu_ps(&accptr[i * ldacc + j], vacc);+ }+ if (j < col) {+ for (; j < col; j++) {+ accptr[i * ldacc + j] -= zpf * reduce[j];+ }+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE remove_wei_zeropoint_bias(float* accptr, int ldacc, int row, int col, int8_t* zps,+ float* scales, int lds, const float* reduce) {+ int constexpr VLen = 8;+ auto col8 = utils::padto_le(col, VLen);+ const int32_t mask[] = {-1, -1, 0, 0};+ for (int i = 0; i < row; i++) {+ auto vreduce = _mm256_set1_ps(-reduce[i * lds]);+ int j = 0;+ for (; j < col8; j += VLen) {+ auto vzp_s32 = _mm256_cvtepi8_epi32(_mm_maskload_epi32(reinterpret_cast<const int*>(zps + j),+ _mm_loadu_si128(reinterpret_cast<const __m128i*>(mask))));+ auto vzp_f32 = _mm256_cvtepi32_ps(vzp_s32);+ auto vzp = _mm256_mul_ps(vzp_f32, _mm256_loadu_ps(scales + j));+ auto vacc = _mm256_loadu_ps(&accptr[i * ldacc + j]);+ vacc = _mm256_fmadd_ps(vzp, vreduce, vacc);+ _mm256_storeu_ps(&accptr[i * ldacc + j], vacc);+ }+ if (j < col) {+ for (; j < col; j++) {+ accptr[i * ldacc + j] -= static_cast<float>(zps[j]) * scales[j] * reduce[i * lds];+ }+ }+ }+ return BTLA_CODE::Success;+}++template <bool ZPA, bool ZPB>+static inline BTLA_CODE remove_zeropoint_bias(int32_t* accptr, int ldacc, int row, int col, uint8_t* zpa, int8_t* zpb,+ int lda, int k, const int32_t* reducea, const int32_t* reduceb) {+ int constexpr VLen = 8;+ auto col16 = utils::padto_le(col, VLen);+ auto vk = _mm256_set1_epi32(k);+ for (int i = 0; i < row; i++) {+ auto vreducea = ZPB ? _mm256_set1_epi32(reducea[i * lda]) : _mm256_set1_epi32(0);+ auto vzpa = ZPA ? _mm256_set1_epi32(int(zpa[i * lda])) : _mm256_set1_epi32(0);+ int j = 0;+ for (; j < col16; j += VLen) {+ auto vzpb = ZPB ? load_s8_s32(zpb + j) : _mm256_set1_epi32(0);+ auto vreduceb = ZPA ? _mm256_loadu_si256((__m256i*)(reduceb + j)) : _mm256_set1_epi32(0);+ auto vacc = _mm256_loadu_si256((__m256i*)&accptr[i * ldacc + j]);+ __m256i tmp;+ if constexpr (ZPA) {+ tmp = _mm256_mullo_epi32(vzpa, vreduceb);+ vacc = _mm256_sub_epi32(vacc, tmp);+ }+ if constexpr (ZPB) {+ tmp = _mm256_mullo_epi32(vzpb, vreducea);+ vacc = _mm256_sub_epi32(vacc, tmp);+ }+ if constexpr (ZPA && ZPB) {+ tmp = _mm256_mullo_epi32(vzpb, vk);+ tmp = _mm256_mullo_epi32(tmp, vzpa);+ vacc = _mm256_sub_epi32(vacc, tmp);+ }+ _mm256_storeu_si256((__m256i*)&accptr[i * ldacc + j], vacc);+ }+ if (j < col) {+ int32_t vreducea = ZPB ? reducea[i * lda] : 0;+ uint8_t vzpa = ZPA ? zpa[i * lda] : 0;+ for (; j < col; j++) {+ auto vzpb = ZPB ? zpb[j] : 0;+ accptr[i * ldacc + j] -= ZPB ? vzpb * vreducea : 0;+ accptr[i * ldacc + j] -= ZPA ? vzpa * reduceb[j] : 0;+ accptr[i * ldacc + j] -= ZPB && ZPA ? vzpa * vzpb * k : 0;+ }+ }+ }+ return BTLA_CODE::Success;+}+template <int NTILE>+static inline BTLA_CODE decompress_kblock_s4_s8_pack4_row(utils::int4x2* srcptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 4;+ __m256i v_zp_y[NReg];+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(8);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi32(zptr + i * 8, vindex);+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 16, vmask);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s4_s8_pack2_row(utils::int4x2* srcptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 2;+ int constexpr Unroll = 2;+ __m256i v_zp_y[NReg];+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(8);+ const auto vindex = _mm256_set_epi8(14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8,+ 8, 6, 6, 4, 4, 2, 2, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ memcpy(tmp, zptr, NTILE * sizeof(int8_t));+ memcpy(tmp + NTILE, zptr, NTILE * sizeof(int8_t));+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi16_v16(tmp + i * 16, vindex);+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, PackRow * Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += PackRow * Unroll) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 16, vmask);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ memcpy(tmp, srcptr + (ir + ib) * NTILE / 2, k_tail * NTILE / 2);+ auto tmpout = tmp + Unroll * PackRow * NTILE / 2;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(tmp + i * 16, vmask);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(tmpout + i * 32), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s4_s8_pack1_row(utils::int4x2* srcptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ int constexpr UnpackLoop = Unroll * NTILE / 32;+ __m256i v_zp_y[UnpackLoop];+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(8);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, zptr, NTILE * sizeof(int8_t));+ }+ for (int i = 0; i < UnpackLoop; i++) {+ v_zp_y[i] = _mm256_loadu_si256((const __m256i*)(tmp + i * 32));+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += Unroll) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ for (int i = 0; i < UnpackLoop; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 16, vmask);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }++ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ memcpy(tmp, srcptr + (ir + ib) * NTILE / 2, k_tail * NTILE / 2);+ auto tmpout = tmp + Unroll * NTILE / 2;+ for (int i = 0; i < UnpackLoop; i++) {+ auto v_s8_y = unpack_4bits(tmp + i * 16, vmask);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(tmpout + i * 32), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s4_s8(utils::int4x2* srcptr, int8_t* dstptr, size_t elesize, int8_t* tmp,+ size_t tmpsize) {+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ size_t velt = utils::padto_le(elesize, 32);+ size_t i = 0;+ auto vbias = _mm256_set1_epi8(8);+ for (; i < velt; i += 32) {+ auto vout_y = unpack_4bits(reinterpret_cast<int8_t*>(srcptr + i / 2), vmask);+ vout_y = _mm256_sub_epi8(vout_y, vbias);+ _mm256_storeu_si256((__m256i*)(dstptr + i), vout_y);+ }+ if (velt < elesize) {+ if (elesize >= 32) {+ i = elesize - 32;+ auto vout_y = unpack_4bits(reinterpret_cast<int8_t*>(srcptr + i / 2), vmask);+ vout_y = _mm256_sub_epi8(vout_y, vbias);+ _mm256_storeu_si256((__m256i*)(dstptr + i), vout_y);+ } else {+ ref::decompress_kblock_s4_s8<1, 1>(srcptr + i / 2, nullptr, dstptr + i, 0, 0, 0, 0, 1,+ static_cast<int>(elesize - i), nullptr, 0);+ }+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s4_s8(utils::int4x2* srcptr, int8_t* zpptr, int8_t* dstptr, int blocksize,+ int ldzp, int n_offset, int k_offset, int row, int col, int8_t* tmp,+ size_t tmpsize) {+ if (zpptr) {+ typedef BTLA_CODE (*decompfunc)(utils::int4x2* srcptr, int8_t* zpptr, int8_t* dstptr, int blocksize, int ldzp,+ int n_offset, int k_offset, int row, int8_t* tmp, size_t tmpsize);+ decompfunc func = nullptr;+ if (col == NTILE) {+ if constexpr (PackRow == 4) {+ func = &decompress_kblock_s4_s8_pack4_row<NTILE>;+ }+ if constexpr (PackRow == 1) {+ func = &decompress_kblock_s4_s8_pack1_row<NTILE>;+ }+ if constexpr (PackRow == 2) {+ func = &decompress_kblock_s4_s8_pack2_row<NTILE>;+ }+ if (func) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ (*func)(srcptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, head_size, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ (*func)(srcptr + head_size * NTILE / 2, zpptr, dstptr + head_size * NTILE, blocksize, ldzp, n_offset,+ head_end, body_size, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ }+ assert(0);+ return BTLA_CODE::NotSupport;+ } else {+ size_t elesize = static_cast<size_t>(row) * col;+ return decompress_s4_s8(srcptr, dstptr, elesize, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s2_s8_pack4_row(utils::bit2x4* srcptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 4;+ __m256i v_zp_y[NReg];+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vbias = _mm256_set1_epi8(2);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi32(zptr + i * 8, vindex);+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b2ptr = srcptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_2bits(b2ptr + i * 8, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s2_s8_pack2_row(utils::bit2x4* srcptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 2;+ int constexpr Unroll = 2;+ __m256i v_zp_y[NReg];+ const auto vindex = _mm256_set_epi8(14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8,+ 8, 6, 6, 4, 4, 2, 2, 0, 0);+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vbias = _mm256_set1_epi8(2);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ memcpy(tmp, zptr, NTILE * sizeof(int8_t));+ memcpy(tmp + NTILE, zptr, NTILE * sizeof(int8_t));+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi16_v16(tmp + i * 16, vindex);+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, PackRow * Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += PackRow * Unroll) {+ auto b2ptr = srcptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_2bits(b2ptr + i * 8, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ memcpy(tmp, srcptr + (ir + ib) * NTILE / 4, k_tail * NTILE / 4);+ auto tmpout = tmp + Unroll * PackRow * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_2bits((utils::bit2x4*)(tmp + i * 8), vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(tmpout + i * 32), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s2_s8_pack1_row(utils::bit2x4* srcptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ int constexpr UnpackLoop = Unroll * NTILE / 32;+ __m256i v_zp_y[UnpackLoop];+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vbias = _mm256_set1_epi8(2);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, zptr, NTILE * sizeof(int8_t));+ }+ for (int i = 0; i < UnpackLoop; i++) {+ v_zp_y[i] = _mm256_loadu_si256((const __m256i*)(tmp + i * 32));+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += Unroll) {+ auto b2ptr = srcptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < UnpackLoop; i++) {+ auto v_s8_y = unpack_2bits(b2ptr + i * 8, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }++ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ memcpy(tmp, srcptr + (ir + ib) * NTILE / 4, k_tail * NTILE / 4);+ auto tmpout = tmp + Unroll * NTILE / 4;+ for (int i = 0; i < UnpackLoop; i++) {+ auto v_s8_y = unpack_2bits((utils::bit2x4*)(tmp + i * 8), vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(tmpout + i * 32), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s2_s8(utils::bit2x4* bit2ptr, int8_t* dstptr, size_t unpack_elt, int8_t* tmp,+ size_t tmpsize) {+ int constexpr VBits = 256;+ int constexpr VElt = VBits / 8;+ size_t i = 0;+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vbias = _mm256_set1_epi8(2);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ size_t elt_pad = utils::padto_le(unpack_elt, VElt);+ for (; i < elt_pad; i += VElt) {+ auto vout = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vout = _mm256_sub_epi8(vout, vbias);+ _mm256_storeu_si256((__m256i*)(dstptr + i), vout);+ }+ if (elt_pad < unpack_elt) {+ if (unpack_elt >= 32) {+ i = unpack_elt - 32;+ auto vout = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vout = _mm256_sub_epi8(vout, vbias);+ _mm256_storeu_si256((__m256i*)(dstptr + i), vout);+ } else {+ ref::decompress_s2_s8(bit2ptr + i / 4, dstptr + i, unpack_elt - i, tmp, tmpsize);+ }+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s2_s8(utils::bit2x4* bit2ptr, int8_t* zpptr, int8_t* dstptr, int blocksize,+ int ldzp, int n_offset, int k_offset, int row, int col, int8_t* tmp,+ size_t tmpsize) {+ if (zpptr) {+ typedef BTLA_CODE (*decompfunc)(utils::bit2x4* srcptr, int8_t* zpptr, int8_t* dstptr, int blocksize, int ldzp,+ int n_offset, int k_offset, int row, int8_t* tmp, size_t tmpsize);+ decompfunc func = nullptr;+ if (col == NTILE) {+ if constexpr (PackRow == 4) {+ func = &decompress_kblock_s2_s8_pack4_row<NTILE>;+ }+ if constexpr (PackRow == 1) {+ func = &decompress_kblock_s2_s8_pack1_row<NTILE>;+ }+ if constexpr (PackRow == 2) {+ func = &decompress_kblock_s2_s8_pack2_row<NTILE>;+ }+ if (func) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ (*func)(bit2ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, head_size, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ (*func)(bit2ptr + head_size * NTILE / 4, zpptr, dstptr + head_size * NTILE, blocksize, ldzp, n_offset,+ head_end, body_size, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ }+ assert(0);+ return BTLA_CODE::NotSupport;+ } else {+ size_t elesize = static_cast<size_t>(row) * col;+ return decompress_s2_s8(bit2ptr, dstptr, elesize, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s3_s8_pack4_row(utils::bit2x4* srcptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 4;+ __m256i v_zp_y[NReg];+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vbias = _mm256_set1_epi8(4);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi32(zptr + i * 8, vindex);+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b2ptr = srcptr + (ir + ib) * NTILE / 4;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_2bits(b2ptr + i * 8, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr + i * 4, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s3_s8_pack2_row(utils::bit2x4* srcptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 2;+ int constexpr Unroll = 2;+ __m256i v_zp_y[NReg];+ const auto vindex = _mm256_set_epi8(14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8,+ 8, 6, 6, 4, 4, 2, 2, 0, 0);+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vbias = _mm256_set1_epi8(4);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));++ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ memcpy(tmp, zptr, NTILE * sizeof(int8_t));+ memcpy(tmp + NTILE, zptr, NTILE * sizeof(int8_t));+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi16_v16(tmp + i * 16, vindex);+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, PackRow * Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += PackRow * Unroll) {+ auto b2ptr = srcptr + (ir + ib) * NTILE / 4;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_2bits(b2ptr + i * 8, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr + i * 4, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb2ptr = tmp;+ memcpy(tmpb2ptr, srcptr + (ir + ib) * NTILE / 4, k_tail * NTILE / 4);+ auto tmpb1ptr = tmp + Unroll * NTILE / 2;+ memcpy(tmpb1ptr, bit1ptr + (ir + ib) * NTILE / 8, k_tail * NTILE / 8);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_2bits((utils::bit2x4*)(tmpb2ptr + i * 8), vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits((utils::bit1x8*)(tmpb1ptr + i * 4), bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(tmpout + i * 32), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s3_s8_pack1_row(utils::bit2x4* srcptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ int constexpr UnpackLoop = Unroll * NTILE / 32;+ __m256i v_zp_y[UnpackLoop];+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vbias = _mm256_set1_epi8(4);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, zptr, NTILE * sizeof(int8_t));+ }+ for (int i = 0; i < UnpackLoop; i++) {+ v_zp_y[i] = _mm256_loadu_si256((const __m256i*)(tmp + i * 32));+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += Unroll) {+ auto b2ptr = srcptr + (ir + ib) * NTILE / 4;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < UnpackLoop; i++) {+ auto v_s8_y = unpack_2bits(b2ptr + i * 8, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr + i * 4, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }++ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb2ptr = tmp;+ memcpy(tmpb2ptr, srcptr + (ir + ib) * NTILE / 4, k_tail * NTILE / 4);+ auto tmpb1ptr = tmp + Unroll * NTILE / 2;+ memcpy(tmpb1ptr, bit1ptr + (ir + ib) * NTILE / 8, k_tail * NTILE / 8);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < UnpackLoop; i++) {+ auto v_s8_y = unpack_2bits((utils::bit2x4*)(tmpb2ptr + i * 8), vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits((utils::bit1x8*)(tmpb1ptr + i * 4), bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(tmpout + i * 32), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s3_s8(utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr, int8_t* dstptr,+ size_t unpack_elt, int8_t* tmp, size_t tmpsize) {+ int constexpr VBits = 256;+ int constexpr VElt = VBits / 8;+ size_t i = 0;+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vbias = _mm256_set1_epi8(4);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ size_t elt_pad = utils::padto_le(unpack_elt, VElt);+ for (; i < elt_pad; i += VElt) {+ auto vout = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(bit1ptr + i / 8, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vout = _mm256_or_si256(vout, vb1);+ vout = _mm256_sub_epi8(vout, vbias);+ _mm256_storeu_si256((__m256i*)(dstptr + i), vout);+ }+ if (elt_pad < unpack_elt) {+ if (unpack_elt >= 32) {+ i = unpack_elt - 32;+ auto vout = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(bit1ptr + i / 8, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vout = _mm256_or_si256(vout, vb1);+ vout = _mm256_sub_epi8(vout, vbias);+ _mm256_storeu_si256((__m256i*)(dstptr + i), vout);+ } else {+ ref::decompress_s3_s8(bit2ptr + i / 4, bit1ptr + i / 8, dstptr + i, unpack_elt - i, tmp, tmpsize);+ }+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s3_s8(utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset, int k_offset,+ int row, int col, int8_t* tmp, size_t tmpsize) {+ if (zpptr) {+ typedef BTLA_CODE (*decompfunc)(utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row, int8_t* tmp,+ size_t tmpsize);+ decompfunc func = nullptr;+ if (col == NTILE) {+ if constexpr (PackRow == 1) {+ func = &decompress_kblock_s3_s8_pack1_row<NTILE>;+ }+ if constexpr (PackRow == 2) {+ func = &decompress_kblock_s3_s8_pack2_row<NTILE>;+ }+ if constexpr (PackRow == 4) {+ func = &decompress_kblock_s3_s8_pack4_row<NTILE>;+ }+ if (func) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ (*func)(bit2ptr, bit1ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, head_size, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ (*func)(bit2ptr + head_size * NTILE / 4, bit1ptr + head_size * NTILE / 8, zpptr, dstptr + head_size * NTILE,+ blocksize, ldzp, n_offset, head_end, body_size, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ }+ assert(0);+ return BTLA_CODE::NotSupport;+ } else {+ size_t elesize = static_cast<size_t>(row) * col;+ return decompress_s3_s8(bit2ptr, bit1ptr, dstptr, elesize, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s1_s8_pack4_row(utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 4;+ __m256i v_zp_y[NReg];+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);+ int constexpr FullRange = 1 << (1 - 1);+ auto vbias = _mm256_set1_epi8(FullRange);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi32(zptr + i * 8, vindex);+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr + i * 4, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), vb1);+ }+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s1_s8_pack2_row(utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 2;+ int constexpr Unroll = 2;+ __m256i v_zp_y[NReg];+ const auto vindex = _mm256_set_epi8(14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8,+ 8, 6, 6, 4, 4, 2, 2, 0, 0);+ int constexpr FullRange = 1 << (1 - 1);+ auto vbias = _mm256_set1_epi8(FullRange);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));++ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ memcpy(tmp, zptr, NTILE * sizeof(int8_t));+ memcpy(tmp + NTILE, zptr, NTILE * sizeof(int8_t));+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi16_v16(tmp + i * 16, vindex);+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, PackRow * Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += PackRow * Unroll) {+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr + i * 4, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), vb1);+ }+ }+ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb1ptr = tmp;+ memcpy(tmpb1ptr, bit1ptr + (ir + ib) * NTILE / 8, k_tail * NTILE / 8);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits((utils::bit1x8*)(tmpb1ptr + i * 4), bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(tmpout + i * 32), vb1);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s1_s8_pack1_row(utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ int constexpr UnpackLoop = Unroll * NTILE / 32;+ int constexpr FullRange = 1 << (1 - 1);+ auto vbias = _mm256_set1_epi8(FullRange);+ __m256i v_zp_y[UnpackLoop];++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, zptr, NTILE * sizeof(int8_t));+ }+ for (int i = 0; i < UnpackLoop; i++) {+ v_zp_y[i] = _mm256_loadu_si256((const __m256i*)(tmp + i * 32));+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += Unroll) {+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < UnpackLoop; i++) {+ auto vb1 = unpack_1bits(b1ptr + i * 4, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), vb1);+ }+ }++ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb1ptr = tmp;+ memcpy(tmpb1ptr, bit1ptr + (ir + ib) * NTILE / 8, k_tail * NTILE / 8);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < UnpackLoop; i++) {+ auto vb1 = unpack_1bits((utils::bit1x8*)(tmpb1ptr + i * 4), bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(tmpout + i * 32), vb1);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s1_s8(utils::bit1x8* bit1ptr, int8_t* dstptr, size_t unpack_elt, int8_t* tmp,+ size_t tmpsize) {+ int constexpr VBits = 256;+ int constexpr VElt = VBits / 8;+ size_t i = 0;+ int constexpr FullRange = 1 << (1 - 1);+ auto vbias = _mm256_set1_epi8(FullRange);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ size_t elt_pad = utils::padto_le(unpack_elt, VElt);+ for (; i < elt_pad; i += VElt) {+ auto vb1 = unpack_1bits(bit1ptr + i / 8, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, vbias);+ _mm256_storeu_si256((__m256i*)(dstptr + i), vb1);+ }+ if (elt_pad < unpack_elt) {+ if (unpack_elt >= 32) {+ i = unpack_elt - 32;+ auto vb1 = unpack_1bits(bit1ptr + i / 8, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, vbias);+ _mm256_storeu_si256((__m256i*)(dstptr + i), vb1);+ } else {+ ref::decompress_s1_s8(bit1ptr + i / 8, dstptr + i, unpack_elt - i, tmp, tmpsize);+ }+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s1_s8(utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr, int blocksize,+ int ldzp, int n_offset, int k_offset, int row, int col, int8_t* tmp,+ size_t tmpsize) {+ if (zpptr) {+ typedef BTLA_CODE (*decompfunc)(utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr, int blocksize, int ldzp,+ int n_offset, int k_offset, int row, int8_t* tmp, size_t tmpsize);+ decompfunc func = nullptr;+ if (col == NTILE) {+ if constexpr (PackRow == 1) {+ func = &decompress_kblock_s1_s8_pack1_row<NTILE>;+ }+ if constexpr (PackRow == 2) {+ func = &decompress_kblock_s1_s8_pack2_row<NTILE>;+ }+ if constexpr (PackRow == 4) {+ func = &decompress_kblock_s1_s8_pack4_row<NTILE>;+ }+ if (func) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ (*func)(bit1ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, head_size, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ (*func)(bit1ptr + head_size * NTILE / 8, zpptr, dstptr + head_size * NTILE, blocksize, ldzp, n_offset,+ head_end, body_size, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ }+ assert(0);+ return BTLA_CODE::NotSupport;+ } else {+ size_t elesize = static_cast<size_t>(row) * col;+ return decompress_s1_s8(bit1ptr, dstptr, elesize, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s5_s8_pack4_row(utils::bit4x2* srcptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 4;+ __m256i v_zp_y[NReg];+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);+ int constexpr FullRange = 1 << (5 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi32(zptr + i * 8, vindex);+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 16, vmask);+ auto vb1 = unpack_1bits(b1ptr + i * 4, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s5_s8_pack2_row(utils::bit4x2* srcptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 2;+ int constexpr Unroll = 2;+ __m256i v_zp_y[NReg];+ const auto vindex = _mm256_set_epi8(14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8,+ 8, 6, 6, 4, 4, 2, 2, 0, 0);+ int constexpr FullRange = 1 << (5 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));++ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ memcpy(tmp, zptr, NTILE * sizeof(int8_t));+ memcpy(tmp + NTILE, zptr, NTILE * sizeof(int8_t));+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi16_v16(tmp + i * 16, vindex);+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, PackRow * Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += PackRow * Unroll) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 16, vmask);+ auto vb1 = unpack_1bits(b1ptr + i * 4, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb4ptr = tmp;+ memcpy(tmpb4ptr, srcptr + (ir + ib) * NTILE / 2, k_tail * NTILE / 2);+ auto tmpb1ptr = tmp + Unroll * NTILE / 2;+ memcpy(tmpb1ptr, bit1ptr + (ir + ib) * NTILE / 8, k_tail * NTILE / 8);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits((utils::bit4x2*)(tmpb4ptr + i * 16), vmask);+ auto vb1 = unpack_1bits((utils::bit1x8*)(tmpb1ptr + i * 4), bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(tmpout + i * 32), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s5_s8_pack1_row(utils::bit4x2* srcptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ int constexpr UnpackLoop = Unroll * NTILE / 32;+ int constexpr FullRange = 1 << (5 - 1);+ __m256i v_zp_y[UnpackLoop];+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, zptr, NTILE * sizeof(int8_t));+ }+ for (int i = 0; i < UnpackLoop; i++) {+ v_zp_y[i] = _mm256_loadu_si256((const __m256i*)(tmp + i * 32));+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += Unroll) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < UnpackLoop; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 16, vmask);+ auto vb1 = unpack_1bits(b1ptr + i * 4, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }++ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb4ptr = tmp;+ memcpy(tmpb4ptr, srcptr + (ir + ib) * NTILE / 2, k_tail * NTILE / 2);+ auto tmpb1ptr = tmp + Unroll * NTILE / 2;+ memcpy(tmpb1ptr, bit1ptr + (ir + ib) * NTILE / 8, k_tail * NTILE / 8);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < UnpackLoop; i++) {+ auto v_s8_y = unpack_4bits((utils::bit4x2*)(tmpb4ptr + i * 16), vmask);+ auto vb1 = unpack_1bits((utils::bit1x8*)(tmpb1ptr + i * 4), bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(tmpout + i * 32), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s5_s8(utils::bit4x2* bit4ptr, utils::bit1x8* bit1ptr, int8_t* dstptr,+ size_t unpack_elt, int8_t* tmp, size_t tmpsize) {+ int constexpr VBits = 256;+ int constexpr VElt = VBits / 8;+ size_t i = 0;+ int constexpr FullRange = 1 << (5 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ size_t elt_pad = utils::padto_le(unpack_elt, VElt);+ for (; i < elt_pad; i += VElt) {+ auto vout = unpack_4bits(bit4ptr + i / 2, vmask);+ auto vb1 = unpack_1bits(bit1ptr + i / 8, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ vout = _mm256_or_si256(vout, vb1);+ vout = _mm256_sub_epi8(vout, vbias);+ _mm256_storeu_si256((__m256i*)(dstptr + i), vout);+ }+ if (elt_pad < unpack_elt) {+ if (unpack_elt >= 32) {+ i = unpack_elt - 32;+ auto vout = unpack_4bits(bit4ptr + i / 2, vmask);+ auto vb1 = unpack_1bits(bit1ptr + i / 8, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ vout = _mm256_or_si256(vout, vb1);+ vout = _mm256_sub_epi8(vout, vbias);+ _mm256_storeu_si256((__m256i*)(dstptr + i), vout);+ } else {+ ref::decompress_s5_s8(bit4ptr + i / 2, bit1ptr + i / 8, dstptr + i, unpack_elt - i, tmp, tmpsize);+ }+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s5_s8(utils::bit4x2* bit4ptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset, int k_offset,+ int row, int col, int8_t* tmp, size_t tmpsize) {+ if (zpptr) {+ typedef BTLA_CODE (*decompfunc)(utils::bit4x2* bit4ptr, utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row, int8_t* tmp,+ size_t tmpsize);+ decompfunc func = nullptr;+ if (col == NTILE) {+ if constexpr (PackRow == 1) {+ func = &decompress_kblock_s5_s8_pack1_row<NTILE>;+ }+ if constexpr (PackRow == 2) {+ func = &decompress_kblock_s5_s8_pack2_row<NTILE>;+ }+ if constexpr (PackRow == 4) {+ func = &decompress_kblock_s5_s8_pack4_row<NTILE>;+ }+ if (func) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ (*func)(bit4ptr, bit1ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, head_size, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ (*func)(bit4ptr + head_size * NTILE / 2, bit1ptr + head_size * NTILE / 8, zpptr, dstptr + head_size * NTILE,+ blocksize, ldzp, n_offset, head_end, body_size, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ }+ assert(0);+ return BTLA_CODE::NotSupport;+ } else {+ size_t elesize = static_cast<size_t>(row) * col;+ return decompress_s5_s8(bit4ptr, bit1ptr, dstptr, elesize, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s7_s8_pack4_row(utils::bit4x2* srcptr, utils::bit2x4* bit2ptr,+ utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 4;+ __m256i v_zp_y[NReg];+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);+ int constexpr FullRange = 1 << (7 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ uint32_t mask0 = 0x03030303;+ auto vmask0 = _mm256_set1_epi32(*(int32_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi32(zptr + i * 8, vindex);+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ auto b2ptr = bit2ptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 16, vmask);+ auto vb1 = unpack_1bits(b1ptr + i * 4, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(b2ptr + i * 8, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_or_si256(v_s8_y, vb2);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s7_s8_pack2_row(utils::bit4x2* srcptr, utils::bit2x4* bit2ptr,+ utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 2;+ int constexpr Unroll = 2;+ __m256i v_zp_y[NReg];+ const auto vindex = _mm256_set_epi8(14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8,+ 8, 6, 6, 4, 4, 2, 2, 0, 0);+ int constexpr FullRange = 1 << (7 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ uint32_t mask0 = 0x03030303;+ auto vmask0 = _mm256_set1_epi32(*(int32_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));++ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ memcpy(tmp, zptr, NTILE * sizeof(int8_t));+ memcpy(tmp + NTILE, zptr, NTILE * sizeof(int8_t));+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi16_v16(tmp + i * 16, vindex);+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, PackRow * Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += PackRow * Unroll) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ auto b2ptr = bit2ptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 16, vmask);+ auto vb1 = unpack_1bits(b1ptr + i * 4, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(b2ptr + i * 8, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_or_si256(v_s8_y, vb2);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb4ptr = tmp;+ memcpy(tmpb4ptr, srcptr + (ir + ib) * NTILE / 2, k_tail * NTILE / 2);+ auto tmpb1ptr = tmp + Unroll * NTILE / 2;+ memcpy(tmpb1ptr, bit1ptr + (ir + ib) * NTILE / 8, k_tail * NTILE / 8);+ auto tmpb2ptr = tmp + Unroll * NTILE * 3 / 4;+ memcpy(tmpb2ptr, bit2ptr + (ir + ib) * NTILE / 4, k_tail * NTILE / 4);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits((utils::bit4x2*)(tmpb4ptr + i * 16), vmask);+ auto vb1 = unpack_1bits((utils::bit1x8*)(tmpb1ptr + i * 4), bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits((utils::bit2x4*)(tmpb2ptr + i * 8), vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_or_si256(v_s8_y, vb2);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(tmpout + i * 32), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s7_s8_pack1_row(utils::bit4x2* srcptr, utils::bit2x4* bit2ptr,+ utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ int constexpr UnpackLoop = Unroll * NTILE / 32;+ int constexpr FullRange = 1 << (7 - 1);+ __m256i v_zp_y[UnpackLoop];+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ uint32_t mask0 = 0x03030303;+ auto vmask0 = _mm256_set1_epi32(*(int32_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, zptr, NTILE * sizeof(int8_t));+ }+ for (int i = 0; i < UnpackLoop; i++) {+ v_zp_y[i] = _mm256_loadu_si256((const __m256i*)(tmp + i * 32));+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += Unroll) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ auto b2ptr = bit2ptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < UnpackLoop; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 16, vmask);+ auto vb1 = unpack_1bits(b1ptr + i * 4, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(b2ptr + i * 8, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_or_si256(v_s8_y, vb2);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }++ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb4ptr = tmp;+ memcpy(tmpb4ptr, srcptr + (ir + ib) * NTILE / 2, k_tail * NTILE / 2);+ auto tmpb1ptr = tmp + Unroll * NTILE / 2;+ memcpy(tmpb1ptr, bit1ptr + (ir + ib) * NTILE / 8, k_tail * NTILE / 8);+ auto tmpb2ptr = tmp + Unroll * NTILE * 3 / 4;+ memcpy(tmpb2ptr, bit2ptr + (ir + ib) * NTILE / 4, k_tail * NTILE / 4);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < UnpackLoop; i++) {+ auto v_s8_y = unpack_4bits((utils::bit4x2*)(tmpb4ptr + i * 16), vmask);+ auto vb1 = unpack_1bits((utils::bit1x8*)(tmpb1ptr + i * 4), bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits((utils::bit2x4*)(tmpb2ptr + i * 8), vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_or_si256(v_s8_y, vb2);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(tmpout + i * 32), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s7_s8(utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr,+ int8_t* dstptr, size_t unpack_elt, int8_t* tmp, size_t tmpsize) {+ int constexpr VBits = 256;+ int constexpr VElt = VBits / 8;+ size_t i = 0;+ int constexpr FullRange = 1 << (7 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ uint32_t mask0 = 0x03030303;+ auto vmask0 = _mm256_set1_epi32(*(int32_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ size_t elt_pad = utils::padto_le(unpack_elt, VElt);+ for (; i < elt_pad; i += VElt) {+ auto vout = unpack_4bits(bit4ptr + i / 2, vmask);+ auto vb1 = unpack_1bits(bit1ptr + i / 8, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ vout = _mm256_or_si256(vout, vb1);+ vout = _mm256_or_si256(vout, vb2);+ vout = _mm256_sub_epi8(vout, vbias);+ _mm256_storeu_si256((__m256i*)(dstptr + i), vout);+ }+ if (elt_pad < unpack_elt) {+ if (unpack_elt >= 32) {+ i = unpack_elt - 32;+ auto vout = unpack_4bits(bit4ptr + i / 2, vmask);+ auto vb1 = unpack_1bits(bit1ptr + i / 8, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ vout = _mm256_or_si256(vout, vb1);+ vout = _mm256_or_si256(vout, vb2);+ vout = _mm256_sub_epi8(vout, vbias);+ _mm256_storeu_si256((__m256i*)(dstptr + i), vout);+ } else {+ ref::decompress_s7_s8(bit4ptr + i / 2, bit2ptr + i / 4, bit1ptr + i / 8, dstptr + i, unpack_elt - i, tmp,+ tmpsize);+ }+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s7_s8(utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr,+ int8_t* zpptr, int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int col, int8_t* tmp, size_t tmpsize) {+ if (zpptr) {+ typedef BTLA_CODE (*decompfunc)(utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr,+ int8_t* zpptr, int8_t* dstptr, int blocksize, int ldzp, int n_offset, int k_offset,+ int row, int8_t* tmp, size_t tmpsize);+ decompfunc func = nullptr;+ if (col == NTILE) {+ if constexpr (PackRow == 1) {+ func = &decompress_kblock_s7_s8_pack1_row<NTILE>;+ }+ if constexpr (PackRow == 2) {+ func = &decompress_kblock_s7_s8_pack2_row<NTILE>;+ }+ if constexpr (PackRow == 4) {+ func = &decompress_kblock_s7_s8_pack4_row<NTILE>;+ }+ if (func) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ (*func)(bit4ptr, bit2ptr, bit1ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, head_size, tmp,+ tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ (*func)(bit4ptr + head_size * NTILE / 2, bit2ptr + head_size * NTILE / 4, bit1ptr + head_size * NTILE / 8,+ zpptr, dstptr + head_size * NTILE, blocksize, ldzp, n_offset, head_end, body_size, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ }+ assert(0);+ return BTLA_CODE::NotSupport;+ } else {+ size_t elesize = static_cast<size_t>(row) * col;+ return decompress_s7_s8(bit4ptr, bit2ptr, bit1ptr, dstptr, elesize, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s6_s8_pack4_row(utils::bit4x2* srcptr, utils::bit2x4* bit2ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 4) == 0);+ int constexpr PackRow = 4;+ __m256i v_zp_y[NReg];+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);+ int constexpr FullRange = 1 << (6 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ uint32_t mask0 = 0x03030303;+ auto vmask0 = _mm256_set1_epi32(*(int32_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi32(zptr + i * 8, vindex);+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b2ptr = bit2ptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 16, vmask);+ auto vb1 = unpack_2bits(b2ptr + i * 8, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s6_s8_pack2_row(utils::bit4x2* srcptr, utils::bit2x4* bit2ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 2;+ int constexpr Unroll = 2;+ __m256i v_zp_y[NReg];+ const auto vindex = _mm256_set_epi8(14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8,+ 8, 6, 6, 4, 4, 2, 2, 0, 0);+ int constexpr FullRange = 1 << (6 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ uint32_t mask0 = 0x03030303;+ auto vmask0 = _mm256_set1_epi32(*(int32_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);++ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ memcpy(tmp, zptr, NTILE * sizeof(int8_t));+ memcpy(tmp + NTILE, zptr, NTILE * sizeof(int8_t));+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi16_v16(tmp + i * 16, vindex);+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, PackRow * Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += PackRow * Unroll) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b2ptr = bit2ptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 16, vmask);+ auto vb1 = unpack_2bits(b2ptr + i * 8, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb4ptr = tmp;+ memcpy(tmpb4ptr, srcptr + (ir + ib) * NTILE / 2, k_tail * NTILE / 2);+ auto tmpb2ptr = tmp + Unroll * NTILE / 2;+ memcpy(tmpb2ptr, bit2ptr + (ir + ib) * NTILE / 4, k_tail * NTILE / 4);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits((utils::bit4x2*)(tmpb4ptr + i * 16), vmask);+ auto vb1 = unpack_2bits((utils::bit2x4*)(tmpb2ptr + i * 8), vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(tmpout + i * 32), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s6_s8_pack1_row(utils::bit4x2* srcptr, utils::bit2x4* bit2ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ static_assert((NTILE % 8) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ int constexpr UnpackLoop = Unroll * NTILE / 32;+ int constexpr FullRange = 1 << (6 - 1);+ __m256i v_zp_y[UnpackLoop];+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ uint32_t mask0 = 0x03030303;+ auto vmask0 = _mm256_set1_epi32(*(int32_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, zptr, NTILE * sizeof(int8_t));+ }+ for (int i = 0; i < UnpackLoop; i++) {+ v_zp_y[i] = _mm256_loadu_si256((const __m256i*)(tmp + i * 32));+ v_zp_y[i] = _mm256_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += Unroll) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b2ptr = bit2ptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < UnpackLoop; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 16, vmask);+ auto vb1 = unpack_2bits(b2ptr + i * 8, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(dstptr + i * 32 + (ir + ib) * NTILE), v_s8_y);+ }+ }++ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb4ptr = tmp;+ memcpy(tmpb4ptr, srcptr + (ir + ib) * NTILE / 2, k_tail * NTILE / 2);+ auto tmpb2ptr = tmp + Unroll * NTILE / 2;+ memcpy(tmpb2ptr, bit2ptr + (ir + ib) * NTILE / 4, k_tail * NTILE / 4);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < UnpackLoop; i++) {+ auto v_s8_y = unpack_4bits((utils::bit4x2*)(tmpb4ptr + i * 16), vmask);+ auto vb1 = unpack_2bits((utils::bit2x4*)(tmpb2ptr + i * 8), vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ v_s8_y = _mm256_or_si256(v_s8_y, vb1);+ v_s8_y = _mm256_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm256_storeu_si256((__m256i*)(tmpout + i * 32), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s6_s8(utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, int8_t* dstptr,+ size_t unpack_elt, int8_t* tmp, size_t tmpsize) {+ int constexpr VBits = 256;+ int constexpr VElt = VBits / 8;+ size_t i = 0;+ int constexpr FullRange = 1 << (6 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ uint32_t mask0 = 0x03030303;+ auto vmask0 = _mm256_set1_epi32(*(int32_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ size_t elt_pad = utils::padto_le(unpack_elt, VElt);+ for (; i < elt_pad; i += VElt) {+ auto vout = unpack_4bits(bit4ptr + i / 2, vmask);+ auto vb1 = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vout = _mm256_or_si256(vout, vb1);+ vout = _mm256_sub_epi8(vout, vbias);+ _mm256_storeu_si256((__m256i*)(dstptr + i), vout);+ }+ if (elt_pad < unpack_elt) {+ if (unpack_elt >= 32) {+ i = unpack_elt - 32;+ auto vout = unpack_4bits(bit4ptr + i / 2, vmask);+ auto vb1 = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vout = _mm256_or_si256(vout, vb1);+ vout = _mm256_sub_epi8(vout, vbias);+ _mm256_storeu_si256((__m256i*)(dstptr + i), vout);+ } else {+ ref::decompress_s6_s8(bit4ptr + i / 2, bit2ptr + i / 4, dstptr + i, unpack_elt - i, tmp, tmpsize);+ }+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s6_s8(utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset, int k_offset,+ int row, int col, int8_t* tmp, size_t tmpsize) {+ if (zpptr) {+ typedef BTLA_CODE (*decompfunc)(utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row, int8_t* tmp,+ size_t tmpsize);+ decompfunc func = nullptr;+ if (col == NTILE) {+ if constexpr (PackRow == 1) {+ func = &decompress_kblock_s6_s8_pack1_row<NTILE>;+ }+ if constexpr (PackRow == 2) {+ func = &decompress_kblock_s6_s8_pack2_row<NTILE>;+ }+ if constexpr (PackRow == 4) {+ func = &decompress_kblock_s6_s8_pack4_row<NTILE>;+ }+ if (func) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ (*func)(bit4ptr, bit2ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, head_size, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ (*func)(bit4ptr + head_size * NTILE / 2, bit2ptr + head_size * NTILE / 4, zpptr, dstptr + head_size * NTILE,+ blocksize, ldzp, n_offset, head_end, body_size, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ }+ assert(0);+ return BTLA_CODE::NotSupport;+ } else {+ size_t elesize = static_cast<size_t>(row) * col;+ return decompress_s6_s8(bit4ptr, bit2ptr, dstptr, elesize, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++template <bool WITH_SCALE, typename _DST_T, int _PACK_ROW, typename _S_T>+inline BTLA_CODE decompress_kblock_f8_fp(utils::f8* srcptr, _DST_T* dstptr, int row, int col, int ld_src, int ld_dst,+ _S_T* scales, int k_offset, int kblock, int NPad, BTLA_DTYPE src_f8_type) {+ int align_col = col / 16 * 16;+ int col_tail = col - align_col;+ auto ebits = utils::bestla_dtype_get_f8_ebits(src_f8_type);+ auto mantissabit = 7 - ebits;+ auto sign_revert_and_mask = _mm256_set1_epi32(0x80000000);+ auto e_revert_and_mask = _mm256_set1_epi32(0x0000007f);+ auto e_revert_shift = _mm256_set1_epi32(1);+ e_revert_shift = _mm256_slli_epi32(e_revert_shift, ebits - 1);+ e_revert_shift = _mm256_sub_epi32(e_revert_shift, _mm256_set1_epi32(128));+ auto mantissa_revert_and_mask = _mm256_set1_epi32(0x007fffff);+ auto packrow2_permute_idx = _mm256_setr_epi32(0, 0, 1, 1, 2, 2, 3, 3);+ for (int i = 0; i < row; i++) {+ int kpos = (k_offset + i) / kblock;+ auto sptr = scales + kpos * NPad;+ int j = 0;+ auto quant = [&]() {+ auto sign_revert = _mm256_cvtepi8_epi32(_mm_loadu_si128(reinterpret_cast<__m128i*>(srcptr + i * ld_src + j)));+ auto e_revert = sign_revert;+ auto mantissa_revert = sign_revert;+ sign_revert = _mm256_slli_epi32(sign_revert, 24);+ sign_revert = _mm256_and_si256(sign_revert, sign_revert_and_mask);+ e_revert = _mm256_and_si256(e_revert, e_revert_and_mask);+ e_revert = _mm256_srli_epi32(e_revert, mantissabit);+ if constexpr (WITH_SCALE && std::is_same_v<_S_T, utils::f8>) {+ auto scale = _mm256_cvtepi8_epi32(_mm_loadu_si128(reinterpret_cast<__m128i*>(sptr + j / _PACK_ROW)));+ if constexpr (_PACK_ROW == 2) scale = _mm256_permutevar8x32_epi32(packrow2_permute_idx, scale);+ e_revert = _mm256_add_epi32(e_revert, scale);+ }+ e_revert = _mm256_sub_epi32(e_revert, e_revert_shift);+ e_revert = _mm256_slli_epi32(e_revert, 23);+ mantissa_revert = _mm256_slli_epi32(mantissa_revert, 23 - mantissabit);+ mantissa_revert = _mm256_and_si256(mantissa_revert, mantissa_revert_and_mask);+ auto fp_v = _mm256_or_ps(_mm256_castsi256_ps(sign_revert), _mm256_castsi256_ps(e_revert));+ fp_v = _mm256_or_ps(fp_v, _mm256_castsi256_ps(mantissa_revert));+ if constexpr (WITH_SCALE && std::is_same_v<_S_T, float>) {+ auto scale = _mm256_loadu_ps(sptr + j / _PACK_ROW);+ if constexpr (_PACK_ROW == 2) scale = _mm256_permutevar8x32_ps(scale, packrow2_permute_idx);+ fp_v = _mm256_mul_ps(fp_v, scale);+ }+ if constexpr (std::is_same_v<_DST_T, float>) {+ _mm256_storeu_ps(dstptr + i * ld_dst + j, fp_v);+ } else {+ assert(0);+ }+ };+ for (; j < align_col; j += 8) quant();+ for (; j < col; j++) {+ auto fp_v = ref::f8_to_fp32(srcptr[i * ld_src + j], src_f8_type);+ if constexpr (WITH_SCALE) {+ if constexpr (std::is_same_v<_S_T, utils::f8>) {+ dstptr[i * ld_dst + j] = sptr[j / _PACK_ROW].mul(fp_v);+ } else if constexpr (std::is_same_v<_S_T, float>) {+ dstptr[i * ld_dst + j] = fp_v * sptr[j / _PACK_ROW];+ }+ } else {+ dstptr[i * ld_dst + j] = fp_v;+ }+ }+ }+ return BTLA_CODE::Success;+}++template <typename SCA_T>+static inline BTLA_CODE accum_alphaN_f32_f32(const SCA_T* alpha, const float* srcptr, const int srcstep, float* dstptr,+ const int dststep, const int M, const int N) {+ int constexpr Vlen = 8;+ auto vN = utils::padto_le(N, Vlen);+ int j = 0;+ for (; j < vN; j += Vlen) {+ __m256 valpha;+ if constexpr (std::is_same_v<SCA_T, float>) {+ valpha = _mm256_loadu_ps(alpha + j);+ } else if constexpr (std::is_same_v<SCA_T, utils::bf16>) {+ auto tmp = _mm_loadu_si128(reinterpret_cast<const __m128i*>(alpha + j));+ valpha = ymm_cvt_bf16_fp32(tmp);+ } else if constexpr (std::is_same_v<SCA_T, utils::f8>) {+ auto ebit = _mm256_cvtepi8_epi32(_mm_loadu_si128(reinterpret_cast<const __m128i*>(alpha + j)));+ ebit = _mm256_add_epi32(_mm256_set1_epi32(127), ebit);+ valpha = _mm256_castsi256_ps(_mm256_slli_epi32(ebit, 23));+ }+ for (size_t i = 0; i < M; i++) {+ auto vsrc = _mm256_loadu_ps(srcptr + i * srcstep + j);+ auto vsrc1 = _mm256_loadu_ps(dstptr + i * dststep + j);+ auto vdst = _mm256_fmadd_ps(valpha, vsrc, vsrc1);+ _mm256_storeu_ps(dstptr + i * dststep + j, vdst);+ }+ }+ for (; j < N; j += 1) {+ for (size_t i = 0; i < M; i++) {+ if constexpr (!std::is_same_v<SCA_T, utils::f8>) {+ dstptr[i * dststep + j] += alpha[j] * srcptr[i * srcstep + j];+ } else {+ dstptr[i * dststep + j] += alpha[j].mul(srcptr[i * srcstep + j]);+ }+ }+ }+ return BTLA_CODE::Success;+}++template <int N, typename _DST_T, BTLA_DTYPE F4_T, bool MULS_T>+static inline void dequant_f4_N(_DST_T* dstptr, int8_t* srcptr, __m256* vscales, __m256 vLutL, __m256 vLutH) {+ static_assert(N % 8 == 0);+ int constexpr VLoop = N / 8;+ auto v7 = _mm256_set1_epi32(7);+ auto v8 = _mm256_set1_epi32(8);+ for (int iv = 0; iv < VLoop; iv++) {+ auto idx = _mm_loadl_epi64(reinterpret_cast<__m128i*>(srcptr + iv * 8));+ auto pad_idx = _mm256_cvtepu8_epi32(idx);+ auto mskgt8 = _mm256_cmpgt_epi32(pad_idx, v7);+ auto fp32_dq_v0 = _mm256_permutevar8x32_ps(vLutL, pad_idx);+ pad_idx = _mm256_sub_epi32(pad_idx, v8);+ auto fp32_dq_v1 = _mm256_permutevar8x32_ps(vLutH, pad_idx);+ auto fp32_dq_v = _mm256_blendv_ps(fp32_dq_v0, fp32_dq_v1, _mm256_castsi256_ps(mskgt8));+ if constexpr (MULS_T) {+ fp32_dq_v = _mm256_mul_ps(fp32_dq_v, vscales[iv]);+ }+ if constexpr (std::is_same_v<_DST_T, float>) {+ _mm256_storeu_ps(dstptr + iv * 8, fp32_dq_v);+ } else if constexpr (std::is_same_v<_DST_T, utils::bf16>) {+ auto bf16v = ymm_cvt_fp32_bf16(fp32_dq_v);+ _mm_storeu_si128(reinterpret_cast<__m128i*>(dstptr + iv * 8), bf16v);+ }+ }+}++template <int N, BTLA_DTYPE QT_T>+static inline void convert_s4_s8_N_avx2(int8_t* dstptr, int8_t* srcptr, __m256i mask) {+ static_assert(N % 2 == 0);+ static_assert(N <= 64);+ const auto vbias = _mm256_set1_epi8(8);+ if constexpr (N == 32) {+ auto dst0 = unpack_4bits(srcptr, mask);+ if constexpr (QT_T == BTLA_DTYPE::S4_CLIP) {+ dst0 = _mm256_sub_epi8(dst0, vbias);+ }+ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstptr), dst0);+ } else if constexpr (N > 32) {+ auto dst0 = unpack_4bits(srcptr, mask);+ if constexpr (QT_T == BTLA_DTYPE::S4_CLIP) {+ dst0 = _mm256_sub_epi8(dst0, vbias);+ }+ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstptr), dst0);+ int8_t temp[32];+ memcpy(temp, srcptr + 16, (N - 32) / 2);+ dst0 = unpack_4bits(temp, mask);+ if constexpr (QT_T == BTLA_DTYPE::S4_CLIP) {+ dst0 = _mm256_sub_epi8(dst0, vbias);+ }+ _mm256_storeu_si256(reinterpret_cast<__m256i*>(temp), dst0);+ memcpy(dstptr + 32, temp, (N - 32));+ } else {+ int8_t temp[32];+ memcpy(temp, srcptr, N / 2);+ auto dst0 = unpack_4bits(temp, mask);+ if constexpr (QT_T == BTLA_DTYPE::S4_CLIP) {+ dst0 = _mm256_sub_epi8(dst0, vbias);+ }+ _mm256_storeu_si256(reinterpret_cast<__m256i*>(temp), dst0);+ memcpy(dstptr, temp, N);+ }+}++template <BTLA_DTYPE F4_T, typename DST_T>+inline BTLA_CODE decompress_kblock_f4_fp_noscale(utils::f4x2* srcptr, DST_T* dstptr, int row, int col, int ld_src,+ int ld_dst, int8_t* tmp, size_t tmpsize) {+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ float* LUT;+ static_assert(F4_T == BTLA_DTYPE::F4_BNB || F4_T == BTLA_DTYPE::F4_NF4 || F4_T == BTLA_DTYPE::F4_E2M1,+ "Unsupported F4 type");+ if constexpr (F4_T == BTLA_DTYPE::F4_BNB) {+ LUT = fp4_bnb_dequant_fp32_LUT;+ } else if constexpr (F4_T == BTLA_DTYPE::F4_NF4) {+ LUT = nf4_dequant_fp32_LUT;+ } else if constexpr (F4_T == BTLA_DTYPE::F4_E2M1) {+ LUT = fp4_e2m1_dequant_fp32_LUT;+ }+ auto vLutL = _mm256_loadu_ps(LUT);+ auto vLutH = _mm256_loadu_ps(LUT + 8);+ if (col == ld_src) {+ size_t elesize = static_cast<size_t>(row) * col;+ size_t velt = utils::padto_le(elesize, 32);+ size_t i = 0;+ assert(tmpsize >= 32);+ for (; i < velt; i += 32) {+ convert_s4_s8_N_avx2<32, F4_T>(tmp, reinterpret_cast<int8_t*>(srcptr + i / 2), vmask);+ dequant_f4_N<32, DST_T, F4_T, false>(dstptr + i, tmp, nullptr, vLutL, vLutH);+ }+ for (; i < elesize; i += 2) {+ auto tmp = srcptr[i / 2];+ dstptr[i + 0] = static_cast<DST_T>(ref::f4_unpack<F4_T>(tmp.x));+ dstptr[i + 1] = static_cast<DST_T>(ref::f4_unpack<F4_T>(tmp.y));+ }+ return BTLA_CODE::Success;+ }+ return BTLA_CODE::Success;+}++template <BTLA_DTYPE QT_T, bool _IS_SYM, int _NCOL, typename _ST, typename _DST_T>+static inline BTLA_CODE decompress_kblock_bit4_packrow1(utils::bit4x2* srcptr, _DST_T* dstptr, int row, int col,+ int ld_src, int ld_dst, _ST* scales, int8_t* zero_points,+ int k_offset, int kblock, int NPad, int8_t* tmpbuf,+ size_t tmpsize) {+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ float* LUT = nullptr;+ if constexpr (QT_T == BTLA_DTYPE::F4_BNB) {+ LUT = fp4_bnb_dequant_fp32_LUT;+ } else if constexpr (QT_T == BTLA_DTYPE::F4_NF4) {+ LUT = nf4_dequant_fp32_LUT;+ } else if constexpr (QT_T == BTLA_DTYPE::F4_E2M1) {+ LUT = fp4_e2m1_dequant_fp32_LUT;+ }+ __m256 vLutL, vLutH;+ if (LUT) {+ vLutL = _mm256_loadu_ps(LUT);+ vLutH = _mm256_loadu_ps(LUT + 8);+ }+ int constexpr NReg = _NCOL / 8;+ assert(col == _NCOL);+ assert(ld_src == _NCOL);+ assert(ld_dst == _NCOL);+ __m256 vscales[NReg];+ __m256i vzps[NReg];+ int constexpr UnrollRow = 4;+ assert(kblock % UnrollRow == 0);+ int constexpr NTile = 32;+ int constexpr Loop32 = _NCOL * UnrollRow / NTile;+ assert(tmpsize >= (_NCOL * UnrollRow));+ int row0 = kblock - k_offset % kblock;+ row0 = row0 == kblock ? 0 : row0;+ row0 = row0 > row ? row : row0;+ int row1 = row - row0;+ int irow = 0;+ auto dequantize = [&](_DST_T* dstptr, int8_t* srcptr, __m256* vscales, __m256i* vzps = nullptr) {+ if constexpr (QT_T == BTLA_DTYPE::S4_CLIP) {+ dequant_s8_N_avx2<_NCOL, _IS_SYM>(dstptr, srcptr, vscales, vzps);+ } else {+ dequant_f4_N<_NCOL, _DST_T, QT_T, true>(dstptr, srcptr, vscales, vLutL, vLutH);+ }+ };+ if (row0) {+ int rowpad4 = utils::padto_le(row0, UnrollRow);+ for (int iv = 0; iv < NReg; iv++) {+ vscales[iv] = _mm256_loadu_ps(scales + (k_offset + irow) / kblock * NPad + iv * 8);+ if constexpr (!_IS_SYM) {+ auto tmp =+ _mm_loadl_epi64(reinterpret_cast<__m128i*>(zero_points + (k_offset + irow) / kblock * NPad + iv * 8));+ vzps[iv] = _mm256_cvtepi8_epi32(tmp);+ }+ }+ for (; irow < rowpad4; irow += UnrollRow) {+ for (int iter16 = 0; iter16 < Loop32; iter16++)+ convert_s4_s8_N_avx2<NTile, QT_T>(+ tmpbuf + iter16 * NTile, reinterpret_cast<int8_t*>(srcptr + irow * ld_src / 2 + NTile / 2 * iter16), vmask);+ for (int iterr = 0; iterr < UnrollRow; iterr++)+ dequantize(dstptr + (irow + iterr) * ld_dst, tmpbuf + iterr * _NCOL, vscales, vzps);+ }+ for (; irow < row0; irow++) {+ convert_s4_s8_N_avx2<_NCOL, QT_T>(tmpbuf, reinterpret_cast<int8_t*>(srcptr + irow * ld_src / 2), vmask);++ dequantize(dstptr + irow * ld_dst, tmpbuf, vscales, vzps);+ }+ }++ int row1_blk = utils::padto_le(row1, kblock) + row0;+ for (; irow < row1_blk; irow += kblock) {+ for (int iv = 0; iv < NReg; iv++) {+ vscales[iv] = _mm256_loadu_ps(scales + (k_offset + irow) / kblock * NPad + iv * 8);+ if constexpr (!_IS_SYM) {+ auto tmp =+ _mm_loadl_epi64(reinterpret_cast<__m128i*>(zero_points + (k_offset + irow) / kblock * NPad + iv * 8));+ vzps[iv] = _mm256_cvtepi8_epi32(tmp);+ }+ }+ for (int irr = 0; irr < kblock; irr += UnrollRow) {+ for (int iter16 = 0; iter16 < Loop32; iter16++)+ convert_s4_s8_N_avx2<NTile, QT_T>(+ tmpbuf + iter16 * NTile, reinterpret_cast<int8_t*>(srcptr + (irow + irr) * ld_src / 2 + NTile / 2 * iter16),+ vmask);+ for (int iterr = 0; iterr < UnrollRow; iterr++)+ dequantize(dstptr + (irow + irr + iterr) * ld_src, tmpbuf + iterr * _NCOL, vscales, vzps);+ }+ }+ if (irow < row) {+ for (int iv = 0; iv < NReg; iv++) {+ vscales[iv] = _mm256_loadu_ps(scales + (k_offset + irow) / kblock * NPad + iv * 8);+ if constexpr (!_IS_SYM) {+ auto tmp =+ _mm_loadl_epi64(reinterpret_cast<__m128i*>(zero_points + (k_offset + irow) / kblock * NPad + iv * 8));+ vzps[iv] = _mm256_cvtepi8_epi32(tmp);+ }+ }+ auto rowre = row - irow;+ int rowpad4 = utils::padto_le(rowre, UnrollRow) + irow;+ for (; irow < rowpad4; irow += UnrollRow) {+ for (int iter16 = 0; iter16 < Loop32; iter16++)+ convert_s4_s8_N_avx2<NTile, QT_T>(+ tmpbuf + iter16 * NTile, reinterpret_cast<int8_t*>(srcptr + irow * ld_src / 2 + NTile / 2 * iter16), vmask);+ for (int iterr = 0; iterr < UnrollRow; iterr++)+ dequantize(dstptr + (irow + iterr) * ld_dst, tmpbuf + iterr * _NCOL, vscales, vzps);+ }+ for (; irow < row; irow++) {+ convert_s4_s8_N_avx2<_NCOL, QT_T>(tmpbuf, reinterpret_cast<int8_t*>(srcptr + irow * ld_src / 2), vmask);+ dequantize(dstptr + irow * ld_dst, tmpbuf, vscales, vzps);+ }+ }+ return BTLA_CODE::Success;+}++template <BTLA_DTYPE S4_T, bool _IS_SYM, typename _ST, typename _DST_T>+static inline BTLA_CODE decompress_kblock_bit4_packrow2(utils::bit4x2* srcptr, _DST_T* dstptr, int row, int col,+ int ld_src, int ld_dst, _ST* scales, int8_t* zero_points,+ int k_offset, int kblock, int NPad, int8_t* tmp,+ size_t tmpsize) {+ return BTLA_CODE::NotSupport;+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s8_fp_row(int8_t* srcptr, DST_T* dstptr, int row, void* scales_, BTLA_DTYPE sdtype,+ int8_t* zero_points, int k_offset, int n_offset, int blocksize, int ldzp,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ if (zero_points == nullptr) {+ for (int ir = 0; ir < row; ir += blocksize) {+ int k_remain = utils::remainsize(ir, row, blocksize);+ int ele_off = (k_offset + ir) / blocksize * ldzp + n_offset;+ if constexpr (PackRow == 1) {+ __m256 vscale_y[NReg];+ if (sdtype == BTLA_DTYPE::F32) {+ auto sptr = (float*)scales_ + ele_off;+ for (int i = 0; i < NReg; i++) vscale_y[i] = _mm256_loadu_ps(sptr + i * 8);+ } else if (sdtype == BTLA_DTYPE::BF16) {+ auto sptr = (utils::bf16*)scales_ + ele_off;+ for (int i = 0; i < NReg; i++) vscale_y[i] = load_bf16_fp32(sptr + i * 8);+ } else if (sdtype == BTLA_DTYPE::F16) {+ auto sptr = (utils::fp16*)scales_ + ele_off;+ for (int i = 0; i < NReg; i++) vscale_y[i] = load_fp16_fp32(sptr + i * 8);+ }+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b8ptr = srcptr + (ir + ib) * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto vdeq_y = dequant_s8_fp(b8ptr + i * 8, vscale_y[i]);+ store_fp32_T(vdeq_y, dstptr + (ir + ib) * NTILE + i * 8);+ }+ }+ } else if constexpr (PackRow == 4) {+ const auto vshuf_index_y = _mm256_set_epi8(15, 14, 13, 12, 15, 14, 13, 12, 11, 10, 9, 8, 11, 10, 9, 8, 7, 6, 5,+ 4, 7, 6, 5, 4, 3, 2, 1, 0, 3, 2, 1, 0);+ __m256 vscale_y[PackRow * NReg];+ for (int i = 0; i < NReg; i++) {+ __m256 vraw;+ if (sdtype == BTLA_DTYPE::F32) {+ auto sptr = (float*)scales_ + ele_off;+ vraw = _mm256_loadu_ps(sptr + i * 8);+ } else if (sdtype == BTLA_DTYPE::BF16) {+ auto sptr = (utils::bf16*)scales_ + ele_off;+ vraw = load_bf16_fp32(sptr + i * 8);+ } else if (sdtype == BTLA_DTYPE::F16) {+ auto sptr = (utils::fp16*)scales_ + ele_off;+ vraw = load_fp16_fp32(sptr + i * 8);+ } else {+ assert(0);+ }+ auto vcast_y = broadcast_ps_1_2<true>(vraw, vshuf_index_y);+ vscale_y[i * PackRow + 0] = broadcast_ps_1_2<true>(vcast_y, vshuf_index_y);+ vscale_y[i * PackRow + 1] = broadcast_ps_1_2<false>(vcast_y, vshuf_index_y);+ vcast_y = broadcast_ps_1_2<false>(vraw, vshuf_index_y);+ vscale_y[i * PackRow + 2] = broadcast_ps_1_2<true>(vcast_y, vshuf_index_y);+ vscale_y[i * PackRow + 3] = broadcast_ps_1_2<false>(vcast_y, vshuf_index_y);+ }+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b8ptr = srcptr + (ir + ib) * NTILE;+ for (int i = 0; i < NReg; i++) {+ for (int ip = 0; ip < PackRow; ip++) {+ auto vdeq_y = dequant_s8_fp(b8ptr + i * 8 * PackRow + ip * 8, vscale_y[i * PackRow + ip]);+ store_fp32_T(vdeq_y, dstptr + (ir + ib) * NTILE + i * 8 * PackRow + ip * 8);+ }+ }+ }+ } else if constexpr (PackRow == 2) {+ const auto vshuf_index_y = _mm256_set_epi8(15, 14, 13, 12, 15, 14, 13, 12, 11, 10, 9, 8, 11, 10, 9, 8, 7, 6, 5,+ 4, 7, 6, 5, 4, 3, 2, 1, 0, 3, 2, 1, 0);+ __m256 vscale_y[PackRow * NReg];+ for (int i = 0; i < NReg; i++) {+ __m256 vraw;+ if (sdtype == BTLA_DTYPE::F32) {+ auto sptr = (float*)scales_ + ele_off;+ vraw = _mm256_loadu_ps(sptr + i * 8);+ } else if (sdtype == BTLA_DTYPE::BF16) {+ auto sptr = (utils::bf16*)scales_ + ele_off;+ vraw = load_bf16_fp32(sptr + i * 8);+ } else if (sdtype == BTLA_DTYPE::F16) {+ auto sptr = (utils::fp16*)scales_ + ele_off;+ vraw = load_fp16_fp32(sptr + i * 8);+ }+ vscale_y[i * PackRow + 0] = broadcast_ps_1_2<true>(vraw, vshuf_index_y);+ vscale_y[i * PackRow + 1] = broadcast_ps_1_2<false>(vraw, vshuf_index_y);+ }+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b8ptr = srcptr + (ir + ib) * NTILE;+ for (int i = 0; i < NReg; i++) {+ for (int ip = 0; ip < PackRow; ip++) {+ auto vdeq_y = dequant_s8_fp(b8ptr + i * 8 * PackRow + ip * 8, vscale_y[i * PackRow + ip]);+ store_fp32_T(vdeq_y, dstptr + (ir + ib) * NTILE + i * 8 * PackRow + ip * 8);+ }+ }+ }+ } else {+ assert(0);+ }+ }+ return BTLA_CODE::Success;+ } else {+ for (int ir = 0; ir < row; ir += blocksize) {+ int k_remain = utils::remainsize(ir, row, blocksize);+ int ele_off = (k_offset + ir) / blocksize * ldzp + n_offset;+ if constexpr (PackRow == 1) {+ __m256 vscale_y[NReg];+ if (sdtype == BTLA_DTYPE::F32) {+ auto sptr = (float*)scales_ + ele_off;+ for (int i = 0; i < NReg; i++) vscale_y[i] = _mm256_loadu_ps(sptr + i * 8);+ } else if (sdtype == BTLA_DTYPE::BF16) {+ auto sptr = (utils::bf16*)scales_ + ele_off;+ for (int i = 0; i < NReg; i++) vscale_y[i] = load_bf16_fp32(sptr + i * 8);+ } else if (sdtype == BTLA_DTYPE::F16) {+ auto sptr = (utils::fp16*)scales_ + ele_off;+ for (int i = 0; i < NReg; i++) vscale_y[i] = load_fp16_fp32(sptr + i * 8);+ }+ __m256i vzp_y[NReg];+ for (int i = 0; i < NReg; i++) vzp_y[i] = load_s8_s32(zero_points + ele_off + i * 8);+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b8ptr = srcptr + (ir + ib) * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto vdeq_y = dequant_s8_fp<true>(b8ptr + i * 8, vscale_y[i], vzp_y[i]);+ store_fp32_T(vdeq_y, dstptr + (ir + ib) * NTILE + i * 8);+ }+ }+ } else if constexpr (PackRow == 4) {+ const auto vshuf_index_y = _mm256_set_epi8(15, 14, 13, 12, 15, 14, 13, 12, 11, 10, 9, 8, 11, 10, 9, 8, 7, 6, 5,+ 4, 7, 6, 5, 4, 3, 2, 1, 0, 3, 2, 1, 0);+ __m256 vscale_y[PackRow * NReg];+ __m256i vzp_y[PackRow * NReg];+ for (int i = 0; i < NReg; i++) {+ __m256 vraw;+ if (sdtype == BTLA_DTYPE::F32) {+ auto sptr = (float*)scales_ + ele_off;+ vraw = _mm256_loadu_ps(sptr + i * 8);+ } else if (sdtype == BTLA_DTYPE::BF16) {+ auto sptr = (utils::bf16*)scales_ + ele_off;+ vraw = load_bf16_fp32(sptr + i * 8);+ } else if (sdtype == BTLA_DTYPE::F16) {+ auto sptr = (utils::fp16*)scales_ + ele_off;+ vraw = load_fp16_fp32(sptr + i * 8);+ } else {+ assert(0);+ }+ auto vcast_y = broadcast_ps_1_2<true>(vraw, vshuf_index_y);+ vscale_y[i * PackRow + 0] = broadcast_ps_1_2<true>(vcast_y, vshuf_index_y);+ vscale_y[i * PackRow + 1] = broadcast_ps_1_2<false>(vcast_y, vshuf_index_y);+ vcast_y = broadcast_ps_1_2<false>(vraw, vshuf_index_y);+ vscale_y[i * PackRow + 2] = broadcast_ps_1_2<true>(vcast_y, vshuf_index_y);+ vscale_y[i * PackRow + 3] = broadcast_ps_1_2<false>(vcast_y, vshuf_index_y);++ auto tmp = load_s8_s32(zero_points + ele_off + i * 8);+ auto vcasti_y = broadcast_epi32_1_2<true>(tmp, vshuf_index_y);+ vzp_y[i * PackRow + 0] = broadcast_epi32_1_2<true>(vcasti_y, vshuf_index_y);+ vzp_y[i * PackRow + 1] = broadcast_epi32_1_2<false>(vcasti_y, vshuf_index_y);+ vcasti_y = broadcast_epi32_1_2<false>(tmp, vshuf_index_y);+ vzp_y[i * PackRow + 2] = broadcast_epi32_1_2<true>(vcasti_y, vshuf_index_y);+ vzp_y[i * PackRow + 3] = broadcast_epi32_1_2<false>(vcasti_y, vshuf_index_y);+ }+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b8ptr = srcptr + (ir + ib) * NTILE;+ for (int i = 0; i < NReg; i++) {+ for (int ip = 0; ip < PackRow; ip++) {+ auto vdeq_y = dequant_s8_fp<true>(b8ptr + i * 8 * PackRow + ip * 8, vscale_y[i * PackRow + ip],+ vzp_y[i * PackRow + ip]);+ store_fp32_T(vdeq_y, dstptr + (ir + ib) * NTILE + i * 8 * PackRow + ip * 8);+ }+ }+ }+ } else if constexpr (PackRow == 2) {+ const auto vshuf_index_y = _mm256_set_epi8(15, 14, 13, 12, 15, 14, 13, 12, 11, 10, 9, 8, 11, 10, 9, 8, 7, 6, 5,+ 4, 7, 6, 5, 4, 3, 2, 1, 0, 3, 2, 1, 0);+ __m256 vscale_y[PackRow * NReg];+ __m256i vzp_y[PackRow * NReg];+ for (int i = 0; i < NReg; i++) {+ __m256 vraw;+ if (sdtype == BTLA_DTYPE::F32) {+ auto sptr = (float*)scales_ + ele_off;+ vraw = _mm256_loadu_ps(sptr + i * 8);+ } else if (sdtype == BTLA_DTYPE::BF16) {+ auto sptr = (utils::bf16*)scales_ + ele_off;+ vraw = load_bf16_fp32(sptr + i * 8);+ }+ vscale_y[i * PackRow + 0] = broadcast_ps_1_2<true>(vraw, vshuf_index_y);+ vscale_y[i * PackRow + 1] = broadcast_ps_1_2<false>(vraw, vshuf_index_y);+ auto tmp = load_s8_s32(zero_points + ele_off + i * 8);+ vzp_y[i * PackRow + 0] = broadcast_epi32_1_2<true>(tmp, vshuf_index_y);+ vzp_y[i * PackRow + 1] = broadcast_epi32_1_2<false>(tmp, vshuf_index_y);+ }+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b8ptr = srcptr + (ir + ib) * NTILE;+ for (int i = 0; i < NReg; i++) {+ for (int ip = 0; ip < PackRow; ip++) {+ auto vdeq_y = dequant_s8_fp<true>(b8ptr + i * 8 * PackRow + ip * 8, vscale_y[i * PackRow + ip],+ vzp_y[i * PackRow + ip]);+ store_fp32_T(vdeq_y, dstptr + (ir + ib) * NTILE + i * 8 * PackRow + ip * 8);+ }+ }+ }+ } else {+ assert(0);+ }+ }+ return BTLA_CODE::Success;+ }+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s8_fp(int8_t* srcptr, DST_T* dstptr, int row, int col, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ auto ret = BTLA_CODE::NotSupport;+ if (col == NTILE) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(srcptr, dstptr, head_size, scales_, sdtype, zero_points,+ k_offset, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(srcptr + head_size * NTILE, dstptr + head_size * NTILE,+ body_size, scales_, sdtype, zero_points, head_end, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ return ret;+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s4_fp_row(utils::int4x2* srcptr, DST_T* dstptr, int row, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ auto ret = decompress_kblock_s4_s8<PackRow, NTILE>(srcptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset, k_offset,+ row, NTILE, tmp, tmpsize);+ assert(ret == BTLA_CODE::Success);+ return decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(tmps8ptr, dstptr, row, scales_, sdtype, nullptr, k_offset,+ n_offset, blocksize, ldzp, tmp, tmpsize);+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s4_fp(utils::int4x2* srcptr, DST_T* dstptr, int row, int col, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ auto ret = BTLA_CODE::NotSupport;+ if (col == NTILE) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ decompress_kblock_s4_fp_row<PackRow, NTILE, DST_T>(srcptr, dstptr, head_size, scales_, sdtype, zero_points,+ k_offset, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ decompress_kblock_s4_fp_row<PackRow, NTILE, DST_T>(srcptr + head_size * NTILE / 2, dstptr + head_size * NTILE,+ body_size, scales_, sdtype, zero_points, head_end, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ return ret;+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s5_fp_row(utils::bit4x2* b4ptr, utils::bit1x8* b1ptr, DST_T* dstptr, int row,+ void* scales_, BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset,+ int n_offset, int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ auto ret = decompress_kblock_s5_s8<PackRow, NTILE>(b4ptr, b1ptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset,+ k_offset, row, NTILE, tmp, tmpsize);+ assert(ret == BTLA_CODE::Success);+ return decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(tmps8ptr, dstptr, row, scales_, sdtype, nullptr, k_offset,+ n_offset, blocksize, ldzp, tmp, tmpsize);+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s5_fp(utils::bit4x2* b4ptr, utils::bit1x8* b1ptr, DST_T* dstptr, int row, int col,+ void* scales_, BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset,+ int n_offset, int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ auto ret = BTLA_CODE::NotSupport;+ if (col == NTILE) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ decompress_kblock_s5_fp_row<PackRow, NTILE, DST_T>(b4ptr, b1ptr, dstptr, head_size, scales_, sdtype, zero_points,+ k_offset, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ decompress_kblock_s5_fp_row<PackRow, NTILE, DST_T>(+ b4ptr + head_size * NTILE / 2, b1ptr + head_size * NTILE / 8, dstptr + head_size * NTILE, body_size, scales_,+ sdtype, zero_points, head_end, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ return ret;+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s6_fp_row(utils::bit4x2* b4ptr, utils::bit2x4* b2ptr, DST_T* dstptr, int row,+ void* scales_, BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset,+ int n_offset, int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ auto ret = decompress_kblock_s6_s8<PackRow, NTILE>(b4ptr, b2ptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset,+ k_offset, row, NTILE, tmp, tmpsize);+ assert(ret == BTLA_CODE::Success);+ return decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(tmps8ptr, dstptr, row, scales_, sdtype, nullptr, k_offset,+ n_offset, blocksize, ldzp, tmp, tmpsize);+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s6_fp(utils::bit4x2* b4ptr, utils::bit2x4* b2ptr, DST_T* dstptr, int row, int col,+ void* scales_, BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset,+ int n_offset, int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ auto ret = BTLA_CODE::NotSupport;+ if (col == NTILE) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ decompress_kblock_s6_fp_row<PackRow, NTILE, DST_T>(b4ptr, b2ptr, dstptr, head_size, scales_, sdtype, zero_points,+ k_offset, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ decompress_kblock_s6_fp_row<PackRow, NTILE, DST_T>(+ b4ptr + head_size * NTILE / 2, b2ptr + head_size * NTILE / 4, dstptr + head_size * NTILE, body_size, scales_,+ sdtype, zero_points, head_end, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ return ret;+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s7_fp_row(utils::bit4x2* b4ptr, utils::bit2x4* b2ptr, utils::bit1x8* b1ptr,+ DST_T* dstptr, int row, void* scales_, BTLA_DTYPE sdtype,+ int8_t* zero_points, int k_offset, int n_offset, int blocksize, int ldzp,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ auto ret = decompress_kblock_s7_s8<PackRow, NTILE>(b4ptr, b2ptr, b1ptr, zero_points, tmps8ptr, blocksize, ldzp,+ n_offset, k_offset, row, NTILE, tmp, tmpsize);+ assert(ret == BTLA_CODE::Success);+ return decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(tmps8ptr, dstptr, row, scales_, sdtype, nullptr, k_offset,+ n_offset, blocksize, ldzp, tmp, tmpsize);+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s7_fp(utils::bit4x2* b4ptr, utils::bit2x4* b2ptr, utils::bit1x8* b1ptr,+ DST_T* dstptr, int row, int col, void* scales_, BTLA_DTYPE sdtype,+ int8_t* zero_points, int k_offset, int n_offset, int blocksize, int ldzp,+ int8_t* tmp, size_t tmpsize) {+ auto ret = BTLA_CODE::NotSupport;+ if (col == NTILE) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ decompress_kblock_s7_fp_row<PackRow, NTILE, DST_T>(b4ptr, b2ptr, b1ptr, dstptr, head_size, scales_, sdtype,+ zero_points, k_offset, n_offset, blocksize, ldzp, tmp,+ tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ decompress_kblock_s7_fp_row<PackRow, NTILE, DST_T>(b4ptr + head_size * NTILE / 2, b2ptr + head_size * NTILE / 4,+ b1ptr + head_size * NTILE / 8, dstptr + head_size * NTILE,+ body_size, scales_, sdtype, zero_points, head_end, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ return ret;+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s3_fp_row(utils::bit2x4* b2ptr, utils::bit1x8* b1ptr, DST_T* dstptr, int row,+ void* scales_, BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset,+ int n_offset, int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ auto ret = decompress_kblock_s3_s8<PackRow, NTILE>(b2ptr, b1ptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset,+ k_offset, row, NTILE, tmp, tmpsize);+ assert(ret == BTLA_CODE::Success);+ return decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(tmps8ptr, dstptr, row, scales_, sdtype, nullptr, k_offset,+ n_offset, blocksize, ldzp, tmp, tmpsize);+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s3_fp(utils::bit2x4* b2ptr, utils::bit1x8* b1ptr, DST_T* dstptr, int row, int col,+ void* scales_, BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset,+ int n_offset, int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ auto ret = BTLA_CODE::NotSupport;+ if (col == NTILE) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ decompress_kblock_s3_fp_row<PackRow, NTILE, DST_T>(b2ptr, b1ptr, dstptr, head_size, scales_, sdtype, zero_points,+ k_offset, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ decompress_kblock_s3_fp_row<PackRow, NTILE, DST_T>(+ b2ptr + head_size * NTILE / 4, b1ptr + head_size * NTILE / 8, dstptr + head_size * NTILE, body_size, scales_,+ sdtype, zero_points, head_end, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ return ret;+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s1_fp_row(utils::bit1x8* b1ptr, DST_T* dstptr, int row, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ auto ret = decompress_kblock_s1_s8<PackRow, NTILE>(b1ptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset, k_offset,+ row, NTILE, tmp, tmpsize);+ assert(ret == BTLA_CODE::Success);+ return decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(tmps8ptr, dstptr, row, scales_, sdtype, nullptr, k_offset,+ n_offset, blocksize, ldzp, tmp, tmpsize);+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s1_fp(utils::bit1x8* b1ptr, DST_T* dstptr, int row, int col, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ auto ret = BTLA_CODE::NotSupport;+ if (col == NTILE) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ decompress_kblock_s1_fp_row<PackRow, NTILE, DST_T>(b1ptr, dstptr, head_size, scales_, sdtype, zero_points,+ k_offset, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ decompress_kblock_s1_fp_row<PackRow, NTILE, DST_T>(b1ptr + head_size * NTILE / 8, dstptr + head_size * NTILE,+ body_size, scales_, sdtype, zero_points, head_end, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ return ret;+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s2_fp_row(utils::bit2x4* b2ptr, DST_T* dstptr, int row, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ auto ret = decompress_kblock_s2_s8<PackRow, NTILE>(b2ptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset, k_offset,+ row, NTILE, tmp, tmpsize);+ assert(ret == BTLA_CODE::Success);+ return decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(tmps8ptr, dstptr, row, scales_, sdtype, nullptr, k_offset,+ n_offset, blocksize, ldzp, tmp, tmpsize);+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s2_fp(utils::bit2x4* b2ptr, DST_T* dstptr, int row, int col, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ auto ret = BTLA_CODE::NotSupport;+ if (col == NTILE) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ decompress_kblock_s2_fp_row<PackRow, NTILE, DST_T>(b2ptr, dstptr, head_size, scales_, sdtype, zero_points,+ k_offset, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ decompress_kblock_s2_fp_row<PackRow, NTILE, DST_T>(b2ptr + head_size * NTILE / 4, dstptr + head_size * NTILE,+ body_size, scales_, sdtype, zero_points, head_end, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ return ret;+}++template <BTLA_DTYPE _F4_T, typename _DST_T, int _PACK_ROW, typename _ST>+static inline BTLA_CODE decompress_kblock_f4_fp(utils::f4x2* srcptr, _DST_T* dstptr, int row, int col, int ld_src,+ int ld_dst, _ST* scales, int k_offset, int kblock, int NPad,+ int8_t* tmp, size_t tmpsize) {+ if constexpr (_PACK_ROW == 1) {+ if (col == 24) {+ return decompress_kblock_bit4_packrow1<_F4_T, true, 24, _ST, _DST_T>(+ srcptr, dstptr, row, col, ld_src, ld_dst, scales, nullptr, k_offset, kblock, NPad, tmp, tmpsize);+ }+ if (col == 48) {+ return decompress_kblock_bit4_packrow1<_F4_T, true, 48, _ST, _DST_T>(+ srcptr, dstptr, row, col, ld_src, ld_dst, scales, nullptr, k_offset, kblock, NPad, tmp, tmpsize);+ }+ } else if constexpr (_PACK_ROW == 2) {+ return decompress_kblock_bit4_packrow2<_F4_T, true, _ST, _DST_T>(srcptr, dstptr, row, col, ld_src, ld_dst, scales,+ nullptr, k_offset, kblock, NPad, tmp, tmpsize);+ }+ assert(0);+ return BTLA_CODE::NotSupport;+}++enum class AVX2_REDUCE_TYPE { MAX, MIN, ADD };+#define AVX2_REDUCE_OP \+ if constexpr (TYPE == AVX2_REDUCE_TYPE::MAX) x = _mm256_max_ps(x, y); \+ if constexpr (TYPE == AVX2_REDUCE_TYPE::MIN) x = _mm256_min_ps(x, y); \+ if constexpr (TYPE == AVX2_REDUCE_TYPE::ADD) x = _mm256_add_ps(x, y);++template <AVX2_REDUCE_TYPE TYPE>+inline float avx2_reduce_ps(__m256 x) {+ __m256 y = _mm256_permute2f128_ps(x, x, 1);+ AVX2_REDUCE_OP+ y = _mm256_permute_ps(x, 0b01001110);+ AVX2_REDUCE_OP+ y = _mm256_permute_ps(x, 0b10110001);+ AVX2_REDUCE_OP+ return _mm256_cvtss_f32(x);+}++#define AVX2_REDUCE_OP_EPI32(dst, src) \+ if constexpr (TYPE == AVX2_REDUCE_TYPE::MAX) dst = _mm256_max_epi32(dst, src); \+ if constexpr (TYPE == AVX2_REDUCE_TYPE::MIN) dst = _mm256_min_epi32(dst, src); \+ if constexpr (TYPE == AVX2_REDUCE_TYPE::ADD) dst = _mm256_add_epi32(dst, src);++#ifndef _mm256_cvtsi256_si32+#define _mm256_cvtsi256_si32(a) (_mm_cvtsi128_si32(_mm256_castsi256_si128(a)))+#endif++template <AVX2_REDUCE_TYPE TYPE>+inline int avx2_reduce_epi32(__m256i xd) {+ auto x = _mm256_castsi256_ps(xd);+ __m256 y = _mm256_permute2f128_ps(x, x, 1);+ auto yd = _mm256_castps_si256(y);+ AVX2_REDUCE_OP_EPI32(xd, yd);+ x = _mm256_castsi256_ps(xd);+ y = _mm256_permute_ps(x, 0b01001110);+ yd = _mm256_castps_si256(y);+ AVX2_REDUCE_OP_EPI32(xd, yd);+ x = _mm256_castsi256_ps(xd);+ y = _mm256_permute_ps(x, 0b10110001);+ yd = _mm256_castps_si256(y);+ AVX2_REDUCE_OP_EPI32(xd, yd);+ return _mm256_cvtsi256_si32(xd);+}++inline __m128i avx2_cvtepi32_epu8(__m256i x) {+ auto out_v = _mm_packus_epi32(_mm256_castsi256_si128(x), _mm256_extractf128_si256(x, 1));+ out_v = _mm_packus_epi16(out_v, out_v);+ return out_v;+}++template <typename SRC_T>+static inline BTLA_CODE quantize_fp_u8_colblock(int row, int col, const SRC_T* srcptr, int ld_src, uint8_t* dstptr,+ int ld_dst, float* scales, int ld_scale, uint8_t* zps, int blocksize,+ float* blkreduce) {+ int constexpr VLen = 8;+ auto vff = _mm256_set1_epi32(255);+ auto v0 = _mm256_set1_epi32(0);+ int constexpr Unroll = 2;+ int vblocksize_un = utils::padto_le(blocksize, VLen * Unroll);+ int vblocksize = utils::padto_le(blocksize, VLen);+ int colblk = utils::padto_le(col, blocksize);+ for (size_t i = 0; i < row; i++) {+ size_t j = 0;+ for (; j < colblk; j += blocksize) {+ __m256 vmaxval = _mm256_set1_ps(0.f);+ __m256 vminval = _mm256_set1_ps(0.f);+ size_t ij = 0;+ for (; ij < vblocksize_un; ij += VLen * Unroll) {+ for (size_t iu = 0; iu < Unroll; iu++) {+ __m256 vsrc = load_T_fp32(&srcptr[(j + ij) + i * ld_src + iu * VLen]);+ vmaxval = _mm256_max_ps(vmaxval, vsrc);+ vminval = _mm256_min_ps(vminval, vsrc);+ }+ }+ if (ij + VLen < vblocksize) {+ for (; ij < vblocksize; ij += VLen) {+ __m256 vsrc = load_T_fp32(&srcptr[(j + ij) + i * ld_src]);+ vmaxval = _mm256_max_ps(vmaxval, vsrc);+ vminval = _mm256_min_ps(vminval, vsrc);+ }+ }+ auto maxval = avx2_reduce_ps<AVX2_REDUCE_TYPE::MAX>(vmaxval);+ auto minval = avx2_reduce_ps<AVX2_REDUCE_TYPE::MIN>(vminval);+ if (ij < blocksize) {+ for (; ij < blocksize; ij++) {+ auto srcval = (float)srcptr[(j + ij) + i * ld_src];+ maxval = std::max(maxval, srcval);+ minval = std::min(minval, srcval);+ }+ }+ float scale = (maxval - minval) / 255;+ uint8_t zp = utils::cast<float, uint8_t>((0 - minval) / scale);+ scales[j / blocksize + i * ld_scale] = scale;+ zps[j / blocksize + i * ld_scale] = zp;+ int sum = 0;+ float rscale = 1.f / scale;+ auto vrscale = _mm256_set1_ps(rscale);+ auto vdzp = _mm256_set1_epi32(zp);+ ij = 0;+ if (blkreduce) {+ for (; ij < vblocksize; ij += VLen) {+ __m256 vsrc = load_T_fp32(&srcptr[(j + ij) + i * ld_src]);+ vsrc = _mm256_mul_ps(vsrc, vrscale);+ auto vdsrc = _mm256_cvtps_epi32(vsrc);+ sum += avx2_reduce_epi32<AVX2_REDUCE_TYPE::ADD>(vdsrc);+ vdsrc = _mm256_add_epi32(vdsrc, vdzp);+ vdsrc = _mm256_min_epi32(vdsrc, vff);+ vdsrc = _mm256_max_epi32(vdsrc, v0);+ auto vbsrc = avx2_cvtepi32_epu8(vdsrc);+ _mm_storel_epi64(reinterpret_cast<__m128i*>(&dstptr[(j + ij) + i * ld_dst]), vbsrc);+ }+ } else {+ for (; ij < vblocksize_un; ij += VLen * Unroll) {+ for (size_t iu = 0; iu < Unroll; iu++) {+ __m256 vsrc = load_T_fp32(&srcptr[(j + ij) + i * ld_src + iu * VLen]);+ vsrc = _mm256_mul_ps(vsrc, vrscale);+ auto vdsrc = _mm256_cvtps_epi32(vsrc);+ vdsrc = _mm256_add_epi32(vdsrc, vdzp);+ vdsrc = _mm256_min_epi32(vdsrc, vff);+ vdsrc = _mm256_max_epi32(vdsrc, v0);+ auto vbsrc = avx2_cvtepi32_epu8(vdsrc);+ _mm_storel_epi64(reinterpret_cast<__m128i*>(&dstptr[(j + ij) + i * ld_dst + iu * VLen]), vbsrc);+ }+ }+ if (ij + VLen < vblocksize) {+ for (; ij < vblocksize; ij += VLen) {+ __m256 vsrc = load_T_fp32(&srcptr[(j + ij) + i * ld_src]);+ vsrc = _mm256_mul_ps(vsrc, vrscale);+ auto vdsrc = _mm256_cvtps_epi32(vsrc);+ vdsrc = _mm256_add_epi32(vdsrc, vdzp);+ vdsrc = _mm256_min_epi32(vdsrc, vff);+ vdsrc = _mm256_max_epi32(vdsrc, v0);+ auto vbsrc = avx2_cvtepi32_epu8(vdsrc);+ _mm_storel_epi64(reinterpret_cast<__m128i*>(&dstptr[(j + ij) + i * ld_dst]), vbsrc);+ }+ }+ }+ for (; ij < blocksize; ij++) {+ auto srcval = (float)srcptr[(j + ij) + i * ld_src];+ srcval = srcval * rscale;+ auto srcint = int(roundf(srcval));+ sum += srcint;+ srcint += zp;+ srcint = std::min(srcint, 0xff);+ srcint = std::max(srcint, 0);+ dstptr[(j + ij) + i * ld_dst] = static_cast<uint8_t>(srcint);+ }+ if (blkreduce) {+ blkreduce[j / blocksize + i * ld_scale] = sum * scale;+ }+ }+ if (j < col) {+ float maxval = 0.f;+ float minval = 0.f;+ for (size_t ij = j; ij < col; ij++) {+ maxval = std::max((float)srcptr[ij + i * ld_src], maxval);+ minval = std::min((float)srcptr[ij + i * ld_src], minval);+ }+ float scale = (maxval - minval) / 255;+ uint8_t zp = utils::cast<float, uint8_t>((0 - minval) / scale);+ float rscale = 1.f / scale;+ scales[j / blocksize + i * ld_scale] = scale;+ zps[j / blocksize + i * ld_scale] = zp;+ int sum = 0;+ for (size_t ij = j; ij < col; ij++) {+ auto srcint = utils::cast<float, int>(srcptr[ij + i * ld_src] * rscale);+ sum += srcint;+ srcint += zp;+ srcint = srcint <= 255 ? srcint : 255;+ srcint = srcint >= 0 ? srcint : 0;+ dstptr[ij + i * ld_dst] = utils::cast<int, uint8_t>(srcint);+ }+ if (blkreduce) {+ blkreduce[j / blocksize + i * ld_scale] = sum * scale;+ }+ }+ }+ return BTLA_CODE::Success;+}++template <typename SRC_T>+static inline BTLA_CODE col_block_reduce_sum(const SRC_T* srcptr, int ldsrc, int row, int col, int blocksize,+ float* reduce, int ldr) {+ int constexpr VLen = 8;+ auto vblock2_ = utils::padto_le(blocksize, VLen * 2);+ auto vblock_ = utils::padto_le(blocksize, VLen);+ for (int i = 0; i < row; i++) {+ for (int j = 0; j < col; j += blocksize) {+ auto tmp = 0.f;+ auto vsum = _mm256_set1_ps(0.f);+ int jj = 0;+ auto vblock2 = j + vblock2_ <= col ? vblock2_ : 0;+ auto vblock = j + vblock_ <= col ? vblock_ : 0;+ for (; jj < vblock2; jj += VLen * 2) {+ auto vtmp = _mm256_loadu_ps(srcptr + i * ldsrc + j + jj);+ auto vtmp1 = _mm256_loadu_ps(srcptr + i * ldsrc + j + jj + VLen);+ auto s0 = avx2_reduce_ps<AVX2_REDUCE_TYPE::ADD>(vtmp);+ auto s1 = avx2_reduce_ps<AVX2_REDUCE_TYPE::ADD>(vtmp1);+ tmp += s0;+ tmp += s1;+ }+ if (jj + VLen <= vblock) {+ for (; jj < vblock; jj += VLen) {+ auto vtmp = _mm256_loadu_ps(srcptr + i * ldsrc + j + jj);+ auto s0 = avx2_reduce_ps<AVX2_REDUCE_TYPE::ADD>(vtmp);+ tmp += s0;+ }+ }+ for (; jj < blocksize; jj++) {+ tmp += *(srcptr + i * ldsrc + j + jj);+ }+ reduce[i * ldr + j / blocksize] = tmp;+ }+ }+ return BTLA_CODE::Success;+}++template <typename T>+static inline BTLA_CODE cvt_fp32_T_2D(const float* src_ptr, T* dst_ptr, int row, int col, int src_step, int dst_step,+ bool zeropadding) {+ const int npadding = (dst_step - col) * sizeof(T);+ constexpr int simd_proc_elt = 8;+ float tmpbuf[simd_proc_elt];+ auto col_body = col / simd_proc_elt * simd_proc_elt;+ auto col_tail = col % simd_proc_elt;+ for (int i = 0; i < row; i++) {+ const auto src = src_ptr + i * src_step;+ const auto dst = dst_ptr + i * dst_step;+ int j = 0;+ for (; j < col_body; j += simd_proc_elt) {+ store_fp32_T(_mm256_loadu_ps(src + j), dst + j);+ }+ if (col_tail > 0) {+ memcpy(tmpbuf, src + j, col_tail * sizeof(src_ptr[0]));+ auto vf32 = _mm256_loadu_ps(tmpbuf);+ store_fp32_T(vf32, (T*)tmpbuf);+ memcpy(dst + j, tmpbuf, col_tail * sizeof(dst[0]));+ }+ if (zeropadding && npadding) std::memset(dst + col, 0, npadding);+ }+ return BTLA_CODE::Success;+}++template <typename T>+static inline BTLA_CODE cvt_T_fp32_2D(const T* src_ptr, float* dst_ptr, int row, int col, int src_step, int dst_step,+ bool zeropadding) {+ const int npadding = (dst_step - col) * sizeof(float);+ constexpr int simd_proc_elt = 8;+ auto col_body = col / simd_proc_elt * simd_proc_elt;+ auto col_tail = col % simd_proc_elt;+ float tmpbuf[simd_proc_elt];+ for (int i = 0; i < row; i++) {+ const auto src = src_ptr + i * src_step;+ const auto dst = dst_ptr + i * dst_step;+ int j = 0;+ for (; j < col_body; j += simd_proc_elt) {+ auto vf32 = load_T_fp32(src + j);+ _mm256_storeu_ps(dst + j, vf32);+ }+ if (col_tail > 0) {+ memcpy(tmpbuf, src + j, col_tail * sizeof(src_ptr[0]));+ auto vf32 = load_T_fp32((T*)tmpbuf);+ _mm256_storeu_ps(tmpbuf, vf32);+ memcpy(dst + j, tmpbuf, col_tail * sizeof(dst[0]));+ }+ if (zeropadding && npadding) std::memset(dst + col, 0, npadding);+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE layernorm(const float* srcptr, const float* scaleptr, const float* biasptr, float epsilon,+ int norm_size, float* dstptr, float* mean_out, float* mean_square_out,+ bool simplified) {+ int constexpr VLen = 8;+ int norm_size8 = utils::padto_le(norm_size, VLen);+ int h = 0;+ __m256 vmean = _mm256_setzero_ps(), vmeansq = _mm256_setzero_ps();+ for (; h < norm_size8; h += VLen) {+ auto tmp = _mm256_loadu_ps(srcptr + h);+ vmean = _mm256_add_ps(vmean, tmp);+ tmp = _mm256_mul_ps(tmp, tmp);+ vmeansq = _mm256_add_ps(vmeansq, tmp);+ }+ float mean = avx2_reduce_ps<AVX2_REDUCE_TYPE::ADD>(vmean);+ float mean_square = avx2_reduce_ps<AVX2_REDUCE_TYPE::ADD>(vmeansq);+ for (; h < norm_size; h++) {+ mean += srcptr[h];+ mean_square += srcptr[h] * srcptr[h];+ }+ mean = mean / norm_size;+ if (simplified) {+ mean_square = std::sqrt(mean_square / norm_size + epsilon);+ } else {+ mean_square = std::sqrt(mean_square / norm_size - mean * mean + epsilon);+ }+ auto vm = _mm256_set1_ps(mean);+ float inv_meansq = 1.f / mean_square;+ auto vms = _mm256_set1_ps(inv_meansq);+ h = 0;+ if (simplified) {+ if (scaleptr) {+ for (; h < norm_size8; h += VLen) {+ auto inp = _mm256_loadu_ps(srcptr + h);+ auto scale = _mm256_loadu_ps(scaleptr + h);+ inp = _mm256_mul_ps(inp, scale);+ inp = _mm256_mul_ps(inp, vms);+ _mm256_storeu_ps(dstptr + h, inp);+ }+ for (; h < norm_size; h++) {+ dstptr[h] = srcptr[h] * inv_meansq * scaleptr[h];+ }+ } else {+ for (; h < norm_size8; h += VLen) {+ auto inp = _mm256_loadu_ps(srcptr + h);+ inp = _mm256_mul_ps(inp, vms);+ _mm256_storeu_ps(dstptr + h, inp);+ }+ for (; h < norm_size; h++) {+ dstptr[h] = srcptr[h] * inv_meansq;+ }+ }++ } else {+ if (scaleptr) {+ if (biasptr == nullptr) {+ for (; h < norm_size8; h += VLen) {+ auto inp = _mm256_loadu_ps(srcptr + h);+ auto scale = _mm256_loadu_ps(scaleptr + h);+ inp = _mm256_sub_ps(inp, vm);+ inp = _mm256_mul_ps(inp, scale);+ inp = _mm256_mul_ps(inp, vms);+ _mm256_storeu_ps(dstptr + h, inp);+ }+ for (; h < norm_size; h++) {+ dstptr[h] = (srcptr[h] - mean) * inv_meansq * scaleptr[h];+ }+ } else {+ for (; h < norm_size8; h += VLen) {+ auto inp = _mm256_loadu_ps(srcptr + h);+ auto scale = _mm256_loadu_ps(scaleptr + h);+ inp = _mm256_sub_ps(inp, vm);+ inp = _mm256_mul_ps(inp, vms);+ inp = _mm256_mul_ps(inp, scale);+ auto bias = _mm256_loadu_ps(biasptr + h);+ inp = _mm256_add_ps(inp, bias);+ _mm256_storeu_ps(dstptr + h, inp);+ }+ for (; h < norm_size; h++) {+ dstptr[h] = (srcptr[h] - mean) * inv_meansq * scaleptr[h] + biasptr[h];+ }+ }+ } else {+ for (; h < norm_size8; h += VLen) {+ auto inp = _mm256_loadu_ps(srcptr + h);+ inp = _mm256_sub_ps(inp, vm);+ inp = _mm256_mul_ps(inp, vms);+ _mm256_storeu_ps(dstptr + h, inp);+ }+ for (; h < norm_size; h++) {+ dstptr[h] = (srcptr[h] - mean) * inv_meansq;+ }+ }+ }++ if (mean_out) {+ *mean_out = mean;+ }+ if (mean_square_out) {+ *mean_square_out = mean_square;+ }+ return BTLA_CODE::Success;+}++template <BTLA_DTYPE S3_T, typename _DST_T>+inline BTLA_CODE decompress_kblock_s3_s8fp(utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr, _DST_T* dstptr,+ int interleave_n_offset, int unpack_elt, int8_t* tmp, size_t tmpsize) {+ auto head_ignore_num = interleave_n_offset % 128;+ const __m256i lowMask = _mm256_set1_epi8(0x03);+ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));++ auto bit3_interleave_decompress_pack128 = [&](utils::bit2x4* src1, utils::bit1x8* src2, int8_t* dst) {+ __m256i bit2_data = _mm256_loadu_si256((const __m256i*)src1);+ int32_t* bit1_ptr = reinterpret_cast<int32_t*>(src2);+ for (int i = 0; i < 4; i++) {+ auto bit1x32 = _mm256_set1_epi32(bit1_ptr[i]);+ bit1x32 = _mm256_srlv_epi32(bit1x32, bit1Shift_1);+ bit1x32 = _mm256_and_si256(bit1x32, bit1Mask);+ bit1x32 = _mm256_mullo_epi32(bit1x32, bit1Shift_2);+ bit1x32 = _mm256_and_si256(highMask, bit1x32);++ auto bit2x32 = _mm256_and_si256(lowMask, _mm256_srli_epi16(bit2_data, 2 * i));+ auto res = _mm256_add_epi8(bit1x32, bit2x32);+ res = _mm256_sub_epi8(res, highMask);+ _mm256_storeu_si256((__m256i*)(dst + 32 * i), res);+ }+ };+ int compress_wei_ptr_offset = 0;+ if (head_ignore_num != 0) {+ assert(head_ignore_num % 8 == 0);++ auto base_bit2ptr = bit2ptr - head_ignore_num / 4;+ auto base_bit1ptr = bit1ptr - head_ignore_num / 8;+ auto head_write_num = 128 - head_ignore_num;+ bit3_interleave_decompress_pack128(base_bit2ptr, base_bit1ptr, tmp);+ for (int i = 0; i < head_write_num; i++) dstptr[i] = tmp[head_ignore_num + i];+ compress_wei_ptr_offset += head_write_num;+ unpack_elt -= head_write_num;+ }+ auto body_loop = unpack_elt / 128;+ auto tail_proc_num = unpack_elt % 128;++ bestla::kernel::jit::DecompressS3::forward_avx2(bit2ptr + compress_wei_ptr_offset / 4,+ bit1ptr + compress_wei_ptr_offset / 8,+ dstptr + compress_wei_ptr_offset, tmp, body_loop * 128);+ compress_wei_ptr_offset += body_loop * 128;+ if (tail_proc_num > 0) {+ bit3_interleave_decompress_pack128(bit2ptr + compress_wei_ptr_offset / 4, bit1ptr + compress_wei_ptr_offset / 8,+ tmp);+ for (int i = 0; i < tail_proc_num; i++) dstptr[compress_wei_ptr_offset + i] = tmp[i];+ }+ return BTLA_CODE::Success;+}++template <BTLA_DTYPE _S3_T, typename _DST_T, int _PACK_ROW, typename _ST>+static inline BTLA_CODE decompress_kblock_bit3_packrow_fp(utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr,+ _DST_T* dstptr, int interleave_n_offset, int row, int col,+ _ST* scales, int8_t* zero_points, int k_offset, int kblock,+ int NPad, void* tmp, size_t tmpsize) {+ auto unpack_elt = row * col;+ decompress_kblock_s3_s8fp<_S3_T>(bit2ptr, bit1ptr, dstptr, interleave_n_offset, unpack_elt,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ // TODO(zhe): simd version+ for (int i = 0; i < row; i++) {+ int kpos = (k_offset + i) / kblock;+ auto sptr = scales + kpos * NPad;+ for (int j = 0; j < col; j++) {+ float tmp = static_cast<float>(dstptr[i * col + j]);+ if (zero_points != nullptr) tmp -= static_cast<float>(zero_points[kpos * NPad + j / _PACK_ROW]);+ dstptr[i * col + j] = static_cast<_DST_T>(tmp * sptr[j / _PACK_ROW]);+ }+ }++ return BTLA_CODE::Success;+}++template <BTLA_DTYPE S2_T, typename _DST_T>+inline BTLA_CODE decompress_kblock_s2_s8fp(utils::bit2x4* bit2ptr, _DST_T* dstptr, int unpack_elt, int8_t* tmp,+ size_t tmpsize) {+ int constexpr VBits = 256;+ int constexpr VElt = VBits / 8;+ int i = 0;+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ int elt_pad = utils::padto_le(unpack_elt, VElt);+ for (; i < elt_pad; i += VElt) {+ auto vout = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ if (std::is_same_v<_DST_T, int8_t>) {+ _mm256_storeu_si256((__m256i*)(dstptr + i), vout);+ } else {+ _mm256_storeu_si256((__m256i*)tmp, vout);+ for (int j = 0; j < VElt; j++) {+ dstptr[i + j] = tmp[j];+ }+ }+ }+ ref::decompress_kblock_s2_s8fp<S2_T, _DST_T>(bit2ptr + i / 4, dstptr + i, unpack_elt - i, tmp, tmpsize);+ return BTLA_CODE::Success;+}++template <BTLA_DTYPE _S2_T, typename _DST_T, int _PACK_ROW, typename _ST>+static inline BTLA_CODE decompress_kblock_bit2_packrow_fp(utils::bit2x4* bit2ptr, _DST_T* dstptr, int row, int col,+ _ST* scales, int8_t* zero_points, int k_offset, int kblock,+ int NPad, void* tmp, size_t tmpsize) {+ auto unpack_elt = row * col;+ decompress_kblock_s2_s8fp<_S2_T>(bit2ptr, dstptr, unpack_elt, reinterpret_cast<int8_t*>(tmp), tmpsize);+ // TODO(zhe): simd version+ for (int i = 0; i < row; i++) {+ int kpos = (k_offset + i) / kblock;+ auto sptr = scales + kpos * NPad;+ for (int j = 0; j < col; j++) {+ float tmp = static_cast<float>(dstptr[i * col + j]);+ if (zero_points != nullptr) tmp -= static_cast<float>(zero_points[kpos * NPad + j / _PACK_ROW]);+ dstptr[i * col + j] = static_cast<_DST_T>(tmp * sptr[j / _PACK_ROW]);+ }+ }++ return BTLA_CODE::Success;+}++inline __m256 poly_scale_2nd_ps(const __m256i z, const __m256 f, const __m256 c0, const __m256 c1, const __m256 c2) {+ const auto y = _mm256_fmadd_ps(_mm256_fmadd_ps(f, c0, c1), f, c2); // auto y = (f * c0 + c1) * f + c2;+ static const auto mask_exp = _mm256_set1_epi32(0x7f800000);+ static const auto mask_not_exp = _mm256_set1_epi32(~0x7f800000);++ const auto y_exp = _mm256_and_si256(_mm256_castps_si256(y), mask_exp);+ const auto y_not_exp = _mm256_and_si256(_mm256_castps_si256(y), mask_not_exp);++ const auto y_exp_scaled = _mm256_add_epi32(y_exp, _mm256_slli_epi32(z, 23));+ return _mm256_castsi256_ps(_mm256_or_si256(y_not_exp, _mm256_and_si256(y_exp_scaled, mask_exp)));+}++inline __m256 exp_ps_0_1(const __m256 x) {+ static const auto c0 = _mm256_set1_ps(0.240226507f);+ static const auto c1 = _mm256_set1_ps(0.452920674f);+ static const auto c2 = _mm256_set1_ps(0.713483036f);+ static const float v_log2e = std::log2(std::exp(1.f));+ static const auto log2e = _mm256_set1_ps(v_log2e);+ static const auto half = _mm256_set1_ps(.5f);++ static const auto upper_bound = _mm256_set1_ps(88.722838f); // log(max_positive_float)+ static const auto lower_bound = _mm256_set1_ps(-87.336549f); // log(min_positive_float)+ __m256 x1 = _mm256_min_ps(x, upper_bound);+ x1 = _mm256_max_ps(x1, lower_bound);++ x1 = _mm256_fmadd_ps(x1, log2e, half); // auto x1 = x * log2e + _mm256_set1_ps(.5f);+ const auto z = _mm256_floor_ps(x1);+ const auto f = _mm256_sub_ps(x1, z); // auto f = x1 - z;++ return poly_scale_2nd_ps(_mm256_cvtps_epi32(z), f, c0, c1, c2);+}++#ifdef __GNUC__+#pragma GCC diagnostic push+#pragma GCC diagnostic ignored "-Wignored-attributes" // https://stackoverflow.com/a/49216021+#endif+// Interleave 8 xmm vectors of words inplace+static inline std::array<__m128i, 8> tr_x8_word(std::array<__m128i, 8>& src) { // NOLINT [runtime/references]+ std::array<__m128i, 8> dst;++ for (int i = 0; i < 8; i += 2) {+ dst[i + 0] = _mm_unpacklo_epi16(src[i + 0], src[i + 1]);+ dst[i + 1] = _mm_unpackhi_epi16(src[i + 0], src[i + 1]);+ }+ for (int i = 0; i < 8; i += 4) {+ src[i + 0] = _mm_unpacklo_epi32(dst[i + 0], dst[i + 2]);+ src[i + 1] = _mm_unpackhi_epi32(dst[i + 0], dst[i + 2]);+ src[i + 2] = _mm_unpacklo_epi32(dst[i + 1], dst[i + 3]);+ src[i + 3] = _mm_unpackhi_epi32(dst[i + 1], dst[i + 3]);+ }+ dst[0] = _mm_unpacklo_epi64(src[0], src[4]);+ dst[1] = _mm_unpackhi_epi64(src[0], src[4]);+ dst[2] = _mm_unpacklo_epi64(src[1], src[5]);+ dst[3] = _mm_unpackhi_epi64(src[1], src[5]);+ dst[4] = _mm_unpacklo_epi64(src[2], src[6]);+ dst[5] = _mm_unpackhi_epi64(src[2], src[6]);+ dst[6] = _mm_unpacklo_epi64(src[3], src[7]);+ dst[7] = _mm_unpackhi_epi64(src[3], src[7]);+ return dst;+}++template <int tail>+inline std::array<__m128i, 8> load_fp32_fp16_tr_x8_word(const float* a, size_t lda) {+ static_assert(tail > 0 && tail <= 8, "Unexpected tail value.");+ std::array<__m128i, 8> dst;+ for (int i = 0; i < tail; ++i) {+ dst[i] = _mm256_cvtps_ph(_mm256_loadu_ps(a + i * lda), _MM_FROUND_TO_NEAREST_INT);+ }+ for (int i = tail; i < 8; ++i) dst[i] = _mm_setzero_si128();+ return tr_x8_word(dst);+}+constexpr decltype(load_fp32_fp16_tr_x8_word<1>)* load_fp32_fp16_tr_x8_word_tbl[9]{+ load_fp32_fp16_tr_x8_word<1>, load_fp32_fp16_tr_x8_word<1>, load_fp32_fp16_tr_x8_word<2>,+ load_fp32_fp16_tr_x8_word<3>, load_fp32_fp16_tr_x8_word<4>, load_fp32_fp16_tr_x8_word<5>,+ load_fp32_fp16_tr_x8_word<6>, load_fp32_fp16_tr_x8_word<7>, load_fp32_fp16_tr_x8_word<8>};++template <int tail>+inline std::array<__m128i, 8> load_maskz_fp32_fp16_tr_x8_word(const float* a, size_t lda, __m256i mask) {+ static_assert(tail > 0 && tail <= 8, "Unexpected tail value.");+ std::array<__m128i, 8> dst;+ for (int i = 0; i < tail; ++i) {+ dst[i] = _mm256_cvtps_ph(_mm256_maskload_ps(a + i * lda, mask), _MM_FROUND_TO_NEAREST_INT);+ }+ for (int i = tail; i < 8; ++i) dst[i] = _mm_setzero_si128();+ return tr_x8_word(dst);+}+constexpr decltype(load_maskz_fp32_fp16_tr_x8_word<1>)* load_maskz_fp32_fp16_tr_x8_word_tbl[9]{+ load_maskz_fp32_fp16_tr_x8_word<1>, load_maskz_fp32_fp16_tr_x8_word<1>, load_maskz_fp32_fp16_tr_x8_word<2>,+ load_maskz_fp32_fp16_tr_x8_word<3>, load_maskz_fp32_fp16_tr_x8_word<4>, load_maskz_fp32_fp16_tr_x8_word<5>,+ load_maskz_fp32_fp16_tr_x8_word<6>, load_maskz_fp32_fp16_tr_x8_word<7>, load_maskz_fp32_fp16_tr_x8_word<8>};++#ifdef __GNUC__+#pragma GCC diagnostic pop+#endif++template <int MTILE, int NReg, int Unroll>+static inline void accumulate_fp32_s8_fp32(const float* Aptr, int lda, int8_t* Bptr, __m256* vacc, __m256* vsca) {+ if constexpr (MTILE == 1) {+ for (int ikk = 0; ikk < Unroll; ikk++) {+ __m256 va = _mm256_set1_ps(*(Aptr + ikk));+ for (int i = 0; i < NReg; i++) {+ auto ftmp = load_s8_fp32(Bptr + i * 8 + ikk * NReg * 8);+ ftmp = _mm256_mul_ps(ftmp, vsca[i]);+ vacc[i] = _mm256_fmadd_ps(va, ftmp, vacc[i]);+ }+ }+ } else {+ for (int ikk = 0; ikk < Unroll; ikk++) {+ __m256 va[MTILE];+ for (int i = 0; i < NReg; i++) {+ auto ftmp = load_s8_fp32(Bptr + i * 8 + ikk * NReg * 8);+ ftmp = _mm256_mul_ps(ftmp, vsca[i]);+ for (int im = 0; im < MTILE; im++) {+ if (i == 0) {+ va[im] = _mm256_set1_ps(*(Aptr + ikk + im * lda));+ }+ vacc[im * NReg + i] = _mm256_fmadd_ps(va[im], ftmp, vacc[im * NReg + i]);+ }+ }+ }+ }+}++template <int MTILE, int NReg, int Unroll>+static inline void accumulate_fp32_s8_fp32(const float* Aptr, int lda, int8_t* Bptr, __m256* vacc_loc) {+ if constexpr (MTILE == 1) {+ for (int ikk = 0; ikk < Unroll; ikk++) {+ __m256 va = _mm256_set1_ps(*(Aptr + ikk));+ for (int i = 0; i < NReg; i++) {+ auto ftmp = load_s8_fp32(Bptr + i * 8 + ikk * NReg * 8);+ vacc_loc[i] = _mm256_fmadd_ps(va, ftmp, vacc_loc[i]);+ }+ }+ } else {+ for (int ikk = 0; ikk < Unroll; ikk++) {+ __m256 va[MTILE];+ for (int i = 0; i < NReg; i++) {+ auto ftmp = load_s8_fp32(Bptr + i * 8 + ikk * NReg * 8);+ for (int im = 0; im < MTILE; im++) {+ if (i == 0) {+ va[im] = _mm256_set1_ps(*(Aptr + ikk + im * lda));+ }+ vacc_loc[im * NReg + i] = _mm256_fmadd_ps(va[im], ftmp, vacc_loc[im * NReg + i]);+ }+ }+ }+ }+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_4bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto& b4ptr = B.b4ptr;+ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ // Initialize accumulator with zeros+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(8);+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ __m256 v_b_scale[NReg];+ for (int i = 0; i < NReg; i++) {+ v_b_scale[i] = load_T_fp32(bsptr + i * 8);+ }++ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);++ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;++ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, bzptr, NTILE);+ }+ for (int i = 0; i < NReg; i++) {+ bzp[i] = _mm256_loadu_si256((const __m256i*)(tmp + i * 32));+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = kernel::avx2::unpack_4bits((void*)(b4ptr + i * 16 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ _mm256_storeu_si256((__m256i*)(tmp + 32 * i), vb);+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc, v_b_scale);+ }++ } else {+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = kernel::avx2::unpack_4bits((void*)(b4ptr + i * 16 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm256_sub_epi8(vb, vbias);+ _mm256_storeu_si256((__m256i*)(tmp + 32 * i), vb);+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc, v_b_scale);+ }+ }+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_2bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b2ptr = (utils::bit2x4*)B.b2ptr;++ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ // Initialize accumulator with zeros+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0_y = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ auto vbias = _mm256_set1_epi8(2);++ int constexpr KTILE = 1;+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;++ __m256 acc_loc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc_loc[i] = _mm256_setzero_ps();+ }+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);++ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, bzptr, NTILE);+ }+ for (int i = 0; i < NReg; i++) {+ bzp[i] = _mm256_loadu_si256((const __m256i*)(tmp + i * 32));+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ _mm256_storeu_si256((__m256i*)(tmp + 32 * i), vb);+ b2ptr += 8 * Unroll / 4;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }++ } else {+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ vb = _mm256_sub_epi8(vb, vbias);+ _mm256_storeu_si256((__m256i*)(tmp + 32 * i), vb);+ b2ptr += 8 * Unroll / 4;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }+ }++ __m256 v_b_scale[NReg];+ for (int i = 0; i < NReg; i++) {+ v_b_scale[i] = load_T_fp32(bsptr + i * 8);+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NReg; in++) {+ acc[im * NReg + in] = _mm256_fmadd_ps(acc_loc[im * NReg + in], v_b_scale[in], acc[im * NReg + in]);+ }+ }+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_1bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b1ptr = (utils::bit1x8*)B.b1ptr;++ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ int constexpr FullRange = 1 << (1 - 1);+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ auto vbias = _mm256_set1_epi8(FullRange);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ int constexpr KTILE = 1;+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;++ __m256 acc_loc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc_loc[i] = _mm256_setzero_ps();+ }+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);++ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, bzptr, NTILE);+ }+ for (int i = 0; i < NReg; i++) {+ bzp[i] = _mm256_loadu_si256((const __m256i*)(tmp + i * 32));+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, bzp[i]);+ _mm256_storeu_si256((__m256i*)(tmp + 32 * i), vb1);+ b1ptr += 8 * Unroll / 8;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }++ } else {+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, vbias);+ _mm256_storeu_si256((__m256i*)(tmp + 32 * i), vb1);+ b1ptr += 8 * Unroll / 8;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }+ }++ __m256 v_b_scale[NReg];+ for (int i = 0; i < NReg; i++) {+ v_b_scale[i] = load_T_fp32(bsptr + i * 8);+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NReg; in++) {+ acc[im * NReg + in] = _mm256_fmadd_ps(acc_loc[im * NReg + in], v_b_scale[in], acc[im * NReg + in]);+ }+ }+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_3bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b2ptr = (utils::bit2x4*)B.b2ptr;+ auto b1ptr = (utils::bit1x8*)B.b1ptr;++ int blks = k / blocksize;+ int constexpr FullRange = 1 << (3 - 1);+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ // Initialize accumulator with zeros+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0_y = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ auto vbias = _mm256_set1_epi8(FullRange);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ int constexpr KTILE = 1;+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;++ __m256 acc_loc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc_loc[i] = _mm256_setzero_ps();+ }+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);++ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, bzptr, NTILE);+ }+ for (int i = 0; i < NReg; i++) {+ bzp[i] = _mm256_loadu_si256((const __m256i*)(tmp + i * 32));+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ _mm256_storeu_si256((__m256i*)(tmp + 32 * i), vb);+ b2ptr += 8 * Unroll / 4;+ b1ptr += 8 * Unroll / 8;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }++ } else {+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ _mm256_storeu_si256((__m256i*)(tmp + 32 * i), vb);+ b2ptr += 8 * Unroll / 4;+ b1ptr += 8 * Unroll / 8;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }+ }++ __m256 v_b_scale[NReg];+ for (int i = 0; i < NReg; i++) {+ v_b_scale[i] = load_T_fp32(bsptr + i * 8);+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NReg; in++) {+ acc[im * NReg + in] = _mm256_fmadd_ps(acc_loc[im * NReg + in], v_b_scale[in], acc[im * NReg + in]);+ }+ }+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_5bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = (utils::bit4x2*)B.b4ptr;+ auto b1ptr = (utils::bit1x8*)B.b1ptr;++ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ // Initialize accumulator with zeros+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }++ int constexpr FullRange = 1 << (5 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ int constexpr KTILE = 1;+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;++ __m256 acc_loc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc_loc[i] = _mm256_setzero_ps();+ }+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);++ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, bzptr, NTILE);+ }+ for (int i = 0; i < NReg; i++) {+ bzp[i] = _mm256_loadu_si256((const __m256i*)(tmp + i * 32));+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ _mm256_storeu_si256((__m256i*)(tmp + 32 * i), vb);+ b4ptr += 8 * Unroll / 2;+ b1ptr += 8 * Unroll / 8;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }++ } else {+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ _mm256_storeu_si256((__m256i*)(tmp + 32 * i), vb);+ b4ptr += 8 * Unroll / 2;+ b1ptr += 8 * Unroll / 8;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }+ }++ __m256 v_b_scale[NReg];+ for (int i = 0; i < NReg; i++) {+ v_b_scale[i] = load_T_fp32(bsptr + i * 8);+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NReg; in++) {+ acc[im * NReg + in] = _mm256_fmadd_ps(acc_loc[im * NReg + in], v_b_scale[in], acc[im * NReg + in]);+ }+ }+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_6bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = (utils::bit4x2*)B.b4ptr;+ auto b2ptr = (utils::bit2x4*)B.b2ptr;++ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ // Initialize accumulator with zeros+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }++ int constexpr FullRange = 1 << (6 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ uint32_t mask0 = 0x03030303;+ auto vmask0 = _mm256_set1_epi32(*(int32_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ int constexpr KTILE = 1;+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;++ __m256 acc_loc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc_loc[i] = _mm256_setzero_ps();+ }+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);++ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, bzptr, NTILE);+ }+ for (int i = 0; i < NReg; i++) {+ bzp[i] = _mm256_loadu_si256((const __m256i*)(tmp + i * 32));+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ _mm256_storeu_si256((__m256i*)(tmp + 32 * i), vb);+ b4ptr += 8 * Unroll / 2;+ b2ptr += 8 * Unroll / 4;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }++ } else {+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ _mm256_storeu_si256((__m256i*)(tmp + 32 * i), vb);+ b4ptr += 8 * Unroll / 2;+ b2ptr += 8 * Unroll / 4;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }+ }++ __m256 v_b_scale[NReg];+ for (int i = 0; i < NReg; i++) {+ v_b_scale[i] = load_T_fp32(bsptr + i * 8);+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NReg; in++) {+ acc[im * NReg + in] = _mm256_fmadd_ps(acc_loc[im * NReg + in], v_b_scale[in], acc[im * NReg + in]);+ }+ }+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_7bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = (utils::bit4x2*)B.b4ptr;+ auto b2ptr = (utils::bit2x4*)B.b2ptr;+ auto b1ptr = (utils::bit1x8*)B.b1ptr;++ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ // Initialize accumulator with zeros+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }++ int constexpr FullRange = 1 << (7 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ uint32_t mask0 = 0x03030303;+ auto vmask0 = _mm256_set1_epi32(*(int32_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ int constexpr KTILE = 1;+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;++ __m256 acc_loc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc_loc[i] = _mm256_setzero_ps();+ }+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);++ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, bzptr, NTILE);+ }+ for (int i = 0; i < NReg; i++) {+ bzp[i] = _mm256_loadu_si256((const __m256i*)(tmp + i * 32));+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_or_si256(vb, vb2);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ _mm256_storeu_si256((__m256i*)(tmp + 32 * i), vb);+ b4ptr += 8 * Unroll / 2;+ b1ptr += 8 * Unroll / 8;+ b2ptr += 8 * Unroll / 4;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }++ } else {+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_or_si256(vb, vb2);+ vb = _mm256_sub_epi8(vb, vbias);+ _mm256_storeu_si256((__m256i*)(tmp + 32 * i), vb);+ b4ptr += 8 * Unroll / 2;+ b1ptr += 8 * Unroll / 8;+ b2ptr += 8 * Unroll / 4;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }+ }++ __m256 v_b_scale[NReg];+ for (int i = 0; i < NReg; i++) {+ v_b_scale[i] = load_T_fp32(bsptr + i * 8);+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NReg; in++) {+ acc[im * NReg + in] = _mm256_fmadd_ps(acc_loc[im * NReg + in], v_b_scale[in], acc[im * NReg + in]);+ }+ }+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++static inline __m256i _mm256_dpbusd_avx2_epi32(__m256i& c, const __m256i& a, const __m256i& b) {+ const __m256i dot2 = _mm256_maddubs_epi16(a, b);+ const __m256i ones = _mm256_set1_epi16(1);+ const __m256i sum4 = _mm256_madd_epi16(ones, dot2);+ return _mm256_add_epi32(c, sum4);+}++template <typename ScaleT, int NReg, int MTILE>+static inline void gemv_dequant_s32fp32(const float* asptr, int ldzp, const ScaleT* bsptr, __m256i* iacc,+ __m256* facc) {+ __m256 v_a_scale[MTILE];+ for (int im = 0; im < MTILE; im++) {+ v_a_scale[im] = _mm256_set1_ps(*(asptr + im * ldzp));+ }++ for (int i = 0; i < NReg; i++) {+ __m256 v_b_scale = load_T_fp32(bsptr + i * 8);+ for (int im = 0; im < MTILE; im++) {+ auto vtmp = _mm256_mul_ps(v_a_scale[im], v_b_scale);+ auto tmp = _mm256_cvtepi32_ps(iacc[im * NReg + i]);+ facc[im * NReg + i] = _mm256_fmadd_ps(tmp, vtmp, facc[im * NReg + i]);+ }+ }+}++template <int NReg, int MReg>+static inline void gemv_remove_zp(const uint8_t* azptr, int ldzp, __m256i* iacc, __m256i* bacc) {+ if constexpr (MReg == 1) {+ auto zp = int(azptr[0]);+ __m256i v_a_zp = _mm256_set1_epi32(zp);+ for (int in = 0; in < NReg; in++) {+ auto vtmp = _mm256_mullo_epi32(v_a_zp, bacc[in]);+ iacc[in] = _mm256_sub_epi32(iacc[in], vtmp);+ }+ } else {+ __m256i v_a_zp[MReg];+ for (int im = 0; im < MReg; im++) {+ auto zp = int(azptr[im * ldzp]);+ v_a_zp[im] = _mm256_set1_epi32(zp);+ for (int in = 0; in < NReg; in++) {+ auto vtmp = _mm256_mullo_epi32(v_a_zp[im], bacc[in]);+ iacc[im * NReg + in] = _mm256_sub_epi32(iacc[im * NReg + in], vtmp);+ }+ }+ }+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_4bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto& a8ptr = A.aptr;+ auto& b4ptr = B.b4ptr;+ auto& asptr = A.sptr;+ auto& azptr = A.zpptr;++ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ // Initialize accumulator with zeros+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ const __m256i onesu8 = _mm256_set1_epi8(1);+ const __m256i vbias = _mm256_set1_epi8(8);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ __m256i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += 4) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik));++ for (int i = 0; i < NReg; i++) {+ auto vb = kernel::avx2::unpack_4bits((void*)(b4ptr + i * 16 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx2_epi32(iacc[i], va, vb);+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = kernel::avx2::unpack_4bits((void*)(b4ptr + i * 16 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx2_epi32(iacc[j * NReg + i], va[j], vb);+ }+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += 4) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = kernel::avx2::unpack_4bits((void*)(b4ptr + i * 16 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx2_epi32(iacc[i], va, vb);+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = kernel::avx2::unpack_4bits((void*)(b4ptr + i * 16 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx2_epi32(iacc[j * NReg + i], va[j], vb);+ }+ }+ }+ }+ }++ gemv_remove_zp<NReg, MTILE>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_1bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr FullRange = 1 << (1 - 1);+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ const __m256i onesu8 = _mm256_set1_epi8(1);+ const __m256i vbias = _mm256_set1_epi8(FullRange);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ __m256i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb1);+ iacc[i] = _mm256_dpbusd_avx2_epi32(iacc[i], va, vb1);+ b1ptr += 8 * KTILE / 8;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb1);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx2_epi32(iacc[j * NReg + i], va[j], vb1);+ }+ b1ptr += 8 * KTILE / 8;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, vbias);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb1);+ iacc[i] = _mm256_dpbusd_avx2_epi32(iacc[i], va, vb1);+ b1ptr += 8 * KTILE / 8;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, vbias);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb1);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx2_epi32(iacc[j * NReg + i], va[j], vb1);+ }+ b1ptr += 8 * KTILE / 8;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_3bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr FullRange = 1 << (3 - 1);+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0_y = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ const __m256i onesu8 = _mm256_set1_epi8(1);+ const __m256i vbias = _mm256_set1_epi8(FullRange);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ __m256i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx2_epi32(iacc[i], va, vb);+ b2ptr += 8 * KTILE / 4;+ b1ptr += 8 * KTILE / 8;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx2_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b2ptr += 8 * KTILE / 4;+ b1ptr += 8 * KTILE / 8;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx2_epi32(iacc[i], va, vb);++ b2ptr += 8 * KTILE / 4;+ b1ptr += 8 * KTILE / 8;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx2_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b2ptr += 8 * KTILE / 4;+ b1ptr += 8 * KTILE / 8;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_5bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }++ int constexpr FullRange = 1 << (5 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ const __m256i onesu8 = _mm256_set1_epi8(1);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ __m256i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx2_epi32(iacc[i], va, vb);+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx2_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx2_epi32(iacc[i], va, vb);+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx2_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_6bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);++ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }++ int constexpr FullRange = 1 << (6 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ const __m256i onesu8 = _mm256_set1_epi8(1);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ uint32_t mask0 = 0x03030303;+ auto vmask0 = _mm256_set1_epi32(*(int32_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ __m256i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx2_epi32(iacc[i], va, vb);+ b4ptr += 8 * KTILE / 2;+ b2ptr += 8 * KTILE / 4;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx2_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += 8 * KTILE / 2;+ b2ptr += 8 * KTILE / 4;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx2_epi32(iacc[i], va, vb);+ b4ptr += 8 * KTILE / 2;+ b2ptr += 8 * KTILE / 4;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx2_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += 8 * KTILE / 2;+ b2ptr += 8 * KTILE / 4;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_7bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }++ int constexpr FullRange = 1 << (7 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ const __m256i onesu8 = _mm256_set1_epi8(1);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));++ uint32_t mask0 = 0x03030303;+ auto vmask0 = _mm256_set1_epi32(*(int32_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ __m256i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_or_si256(vb, vb2);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx2_epi32(iacc[i], va, vb);+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ b2ptr += 8 * KTILE / 4;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_or_si256(vb, vb2);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx2_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ b2ptr += 8 * KTILE / 4;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_or_si256(vb, vb2);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx2_epi32(iacc[i], va, vb);+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ b2ptr += 8 * KTILE / 4;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_or_si256(vb, vb2);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx2_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ b2ptr += 8 * KTILE / 4;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_2bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);++ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0_y = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ const __m256i onesu8 = _mm256_set1_epi8(1);+ const __m256i vbias = _mm256_set1_epi8(2);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ __m256i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx2_epi32(iacc[i], va, vb);+ b2ptr += 8 * KTILE / 4;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx2_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b2ptr += 8 * KTILE / 4;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx2_epi32(iacc[i], va, vb);+ b2ptr += 8 * KTILE / 4;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx2_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx2_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b2ptr += 8 * KTILE / 4;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename T>+static inline BTLA_CODE mul(const T* src0ptr, const T* src1ptr, T* dstptr, size_t size) {+ int constexpr VLen = 8;+ size_t velt = utils::padto_le(size, VLen);+ size_t i = 0;+ auto vfunc = [&]() {+ auto v0 = load_T_fp32(src0ptr + i);+ auto v1 = load_T_fp32(src1ptr + i);+ auto out = _mm256_mul_ps(v0, v1);+ store_fp32_T(out, dstptr + i);+ };+ for (; i < velt; i += VLen) vfunc();+ if (i < size) {+ if (size >= VLen) {+ i = size - VLen;+ vfunc();+ } else {+ ref::mul(src0ptr + i, src1ptr + i, dstptr + i, size - i);+ }+ }+ return BTLA_CODE::Success;+}++template <typename T>+static inline BTLA_CODE add(const T* src0ptr, const T* src1ptr, T* dstptr, size_t size) {+ int constexpr VLen = 8;+ size_t velt = utils::padto_le(size, VLen);+ size_t i = 0;+ auto vfunc = [&]() {+ auto v0 = load_T_fp32(src0ptr + i);+ auto v1 = load_T_fp32(src1ptr + i);+ auto out = _mm256_add_ps(v0, v1);+ store_fp32_T(out, dstptr + i);+ };+ for (; i < velt; i += VLen) vfunc();+ if (i < size) {+ if (size >= VLen) {+ i = size - VLen;+ vfunc();+ } else {+ ref::add(src0ptr + i, src1ptr + i, dstptr + i, size - i);+ }+ }+ return BTLA_CODE::Success;+}++template <bool HAS_ALIBI, bool HAS_TANH>+static inline BTLA_CODE scale_track_max_fp32_fp32(const float* src, const int src_step, float* dst, float* dst_max,+ int ld_dst, const int M_offset, const int N_offset, const int M,+ const int N, float scale, int causal_offset, float _alibi_slope,+ float tanh_scale, int padding_type, void* tmpcache,+ size_t cachesize) {+ static constexpr float seq15[16]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};+ alignas(32) const uint32_t mask8[9][8]{+ {0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff},+ };++ const auto v_scale = _mm256_set1_ps(scale);+ const auto v_seq7 = _mm256_loadu_ps(seq15);+ const auto alibi_slope = _mm256_set1_ps(_alibi_slope);+ const auto alibi_base = _mm256_mul_ps(alibi_slope, _mm256_add_ps(v_seq7, _mm256_set1_ps(N_offset)));+ const auto alibi_step = _mm256_set1_ps(_alibi_slope * 8);+ const auto infinity_neg = _mm256_set1_ps(-INFINITY);+ for (int i = 0; i < M; ++i) {+ auto alibi_curr = alibi_base;+ const auto N_unmasked = std::min(N, padding_type == 0 ? INT32_MAX+ : padding_type == 1 ? i + M_offset - N_offset + causal_offset + 1+ : causal_offset - N_offset);++ const auto v_mask = _mm256_load_si256(reinterpret_cast<const __m256i*>(mask8[N_unmasked % 8]));+ int j = 0;+ auto v_max = infinity_neg;+ for (; j < N_unmasked - 7; j += 8) {+ const auto xs = _mm256_fmadd_ps(v_scale, _mm256_loadu_ps(src + i * src_step + j), alibi_curr);+ v_max = _mm256_max_ps(v_max, xs);+ _mm256_storeu_ps(dst + i * ld_dst + j, xs);+ if constexpr (HAS_ALIBI) alibi_curr = _mm256_add_ps(alibi_curr, alibi_step);+ }+ if (j < N_unmasked) {+ const auto xs = _mm256_fmadd_ps(v_scale, _mm256_maskload_ps(src + i * src_step + j, v_mask), alibi_curr);+ const auto masked_xs = _mm256_blendv_ps(infinity_neg, xs, _mm256_castsi256_ps(v_mask));+ v_max = _mm256_max_ps(v_max, masked_xs);+ _mm256_storeu_ps(dst + i * ld_dst + j, xs);+ if constexpr (HAS_ALIBI) alibi_curr = _mm256_add_ps(alibi_curr, alibi_step);+ j += 8;+ }+ alignas(32) float dst_tmp[8];+ _mm256_store_ps(dst_tmp, v_max);+ for (int ii = 0; ii < 8; ++ii) dst_max[i] = std::max(dst_max[i], dst_tmp[ii]);+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE weight_cvt_fp16_fp32_n24(const utils::fp16* B, int ldb, bool is_padded, float* dst_ptr,+ int dst_step, int k_size, int n_size, int k_offset, int n_offset,+ void* tmpcache, size_t cachesize) {+ assert(is_padded);+ const auto src = B + k_offset * 24 + n_offset * ldb;+ assert(n_size <= 24);+ assert(n_offset % 24 == 0);+ if (n_size == 24) {+ constexpr auto n_size = 24;+ for (int i = 0; i < k_size; ++i) {+ for (int j = 0; j < n_size; j += 8) {+ const auto cur_src = src + i * 24 + j;+ const auto cur_dst = dst_ptr + i * 24 + j;+ const auto src = load_T_fp32(cur_src);+ _mm256_store_ps(cur_dst, src);+ }+ }+ } else {+ for (int i = 0; i < k_size; ++i) {+ for (int j = 0; j < n_size; j += 8) {+ const auto cur_src = src + i * 24 + j;+ const auto cur_dst = dst_ptr + i * 24 + j;+ const auto src = load_T_fp32(cur_src);+ _mm256_store_ps(cur_dst, src);+ }+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE inplace_precompute_max_softmax_fp32_fp32(int m_size, int n_size, int n_pad_size, bool is_causal,+ float* src, float* dst, const float* s_max,+ float* expsum, int ld_src, int ld_dst) {+ alignas(32) const uint32_t mask8[9][8]{+ {0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff},+ };++ for (int ii = 0; ii < m_size; ++ii) {+ const auto i_src = src + ii * ld_src;+ const auto i_dst = dst + ii * ld_dst;+ const auto curr_n_size = n_size + (is_causal ? ii : 0);+ const auto v_mask = _mm256_load_si256(reinterpret_cast<const __m256i*>(mask8[curr_n_size % 8]));+ { // subtract max+ const auto row_max = _mm256_set1_ps(s_max[ii]);+ for (int jj = 0; jj < curr_n_size; jj += 8) { // should be fine to do extra work on idx >= curr_n_size+ _mm256_storeu_ps(i_src + jj, _mm256_sub_ps(_mm256_loadu_ps(i_src + jj), row_max));+ }+ }+ auto v_sum = _mm256_setzero_ps();+ { // exp & sum+ int jj = 0;+ for (; jj < utils::padto_le(curr_n_size, 8); jj += 8) {+ const auto v_exp = kernel::avx2::exp_ps_0_1(_mm256_loadu_ps(i_src + jj));+ v_sum = _mm256_add_ps(v_sum, v_exp);+ _mm256_storeu_ps(i_src + jj, v_exp);+ }+ if (jj < curr_n_size) {+ const auto v_exp = kernel::avx2::exp_ps_0_1(_mm256_loadu_ps(i_src + jj)); // should be fine to load extra+ v_sum = _mm256_add_ps(v_sum, _mm256_and_ps(v_exp, _mm256_castsi256_ps(v_mask)));+ _mm256_storeu_ps(i_src + jj, v_exp); // should be fine to store some extra+ }++ alignas(32) float sum_tmp[8];+ _mm256_store_ps(sum_tmp, v_sum);+ expsum[ii] = 0.f;+ for (int iii = 0; iii < 8; ++iii) expsum[ii] += sum_tmp[iii];+ v_sum = _mm256_set1_ps(expsum[ii]);+ }+ { // scale & store+ int jj = 0;+ for (; jj < utils::padto_le(curr_n_size, 8); jj += 8) {+ _mm256_store_ps(i_dst + jj, _mm256_div_ps(_mm256_loadu_ps(i_src + jj), v_sum));+ }+ if (jj < curr_n_size) {+ const auto quotient = _mm256_div_ps(_mm256_loadu_ps(i_src + jj), v_sum);+ _mm256_store_ps(i_dst + jj, _mm256_and_ps(quotient, _mm256_castsi256_ps(v_mask)));+ jj += 8;+ }+ if (jj < n_pad_size) memset(i_dst + jj, 0, sizeof(float) * (n_pad_size - jj));+ }+ }++ return BTLA_CODE::Success;+}+#if defined(__INTEL_LLVM_COMPILER)+#pragma clang attribute pop+#elif defined(__GNUC__)+#pragma GCC pop_options+#endif+#endif+} // namespace avx2+} // namespace kernel+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx512_bf16.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx512_bf16.hnew file mode 100644index 000000000..607f8800a--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx512_bf16.h@@ -0,0 +1,188 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include "kernel_avx512f.h"++namespace bestla {+namespace kernel {+namespace avx512f {+namespace avx512_bf16 {+#if CompileBF16()+#if defined(__INTEL_LLVM_COMPILER)+#pragma clang attribute push(__attribute__((target("avx512f,avx512bf16,avx512vl,avx512bw"))), apply_to = function)+#elif defined(__GNUC__)+#pragma GCC push_options+#pragma GCC target("avx512bf16", "avx512vl", "avx512bw")+#endif++static inline __m256i zmm_cvt_fp32_bf16(__m512 vfp32) { return (__m256i)_mm512_cvtneps_pbh(vfp32); }++static inline __m512 load_bf16_fp32(const utils::bf16* srcptr) {+ auto tmp = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(srcptr));+ auto vf32 = zmm_cvt_bf16_fp32(tmp);+ return vf32;+}++static inline BTLA_CODE bf16_cvt_fp32_2D_write_back(const utils::bf16* src_ptr, float* dst_ptr, int row, int col,+ int src_step, int dst_step, bool zeropadding) {+ const int npadding = (dst_step - col) * sizeof(float);+ constexpr int simd_proc_elt = 16;+ auto col_body = col / simd_proc_elt * simd_proc_elt;+ auto col_tail = col % simd_proc_elt;+ const auto tail_mask = _cvtu32_mask16((1U << col_tail) - 1);+ for (int i = 0; i < row; i++) {+ auto src = const_cast<utils::bf16*>(src_ptr + i * src_step);+ auto dst = dst_ptr + i * dst_step;+ int j = 0;+ for (; j < col_body; j += simd_proc_elt) _mm512_storeu_ps(dst + j, load_bf16_fp32(src + j));+ if (col_tail > 0) {+ __m256i tmp = _mm256_setzero_si256();+ tmp = _mm256_mask_loadu_epi16(tmp, tail_mask, src + j);+ _mm512_mask_storeu_ps(dst + j, tail_mask, zmm_cvt_bf16_fp32(tmp));+ }+ if (zeropadding && npadding) std::memset(dst + col, 0, npadding);+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE fp32_cvt_bf16_2D_write_back(const void* raw_srcptr, void* raw_dstptr, int row, int col,+ int srcstride, int dststride, bool zeropadding) {+ auto srcptr = reinterpret_cast<const char*>(raw_srcptr);+ auto dstptr = reinterpret_cast<char*>(raw_dstptr);+ constexpr int simd_proc_elt = 32;+ auto col_body_loop = col / simd_proc_elt;+ auto col_tail = col % simd_proc_elt;+ const uint32_t tail_mask = (1U << col_tail) - 1;+ int npadding = dststride - col * sizeof(utils::bf16);+ for (int i = 0; i < row; i++) {+ auto src = srcptr + i * srcstride;+ auto dst = dstptr + i * dststride;+ int j = 0;+ for (; j < col_body_loop; j++) {+ _mm512_storeu_epi16(+ (dst + (j * simd_proc_elt) * sizeof(utils::bf16)),+ (__m512i)_mm512_cvtne2ps_pbh(_mm512_loadu_ps(src + sizeof(float) * simd_proc_elt * j + sizeof(float) * 16),+ _mm512_loadu_ps(src + sizeof(float) * simd_proc_elt * j + sizeof(float) * 0)));+ }+ if (col_tail > 0) {+ _mm512_mask_storeu_epi16(+ (dst + (j * simd_proc_elt) * sizeof(utils::bf16)), tail_mask, //+ (__m512i)_mm512_cvtne2ps_pbh(+ _mm512_maskz_loadu_ps(tail_mask >> 16, src + sizeof(float) * simd_proc_elt * j + sizeof(float) * 16),+ _mm512_maskz_loadu_ps(tail_mask >> 0, src + sizeof(float) * simd_proc_elt * j + sizeof(float) * 0)));+ }+ if (zeropadding && npadding) {+ std::memset(dst + col * sizeof(utils::bf16), 0, npadding);+ }+ }+ return BTLA_CODE::Success;+}++template <typename T_DST>+static inline BTLA_CODE scale_exp_acc_sum_fp32(const float* src, const int src_step, T_DST* dst, int ld_dst,+ float* dst_sum, const int M_offset, const int N_offset, const int M,+ const int N, float scale, int causal_offset, void* /* tmpcache */,+ size_t /* cachesize */) {+ const auto v_scale = _mm512_set1_ps(scale);+ for (int i = 0; i < M; ++i) {+ const auto N_unmasked = std::min(N, causal_offset < 0 ? INT32_MAX : i + M_offset - N_offset + causal_offset + 1);++ const auto v_mask = _cvtu32_mask16((1U << (N_unmasked % 16)) - 1);+ int j = 0;+ auto v_sum = _mm512_setzero_ps();+ for (; j < N_unmasked - 15; j += 16) {+ const auto v_exp = kernel::avx512f::exp_ps_0_1(_mm512_mul_ps(v_scale, _mm512_loadu_ps(src + i * src_step + j)));+ v_sum = _mm512_add_ps(v_sum, v_exp);+ _mm256_storeu_epi16(dst + i * ld_dst + j, (__m256i)_mm512_cvtneps_pbh(v_exp));+ }+ if (j < N_unmasked) {+ const auto v_exp =+ kernel::avx512f::exp_ps_0_1(_mm512_mul_ps(v_scale, _mm512_maskz_loadu_ps(v_mask, src + i * src_step + j)));+ v_sum = _mm512_mask_add_ps(v_sum, v_mask, v_sum, v_exp);+ _mm256_storeu_epi16(dst + i * ld_dst + j, (__m256i)_mm512_maskz_cvtneps_pbh(v_mask, v_exp));+ j += 16;+ }+ dst_sum[i] += _mm512_reduce_add_ps(v_sum);++ if (j < utils::padto(N, 64)) std::memset(dst + i * ld_dst + j, 0, sizeof(*dst) * (utils::padto(N, 64) - j));+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE inplace_precompute_max_softmax_fp32_bf16(int m_size, int n_size, int n_pad_size, bool is_causal,+ float* src, utils::bf16* dst, const float* s_max,+ float* expsum, int ld_src, int ld_dst) {+ for (int ii = 0; ii < m_size; ++ii) {+ const auto i_src = src + ii * ld_src;+ const auto i_dst = dst + ii * ld_dst;+ const auto curr_n_size = n_size + (is_causal ? ii : 0);+ const auto v_mask = _cvtu32_mask16((1U << (curr_n_size % 16)) - 1);+ const auto v_mask32 = _cvtu32_mask32((1U << (curr_n_size % 32)) - 1);+ { // subtract max+ const auto row_max = _mm512_set1_ps(s_max[ii]);+ for (int jj = 0; jj < curr_n_size; jj += 16) { // should be fine to do extra work on idx >= curr_n_size+ _mm512_storeu_ps(i_src + jj, _mm512_sub_ps(_mm512_loadu_ps(i_src + jj), row_max));+ }+ }+ auto v_sum = _mm512_setzero_ps();+ { // exp & sum+ int jj = 0;+ for (; jj < curr_n_size / 16 * 16; jj += 16) {+ const auto v_exp = kernel::avx512f::exp_ps_0_1(_mm512_loadu_ps(i_src + jj));+ v_sum = _mm512_add_ps(v_sum, v_exp);+ _mm512_storeu_ps(i_src + jj, v_exp);+ }+ if (jj < curr_n_size) {+ const auto v_exp =+ kernel::avx512f::exp_ps_0_1(_mm512_loadu_ps(i_src + jj)); // should be fine to load some extra+ v_sum = _mm512_mask_add_ps(v_sum, v_mask, v_sum, v_exp);+ _mm512_storeu_ps(i_src + jj, v_exp); // should be fine to store some extra+ }+ expsum[ii] = _mm512_reduce_add_ps(v_sum);+ v_sum = _mm512_set1_ps(expsum[ii]);+ }+ { // scale & bf16+ int jj = 0;+ for (; jj < curr_n_size / 32 * 32; jj += 32) {+ const auto v_softmax0 = _mm512_div_ps(_mm512_loadu_ps(i_src + jj), v_sum);+ const auto v_softmax1 = _mm512_div_ps(_mm512_loadu_ps(i_src + jj + 16), v_sum);+ _mm512_storeu_epi16(i_dst + jj, (__m512i)_mm512_cvtne2ps_pbh(v_softmax1, v_softmax0));+ }+ if (jj < curr_n_size) {+ const auto v_softmax0 = _mm512_div_ps(_mm512_loadu_ps(i_src + jj), v_sum);+ const auto v_softmax1 = _mm512_div_ps(_mm512_loadu_ps(i_src + jj + 16), v_sum);+#if defined(__GNUC__) && (__GNUC__ == 13) && (__GNUC_MINOR__ <= 2)+ // There is a bug on gcc 13.1/13.2 what reverse the parameter order;+ // A GUN team member said that it will befixed in GCC 13.3+ _mm512_storeu_epi16(i_dst + jj, (__m512i)_mm512_maskz_cvtne2ps_pbh(v_mask32, v_softmax0, v_softmax1));+#else+ _mm512_storeu_epi16(i_dst + jj, (__m512i)_mm512_maskz_cvtne2ps_pbh(v_mask32, v_softmax1, v_softmax0));+#endif+ jj += 32;+ }+ if (jj < n_pad_size) memset(i_dst + jj, 0, sizeof(utils::bf16) * (n_pad_size - jj));+ }+ }+ return BTLA_CODE::Success;+}+#if defined(__INTEL_LLVM_COMPILER)+#pragma clang attribute pop+#elif defined(__GNUC__)+#pragma GCC pop_options+#endif+#endif+} // namespace avx512_bf16+} // namespace avx512f+} // namespace kernel+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx512_fp16.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx512_fp16.hnew file mode 100644index 000000000..124ca0076--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx512_fp16.h@@ -0,0 +1,477 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include "kernel_avx512f.h"+#include "kernel_avx512_bf16.h"++namespace bestla {+namespace kernel {+namespace avx512f {+namespace avx512_fp16 {+#if CompileFP16()+#if defined(__INTEL_LLVM_COMPILER)+#pragma clang attribute push(__attribute__((target("avx512f,avx512bf16,avx512bw,avx512fp16"))), apply_to = function)+#elif defined(__GNUC__)+#pragma GCC push_options+#pragma GCC target("avx512f", "avx512bf16", "avx512vl", "avx512bw", "avx512fp16")+#endif++inline __m512 zmm_cvt_fp16_fp32(__m256i vfp16) { return _mm512_cvtxph_ps((__m256h)vfp16); }++inline __m256i zmm_cvt_fp32_fp16(__m512 vfp32) { return (__m256i)_mm512_cvtxps_ph(vfp32); }++template <typename T>+static inline void store_fp32_T(__m512 src_y, T* dstptr) {+ if constexpr (std::is_same_v<T, utils::bf16>) {+ auto ymm = avx512_bf16::zmm_cvt_fp32_bf16(src_y);+ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstptr), ymm);+ } else if constexpr (std::is_same_v<T, float>) {+ _mm512_storeu_ps(dstptr, src_y);+ } else if constexpr (std::is_same_v<T, utils::fp16>) {+ auto ymm = zmm_cvt_fp32_fp16(src_y);+ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstptr), ymm);+ } else {+ assert(false);+ }+}++static inline BTLA_CODE fp32_cvt_fp16_2D_write_back(const float* src_ptr, utils::fp16* dst_ptr, int row, int col,+ int src_step, int dst_step, bool zeropadding) {+ const int npadding = (dst_step - col) * sizeof(utils::fp16);+ constexpr int simd_proc_elt = 16;+ auto col_body = col / simd_proc_elt * simd_proc_elt;+ auto col_tail = col % simd_proc_elt;+ const auto tail_mask = _cvtu32_mask16((1U << col_tail) - 1);+ for (int i = 0; i < row; i++) {+ const auto src = src_ptr + i * src_step;+ const auto dst = dst_ptr + i * dst_step;+ int j = 0;+ for (; j < col_body; j += simd_proc_elt) {+ store_fp32_T(_mm512_loadu_ps(src + j), dst + j);+ }+ if (col_tail > 0) {+ auto vf32 = _mm512_maskz_loadu_ps(tail_mask, src + j);+ auto vf16 = zmm_cvt_fp32_fp16(vf32);+ _mm256_mask_storeu_epi16(dst + j, tail_mask, vf16);+ }+ if (zeropadding && npadding) std::memset(dst + col, 0, npadding);+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE fp16_cvt_fp32_2D_write_back(const utils::fp16* src_ptr, float* dst_ptr, int row, int col,+ int src_step, int dst_step, bool zeropadding) {+ const int npadding = (dst_step - col) * sizeof(float);+ constexpr int simd_proc_elt = 16;+ auto col_body = col / simd_proc_elt * simd_proc_elt;+ auto col_tail = col % simd_proc_elt;+ const auto tail_mask = _cvtu32_mask16((1U << col_tail) - 1);+ for (int i = 0; i < row; i++) {+ const auto src = src_ptr + i * src_step;+ const auto dst = dst_ptr + i * dst_step;+ int j = 0;+ for (; j < col_body; j += simd_proc_elt) {+ auto vf32 = load_T_fp32(src + j);+ _mm512_storeu_ps(dst + j, vf32);+ }+ if (col_tail > 0) {+ auto vf16 = _mm256_maskz_loadu_epi16(tail_mask, src + j);+ auto v32 = zmm_cvt_fp16_fp32(vf16);+ _mm512_mask_storeu_ps(dst + j, tail_mask, v32);+ }+ if (zeropadding && npadding) std::memset(dst + col, 0, npadding);+ }+ return BTLA_CODE::Success;+}+#ifdef __GNUC__+#pragma GCC diagnostic push+#pragma GCC diagnostic ignored "-Wignored-attributes" // https://stackoverflow.com/a/49216021+#endif+// Load 2 fp16 vectors; convert them to bf16 and interleave them+template <int tail>+static inline std::array<__m512i, 2> load_fp16_bf16_interleave_word(const utils::fp16* a, size_t lda) {+ static_assert(tail > 0 && tail <= 2, "Unexpected tail value.");+ std::array<__m512i, 2> dst;+ for (int i = 0; i < tail; ++i) {+ dst[i] = (__m512i)(_mm512_cvtne2ps_pbh(load_T_fp32(a + i * lda + 16), load_T_fp32(a + i * lda + 0)));+ }+ for (int i = tail; i < 2; ++i) dst[i] = _mm512_setzero_epi32();+ interleave_word(dst);+ return dst;+}++// load_fp16_bf16_interleave_word with maskz+template <int tail>+static inline std::array<__m512i, 2> load_maskz_fp16_bf16_interleave_word(const utils::fp16* a, size_t lda,+ uint32_t mask) {+ static_assert(tail > 0 && tail <= 2, "Unexpected tail value.");++ const auto mask_lo = mask;+ const auto mask_hi = mask >> 16;+ std::array<__m512i, 2> dst;+ for (int i = 0; i < tail; ++i) {+ dst[i] = (__m512i)(_mm512_cvtne2ps_pbh( //+ zmm_cvt_fp16_fp32(_mm256_maskz_loadu_epi16(mask_hi, a + i * lda + 16)), //+ zmm_cvt_fp16_fp32(_mm256_maskz_loadu_epi16(mask_lo, a + i * lda + 0))));+ }+ for (int i = tail; i < 2; ++i) dst[i] = _mm512_setzero_epi32();+ interleave_word(dst);+ return dst;+}++template <int tail>+static inline std::array<__m512i, 16> load_fp16_bf16_tr_x16_dword(const utils::fp16* a, size_t lda) {+ static_assert(tail > 0 && tail <= 16, "Unexpected tail value.");+ std::array<__m512i, 16> dst;+ for (int i = 0; i < tail; ++i) {+ dst[i] = (__m512i)(_mm512_cvtne2ps_pbh( //+ zmm_cvt_fp16_fp32(_mm256_loadu_epi16(a + i * lda + 16)), //+ zmm_cvt_fp16_fp32(_mm256_loadu_epi16(a + i * lda + 0))));+ }+ for (int i = tail; i < 16; ++i) dst[i] = _mm512_setzero_epi32();+ tr_x16_dword(dst);+ return dst;+}+static constexpr decltype(load_fp16_bf16_tr_x16_dword<1>)* load_fp16_bf16_tr_x16_dword_tbl[17]{+ load_fp16_bf16_tr_x16_dword<1>, load_fp16_bf16_tr_x16_dword<1>, load_fp16_bf16_tr_x16_dword<2>,+ load_fp16_bf16_tr_x16_dword<3>, load_fp16_bf16_tr_x16_dword<4>, load_fp16_bf16_tr_x16_dword<5>,+ load_fp16_bf16_tr_x16_dword<6>, load_fp16_bf16_tr_x16_dword<7>, load_fp16_bf16_tr_x16_dword<8>,+ load_fp16_bf16_tr_x16_dword<9>, load_fp16_bf16_tr_x16_dword<10>, load_fp16_bf16_tr_x16_dword<11>,+ load_fp16_bf16_tr_x16_dword<12>, load_fp16_bf16_tr_x16_dword<13>, load_fp16_bf16_tr_x16_dword<14>,+ load_fp16_bf16_tr_x16_dword<15>, load_fp16_bf16_tr_x16_dword<16>,+};++template <int tail>+static inline std::array<__m512i, 16> load_maskz_fp16_bf16_tr_x16_dword(const utils::fp16* a, size_t lda,+ uint32_t mask) {+ static_assert(tail > 0 && tail <= 16, "Unexpected tail value.");+ std::array<__m512i, 16> dst;++ const auto mask_lo = mask;+ const auto mask_hi = mask >> 16;+ for (int i = 0; i < tail; ++i) {+ dst[i] = (__m512i)(_mm512_cvtne2ps_pbh( //+ zmm_cvt_fp16_fp32(_mm256_maskz_loadu_epi16(mask_hi, a + i * lda + 16)), //+ zmm_cvt_fp16_fp32(_mm256_maskz_loadu_epi16(mask_lo, a + i * lda + 0))));+ }+ for (int i = tail; i < 16; ++i) dst[i] = _mm512_setzero_epi32();+ tr_x16_dword(dst);+ return dst;+}+#ifdef __GNUC__+#pragma GCC diagnostic pop+#endif++static constexpr decltype(load_maskz_fp16_bf16_tr_x16_dword<1>)* load_maskz_fp16_bf16_tr_x16_dword_tbl[17]{+ load_maskz_fp16_bf16_tr_x16_dword<1>, load_maskz_fp16_bf16_tr_x16_dword<1>, load_maskz_fp16_bf16_tr_x16_dword<2>,+ load_maskz_fp16_bf16_tr_x16_dword<3>, load_maskz_fp16_bf16_tr_x16_dword<4>, load_maskz_fp16_bf16_tr_x16_dword<5>,+ load_maskz_fp16_bf16_tr_x16_dword<6>, load_maskz_fp16_bf16_tr_x16_dword<7>, load_maskz_fp16_bf16_tr_x16_dword<8>,+ load_maskz_fp16_bf16_tr_x16_dword<9>, load_maskz_fp16_bf16_tr_x16_dword<10>, load_maskz_fp16_bf16_tr_x16_dword<11>,+ load_maskz_fp16_bf16_tr_x16_dword<12>, load_maskz_fp16_bf16_tr_x16_dword<13>, load_maskz_fp16_bf16_tr_x16_dword<14>,+ load_maskz_fp16_bf16_tr_x16_dword<15>, load_maskz_fp16_bf16_tr_x16_dword<16>,+};++template <typename T_SRC, typename T_DST = T_SRC, int RowPack = 4 / sizeof(T_DST)>+struct padding_interleave_cvt {+ padding_interleave_cvt() = delete;+ static BTLA_CODE forward(const T_SRC* src, T_DST* dst, int NTile, int row, int col, int row_pad, int col_pad,+ int src_step, int dst_step) {+ return BTLA_CODE::NotSupport;+ }+};++template <>+struct padding_interleave_cvt<utils::fp16, utils::bf16, 2> {+ static constexpr int RowPack = 2;+ padding_interleave_cvt() = delete;++ // M x N ===> N/NTile x M/RowPack x NTile x RowPack (leading dim stride = NTile * dststride)+ static BTLA_CODE forward(const utils::fp16* src, utils::bf16* dst, int NTile, int row, int col, int row_pad,+ int col_pad, int src_step, int dst_step) {+ int i = 0;+ for (; i < row / RowPack * RowPack; i += RowPack) {+ int j = 0;+ for (; j < col / NTile * NTile; j += NTile) {+ assert(NTile % 32 == 0);+ for (int jj = 0; jj < NTile; jj += 32) {+ const auto xss = load_fp16_bf16_interleave_word<2>(src + i * src_step + j + jj, src_step);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 0) * RowPack, xss[0]);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 16) * RowPack, xss[1]);+ }+ }+ if (j < col) { // j: tail processing+ int jj = 0;+ for (; j + jj < col / 32 * 32; jj += 32) {+ const auto xss = load_fp16_bf16_interleave_word<2>(src + i * src_step + j + jj, src_step);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 0) * RowPack, xss[0]);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 16) * RowPack, xss[1]);+ }+ if (j + jj < col) { // jj: tail processing+ const uint32_t mask = (1U << (col - j - jj)) - 1;+ const auto xss = load_maskz_fp16_bf16_interleave_word<2>(src + i * src_step + j + jj, src_step, mask);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 0) * RowPack, xss[0]);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 16) * RowPack, xss[1]);+ jj += 32;+ }+ for (; jj < NTile; jj += 32) { // jj: padding zero+ memset(dst + i * NTile + j * dst_step + jj * RowPack, 0, sizeof(utils::bf16) * 32 * RowPack);+ }+ j += NTile;+ }+ for (; j < col_pad; j += NTile) { // j: padding zero+ memset(dst + i * NTile + j * dst_step, 0, sizeof(utils::bf16) * NTile * RowPack);+ }+ }+ if (i < row) { // i: tail processing+ static constexpr int tail_m = 1; // must be 1+ int j = 0;+ for (; j < col / NTile * NTile; j += NTile) {+ assert(NTile % 32 == 0);+ for (int jj = 0; jj < NTile; jj += 32) {+ const auto xss = load_fp16_bf16_interleave_word<tail_m>(src + i * src_step + j + jj, src_step);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 0) * RowPack, xss[0]);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 16) * RowPack, xss[1]);+ }+ }+ if (j < col) { // j: tail processing+ int jj = 0;+ for (; j + jj < col / 32 * 32; jj += 32) {+ const auto xss = load_fp16_bf16_interleave_word<tail_m>(src + i * src_step + j + jj, src_step);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 0) * RowPack, xss[0]);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 16) * RowPack, xss[1]);+ }+ if (j + jj < col) { // jj: tail processing+ const uint32_t mask = (1U << (col - j - jj)) - 1;+ const auto xss = load_maskz_fp16_bf16_interleave_word<tail_m>(src + i * src_step + j + jj, src_step, mask);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 0) * RowPack, xss[0]);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 16) * RowPack, xss[1]);+ jj += 32;+ }+ for (; jj < NTile; jj += 32) { // jj: padding zero+ memset(dst + i * NTile + j * dst_step + jj * RowPack, 0, sizeof(utils::bf16) * 32 * RowPack);+ }+ j += NTile;+ }+ for (; j < col_pad; j += NTile) { // j: padding zero+ memset(dst + i * NTile + j * dst_step, 0, sizeof(utils::bf16) * NTile * RowPack);+ }+ i += RowPack;+ }+ for (; i < row_pad; i += RowPack) { // i: padding zero+ for (int j = 0; j < col_pad; j += NTile) {+ memset(dst + i * NTile + j * dst_step, 0, sizeof(utils::bf16) * NTile * RowPack);+ }+ }+ return BTLA_CODE::Success;+ }+};++template <typename T_SRC, typename T_DST = T_SRC, int ColPack = 4 / sizeof(T_DST)>+struct padding_trans_interleave_cvt {+ padding_trans_interleave_cvt() = delete;+ static BTLA_CODE forward(const T_SRC* src, T_DST* dst, int MTile, int row, int col, int row_pad, int col_pad,+ int src_step, int dst_step) {+ return BTLA_CODE::NotSupport;+ }+};++template <>+struct padding_trans_interleave_cvt<utils::fp16, utils::bf16, 2> {+ static constexpr int ColPack = 2;+ padding_trans_interleave_cvt() = delete;++ static BTLA_CODE forward(const utils::fp16* src, utils::bf16* dst, int MTile, int row, int col, int row_pad,+ int col_pad, int src_step, int dst_step) {+ assert(row_pad % 16 == 0 && col_pad % 32 == 0);+ int i = 0;+ for (; i < row / MTile * MTile; i += MTile) {+ assert(MTile % 16 == 0);+ int j = 0;+ for (; j < col / 32 * 32; j += 32) {+ for (int ii = 0; ii < MTile; ii += 16) {+ assert(MTile % 16 == 0);+ const auto xss = load_fp16_bf16_tr_x16_dword<16>(src + (i + ii) * src_step + j, src_step);+ for (int jj = 0; jj < 32; jj += 2) {+ _mm512_storeu_si512(dst + i * dst_step + ii * ColPack + (j + jj) * MTile, xss[jj / 2]);+ }+ }+ }+ if (j < col) { // j: tail processing+ for (int ii = 0; ii < MTile; ii += 16) {+ assert(MTile % 16 == 0);+ const uint32_t mask = (1U << (col - j)) - 1;+ const auto xss = load_maskz_fp16_bf16_tr_x16_dword<16>(src + (i + ii) * src_step + j, src_step, mask);+ for (int jj = 0; jj < 32; jj += 2) {+ _mm512_storeu_si512(dst + i * dst_step + ii * ColPack + (j + jj) * MTile, xss[jj / 2]);+ }+ }+ j += 32;+ }+ for (; j < col_pad; j += 2) { // j: padding zero+ memset(dst + i * dst_step + j * MTile, 0, 2 * sizeof(utils::bf16) * MTile);+ }+ }+ if (i < row) { // i: tail processing+ int ii = 0;+ for (; i + ii < row / 16 * 16; ii += 16) {+ int j = 0;+ for (; j < col / 32 * 32; j += 32) {+ assert(MTile % 16 == 0);+ const auto xss = load_fp16_bf16_tr_x16_dword<16>(src + (i + ii) * src_step + j, src_step);+ for (int jj = 0; jj < 32; jj += 2) {+ _mm512_storeu_si512(dst + i * dst_step + ii * ColPack + (j + jj) * MTile, xss[jj / 2]);+ }+ }+ if (j < col) { // j: tail processing+ assert(MTile % 16 == 0);+ const uint32_t mask = (1U << (col - j)) - 1;+ const auto xss = load_maskz_fp16_bf16_tr_x16_dword<16>(src + (i + ii) * src_step + j, src_step, mask);+ for (int jj = 0; jj < 32; jj += 2) {+ _mm512_storeu_si512(dst + i * dst_step + ii * ColPack + (j + jj) * MTile, xss[jj / 2]);+ }+ j += 32;+ }+ for (; j < col_pad; j += 2) { // j: padding zero+ memset(dst + i * dst_step + ii * ColPack + j * MTile, 0, 2 * sizeof(utils::bf16) * 16);+ }+ }+ if (i + ii < row) { // ii: tail processing+ const int tbl_idx = row - i - ii;+ int j = 0;+ for (; j < col / 32 * 32; j += 32) {+ assert(MTile % 16 == 0);+ const auto xss = load_fp16_bf16_tr_x16_dword_tbl[tbl_idx](src + (i + ii) * src_step + j, src_step);+ for (int jj = 0; jj < 32; jj += 2) {+ _mm512_storeu_si512(dst + i * dst_step + ii * ColPack + (j + jj) * MTile, xss[jj / 2]);+ }+ }+ if (j < col) { // j: tail processing+ assert(MTile % 16 == 0);+ const uint32_t mask = (1U << (col - j)) - 1;+ const auto xss =+ load_maskz_fp16_bf16_tr_x16_dword_tbl[tbl_idx](src + (i + ii) * src_step + j, src_step, mask);+ for (int jj = 0; jj < 32; jj += 2) {+ _mm512_storeu_si512(dst + i * dst_step + ii * ColPack + (j + jj) * MTile, xss[jj / 2]);+ }+ j += 32;+ }+ for (; j < col_pad; j += 2) { // j: padding zero+ memset(dst + i * dst_step + ii * ColPack + j * MTile, 0, 2 * sizeof(utils::bf16) * 16);+ }+ ii += 16;+ }+ for (; ii < MTile; ii += 16) { // ii: padding zero+ for (int j = 0; j < col_pad; j += 2) {+ memset(dst + i * dst_step + ii * ColPack + j * MTile, 0, 2 * sizeof(utils::bf16) * 16);+ }+ }+ assert(ii == MTile);+ i += MTile;+ }+ assert(row_pad % MTile == 0);+ for (; i < row_pad; i += MTile) { // i: padding zero+ for (int j = 0; j < col_pad; j += 2) {+ memset(dst + i * dst_step + j * MTile, 0, 2 * sizeof(utils::bf16) * MTile);+ }+ }+ return BTLA_CODE::Success;+ }+};++static inline BTLA_CODE scale_track_max_fp16_fp32(const utils::fp16* src, const int src_step, float* dst,+ float* dst_max, int ld_dst, const int M_offset, const int N_offset,+ const int M, const int N, float scale, int causal_offset,+ float alibi_slope, float tanh_scale, void* tmpcache,+ size_t cachesize) {+ const auto v_scale = _mm512_set1_ps(scale);+ for (int i = 0; i < M; ++i) {+ const auto N_unmasked = std::min(N, causal_offset < 0 ? INT32_MAX : i + M_offset - N_offset + causal_offset + 1);++ const auto v_mask = _cvtu32_mask16((1U << (N_unmasked % 16)) - 1);+ int j = 0;+ auto v_max = _mm512_set1_ps(-INFINITY);+ for (; j < N_unmasked - 15; j += 16) {+ const auto xs = _mm512_mul_ps(v_scale, _mm512_cvtxph_ps(_mm256_loadu_ph(src + i * src_step + j)));+ v_max = _mm512_max_ps(v_max, xs);+ _mm512_storeu_ps(dst + i * ld_dst + j, xs);+ }+ if (j < N_unmasked) {+ const auto xs = _mm512_mul_ps(+ v_scale, _mm512_cvtxph_ps(_mm256_castsi256_ph(_mm256_maskz_loadu_epi16(v_mask, src + i * src_step + j))));+ v_max = _mm512_mask_max_ps(v_max, v_mask, v_max, xs);+ _mm512_storeu_ps(dst + i * ld_dst + j, xs);+ j += 16;+ }+ dst_max[i] = std::max(dst_max[i], _mm512_reduce_max_ps(v_max));+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE inplace_precompute_max_softmax_fp32_fp16(int m_size, int n_size, int n_pad_size, bool is_causal,+ float* src, utils::fp16* dst, const float* s_max,+ float* expsum, int ld_src, int ld_dst) {+ for (int ii = 0; ii < m_size; ++ii) {+ const auto i_src = src + ii * ld_src;+ const auto i_dst = dst + ii * ld_dst;+ const auto curr_n_size = n_size + (is_causal ? ii : 0);+ const uint16_t v_mask = _cvtu32_mask16((1U << (curr_n_size % 16)) - 1);+ { // subtract max+ const auto row_max = _mm512_set1_ps(s_max[ii]);+ for (int jj = 0; jj < curr_n_size; jj += 16) { // should be fine to do extra work on idx >= curr_n_size+ _mm512_storeu_ps(i_src + jj, _mm512_sub_ps(_mm512_loadu_ps(i_src + jj), row_max));+ }+ }+ auto v_sum = _mm512_setzero_ps();+ { // exp & sum+ int jj = 0;+ for (; jj < curr_n_size / 16 * 16; jj += 16) {+ const auto v_exp = kernel::avx512f::exp_ps_0_1(_mm512_loadu_ps(i_src + jj));+ v_sum = _mm512_add_ps(v_sum, v_exp);+ _mm512_storeu_ps(i_src + jj, v_exp);+ }+ if (jj < curr_n_size) {+ const auto v_exp =+ kernel::avx512f::exp_ps_0_1(_mm512_loadu_ps(i_src + jj)); // should be fine to load some extra+ v_sum = _mm512_mask_add_ps(v_sum, v_mask, v_sum, v_exp);+ _mm512_storeu_ps(i_src + jj, v_exp); // should be fine to store some extra+ }+ expsum[ii] = _mm512_reduce_add_ps(v_sum);+ v_sum = _mm512_set1_ps(expsum[ii]);+ }+ { // scale & fp16+ int jj = 0;+ for (; jj < curr_n_size / 16 * 16; jj += 16) {+ const auto v_softmax = _mm512_div_ps(_mm512_loadu_ps(i_src + jj), v_sum);+ _mm256_storeu_ph(i_dst + jj, _mm512_cvtxps_ph(v_softmax));+ }+ if (jj < curr_n_size) {+ const auto v_softmax = _mm512_div_ps(_mm512_loadu_ps(i_src + jj), v_sum);+ _mm256_storeu_ph(i_dst + jj, _mm512_maskz_cvtxps_ph(v_mask, v_softmax));+ jj += 16;+ }+ if (jj < n_pad_size) memset(i_dst + jj, 0, sizeof(utils::fp16) * (n_pad_size - jj));+ }+ }+ return BTLA_CODE::Success;+}+#if defined(__INTEL_LLVM_COMPILER)+#pragma clang attribute pop+#elif defined(__GNUC__)+#pragma GCC pop_options+#endif+#endif+} // namespace avx512_fp16+} // namespace avx512f+} // namespace kernel+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx512_vnni.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx512_vnni.hnew file mode 100644index 000000000..53d12b2b0--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx512_vnni.h@@ -0,0 +1,1528 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include "kernel_avx512f.h"++namespace bestla {+namespace kernel {+namespace avx512f {+#if CompileAVX512VNNI()+#if defined(__INTEL_LLVM_COMPILER)+#pragma clang attribute push(__attribute__((target("avx512f,avx512bw,avx512vl,avx512dq,avx512vnni"))), \+ apply_to = function)+#elif defined(__GNUC__)+#pragma GCC push_options+#pragma GCC target("avx512f", "avx512bw", "avx512vl", "avx512dq", "avx512vnni")+#endif++namespace vnni {++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_4bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto& a8ptr = A.aptr;+ auto& b4ptr = B.b4ptr;+ auto& asptr = A.sptr;+ auto& azptr = A.zpptr;+ int constexpr VLen = 16;+ int blks = k / blocksize;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ // Initialize accumulator with zeros+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (4 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ const __m512i onesu8 = _mm512_set1_epi8(1);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);++ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ __m512i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * VLen, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += 4) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits((void*)(b4ptr + i * 32 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_epi32(iacc[i], va, vb);+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits((void*)(b4ptr + i * 32 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], va[j], vb);+ }+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += 4) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits((void*)(b4ptr + i * 32 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_epi32(iacc[i], va, vb);+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits((void*)(b4ptr + i * 32 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], va[j], vb);+ }+ }+ }+ }+ }+ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_4bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto& a8ptr = A.aptr;+ auto& b4ptr = B.b4ptr;+ auto& asptr = A.sptr;++ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ // Initialize accumulator with zeros+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (4 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * VLen, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += 4) {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits((void*)(b4ptr + i * 32 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm512_sign_epi8(vb, va[j]);+ auto vabsa = _mm512_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += 4) {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits((void*)(b4ptr + i * 32 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm512_sub_epi8(vb, vbias);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm512_sign_epi8(vb, va[j]);+ auto vabsa = _mm512_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ }+ }+ }++ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_2bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ int constexpr VLen = 16;+ int blks = k / blocksize;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (2 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);+ const auto onesu8 = _mm512_set1_epi8(1);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ __m512i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_epi32(iacc[i], va, vb);+ b2ptr += VLen * KTILE / 4;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b2ptr += VLen * KTILE / 4;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_epi32(iacc[i], va, vb);+ b2ptr += VLen * KTILE / 4;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b2ptr += VLen * KTILE / 4;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_2bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ int constexpr VLen = 16;+ int blks = k / blocksize;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (2 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);+ const auto onesu8 = _mm512_set1_epi8(1);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }++ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(vbias, bzp[i]);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm512_sign_epi8(vb, va[j]);+ auto vabsa = _mm512_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b2ptr += VLen * KTILE / 4;+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb = _mm512_sub_epi8(vb, vbias);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm512_sign_epi8(vb, va[j]);+ auto vabsa = _mm512_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b2ptr += VLen * KTILE / 4;+ }+ }+ }+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_3bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (3 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ const auto onesu8 = _mm512_set1_epi8(1);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ __m512i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_epi32(iacc[i], va, vb);+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_epi32(iacc[i], va, vb);+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_3bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (3 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm512_sign_epi8(vb, va[j]);+ auto vabsa = _mm512_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm512_sign_epi8(vb, va[j]);+ auto vabsa = _mm512_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ }+ }++ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_1bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (1 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ const auto onesu8 = _mm512_set1_epi8(1);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ __m512i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, bzp[i]);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb1);+ iacc[i] = _mm512_dpbusd_epi32(iacc[i], va, vb1);+ b1ptr += VLen * KTILE / 8;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, bzp[i]);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb1);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], va[j], vb1);+ }+ b1ptr += VLen * KTILE / 8;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, vbias);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb1);+ iacc[i] = _mm512_dpbusd_epi32(iacc[i], va, vb1);+ b1ptr += VLen * KTILE / 8;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, vbias);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb1);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], va[j], vb1);+ }+ b1ptr += VLen * KTILE / 8;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_1bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (1 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, bzp[i]);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm512_sign_epi8(vb1, va[j]);+ auto vabsa = _mm512_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b1ptr += VLen * KTILE / 8;+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, vbias);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm512_sign_epi8(vb1, va[j]);+ auto vabsa = _mm512_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b1ptr += VLen * KTILE / 8;+ }+ }+ }++ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_5bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (5 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ const auto onesu8 = _mm512_set1_epi8(1);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ __m512i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_epi32(iacc[i], va, vb);+ b4ptr += VLen * KTILE / 2;+ b1ptr += VLen * KTILE / 8;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += VLen * KTILE / 2;+ b1ptr += VLen * KTILE / 8;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_epi32(iacc[i], va, vb);+ b4ptr += VLen * KTILE / 2;+ b1ptr += VLen * KTILE / 8;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += VLen * KTILE / 2;+ b1ptr += VLen * KTILE / 8;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_5bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (5 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm512_sign_epi8(vb, va[j]);+ auto vabsa = _mm512_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b4ptr += VLen * KTILE / 2;+ b1ptr += VLen * KTILE / 8;+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm512_sign_epi8(vb, va[j]);+ auto vabsa = _mm512_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b4ptr += VLen * KTILE / 2;+ b1ptr += VLen * KTILE / 8;+ }+ }+ }++ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_6bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);++ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (6 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ const auto onesu8 = _mm512_set1_epi8(1);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ __m512i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_epi32(iacc[i], va, vb);+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_epi32(iacc[i], va, vb);+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_6bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);++ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (6 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm512_sign_epi8(vb, va[j]);+ auto vabsa = _mm512_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm512_sign_epi8(vb, va[j]);+ auto vabsa = _mm512_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ }+ }+ }++ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_7bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (7 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ const auto onesu8 = _mm512_set1_epi8(1);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ __m512i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_or_si512(vb, vb2);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_epi32(iacc[i], va, vb);+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_or_si512(vb, vb2);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_or_si512(vb, vb2);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_epi32(iacc[i], va, vb);+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_or_si512(vb, vb2);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_7bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (7 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_or_si512(vb, vb2);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm512_sign_epi8(vb, va[j]);+ auto vabsa = _mm512_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_or_si512(vb, vb2);+ vb = _mm512_sub_epi8(vb, vbias);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm512_sign_epi8(vb, va[j]);+ auto vabsa = _mm512_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm512_dpbusd_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ }+ }++ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++} // namespace vnni++#if defined(__INTEL_LLVM_COMPILER)+#pragma clang attribute pop+#elif defined(__GNUC__)+#pragma GCC pop_options+#endif+#endif+} // namespace avx512f+} // namespace kernel+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx512f.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx512f.hnew file mode 100644index 000000000..0a662160e--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx512f.h@@ -0,0 +1,6562 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include "bestla.h"+#include "bestla_utils.h"+#include "kernel_jit.h"+#include "kernel_ref.h"++#include <array>+#include <cmath>+#include <cstring>+#include <type_traits>++namespace bestla {+namespace kernel {+namespace avx512f {+#if CompileAVX512F()+#if defined(__INTEL_LLVM_COMPILER)+#pragma clang attribute push(__attribute__((target("avx512f,avx512vl,avx512bw,avx512dq"))), apply_to = function)+#elif defined(__GNUC__)+#pragma GCC push_options+#pragma GCC target("avx512f", "avx512bw", "avx512vl", "avx512dq")+#endif++inline __m512 zmm_cvt_fp16_fp32(__m256i vfp16) { return _mm512_cvtph_ps(vfp16); }++inline __m256i zmm_cvt_fp32_fp16(__m512 vfp32) {+ return _mm512_cvtps_ph(vfp32, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);+}++inline __m512 zmm_cvt_bf16_fp32(__m256i vbf16) {+ auto vf32 = _mm512_cvtepu16_epi32(vbf16);+ return _mm512_castsi512_ps(_mm512_slli_epi32(vf32, 16));+}++static inline __m256i cvt_fp32_to_bf16(const __m512 _src) {+ const auto bf16_and_helper = _mm512_set1_epi32(0x00000001);+ const auto bf16_add_helper = _mm512_set1_epi32(0X00007FFF);+ auto src = _mm512_castps_si512(_src);+ auto round_bias = _mm512_and_epi32(bf16_and_helper, _mm512_bsrli_epi128(src, 2));+ round_bias = _mm512_add_epi32(round_bias, bf16_add_helper);+ auto round_fp32_v = _mm512_add_epi32(round_bias, src);+ auto pack_bf16_value = _mm512_cvtepi32_epi16(_mm512_srli_epi32(round_fp32_v, 16));+ return pack_bf16_value;+}++inline __m256i zmm_cvt_fp32_bf16(__m512 vfp32) {+#if FP32_BF16_FAST+ return _mm512_cvtepi32_epi16(_mm512_bsrli_epi128(_mm512_castps_si512(vfp32), 2));+#else+ return cvt_fp32_to_bf16(vfp32);+#endif+}++inline __m512i zmm_cvt_fp32_bf16_x2(__m512 vfp32_hi, __m512 vfp32_lo) {+ auto lo = zmm_cvt_fp32_bf16(vfp32_lo);+ auto hi = zmm_cvt_fp32_bf16(vfp32_hi);+ __m512i result = _mm512_castsi256_si512(lo);+ result = _mm512_inserti64x4(result, hi, 1);+ return result;+}++static inline __m512 load_bf16_fp32(const utils::bf16* srcptr) {+ auto tmp = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(srcptr));+ auto vf32 = zmm_cvt_bf16_fp32(tmp);+ return vf32;+}++static inline __m512 load_fp16_fp32(const utils::fp16* srcptr) {+ auto tmp = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(srcptr));+ auto vf32 = zmm_cvt_fp16_fp32(tmp);+ return vf32;+}++template <typename T>+static inline void convert_s8_fp_v16(T* dstptr, int8_t* srcptr) {+ auto xmm = _mm_loadu_si128(reinterpret_cast<const __m128i*>(srcptr));+ auto zmm = _mm512_cvtepi8_epi32(xmm);+ auto zmm1 = _mm512_cvtepi32_ps(zmm);+ if constexpr (std::is_same_v<T, utils::bf16>) {+ auto ymm = zmm_cvt_fp32_bf16(zmm1);+ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstptr), ymm);+ } else if constexpr (std::is_same_v<T, utils::fp16>) {+ auto ymm = zmm_cvt_fp32_fp16(zmm1);+ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstptr), ymm);+ } else {+ _mm512_storeu_ps(dstptr, zmm1);+ }+}++static inline __m512i load_s8_s32(int8_t* srcptr) {+ auto xmm = _mm_loadu_si128(reinterpret_cast<__m128i*>(srcptr));+ auto ymm = _mm512_cvtepi8_epi32(xmm);+ return ymm;+}++static inline __m512 load_s8_fp32(int8_t* srcptr) {+ auto src_y = load_s8_s32(srcptr);+ auto dst_y = _mm512_cvtepi32_ps(src_y);+ return dst_y;+}++template <typename T>+static inline void store_fp32_T(__m512 src_y, T* dstptr) {+ if constexpr (std::is_same_v<T, utils::bf16>) {+ auto ymm = zmm_cvt_fp32_bf16(src_y);+ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstptr), ymm);+ } else if constexpr (std::is_same_v<T, float>) {+ _mm512_storeu_ps(dstptr, src_y);+ } else if constexpr (std::is_same_v<T, utils::fp16>) {+ auto ymm = zmm_cvt_fp32_fp16(src_y);+ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstptr), ymm);+ } else {+ assert(false);+ }+}++template <typename T>+static inline __m512 load_T_fp32(const T* srcptr) {+ __m512 vtmp;+ if constexpr (std::is_same_v<T, float>) {+ vtmp = _mm512_loadu_ps(srcptr);+ } else if constexpr (std::is_same_v<T, utils::bf16>) {+ vtmp = load_bf16_fp32(srcptr);+ } else if constexpr (std::is_same_v<T, utils::fp16>) {+ vtmp = load_fp16_fp32(srcptr);+ } else {+ assert(0);+ }+ return vtmp;+}++static inline __m512i unpack_4bits(void* srcptr, __m512i mask) {+ auto raw_data = _mm256_loadu_si256(reinterpret_cast<__m256i*>(srcptr));+ auto ymm0 = _mm512_cvtepu8_epi16(raw_data);+ auto ymm1 = _mm512_slli_epi16(ymm0, 4);+ ymm0 = _mm512_or_si512(ymm0, ymm1);+ ymm0 = _mm512_and_si512(ymm0, mask);+ return ymm0;+}++static inline __m512i unpack_2bits(utils::bit2x4* ptr, const __m512i& vshift_y, const __m512i& vmask0_y,+ const __m512i& vsfhl_mask_y, const __m512i& vorder_y) {+ auto vraw_x = _mm_loadu_si128((const __m128i*)ptr);+ auto vsrc_y = _mm512_broadcast_i64x2(vraw_x);+ auto vordered_y = _mm512_permutex2var_epi32(vsrc_y, vorder_y, vsrc_y);+ auto vs_y = _mm512_srlv_epi32(vordered_y, vshift_y);+ auto v2_y = _mm512_and_si512(vs_y, vmask0_y);+ auto vout_y = _mm512_shuffle_epi8(v2_y, vsfhl_mask_y);+ return vout_y;+}++static inline __m512i unpack_1bits(utils::bit1x8* ptr, const __m512i& zmm_0x00, const __m512i& zmm_0x04) {+ auto bit1_mask1 = _cvtu64_mask64(*(uint64_t*)ptr);+ auto zmm1_ = _mm512_mask_mov_epi8(zmm_0x00, bit1_mask1, zmm_0x04);+ return zmm1_;+}++static inline __m512i unpack_4bits_high(__m256i v4bits, __m512i vmask) {+ auto ymm1 = _mm256_slli_epi32(v4bits, 4);+ auto zmm = _mm512_cvtepi8_epi16(v4bits);+ auto zmm1 = _mm512_cvtepi8_epi16(ymm1);+ zmm = _mm512_slli_epi16(zmm, 8);+ zmm1 = _mm512_mask_mov_epi8(zmm1, 0xaaaaaaaaaaaaaaaa, zmm);+ zmm1 = _mm512_and_epi32(zmm1, vmask);+ return zmm1;+}++static inline void convert_s4_s8_highbits(int8_t* dstptr, int8_t* srcptr, __m512i vmask, int LoadMask) {+ auto ymm = _mm256_maskz_loadu_epi32(__mmask8(LoadMask), reinterpret_cast<const __m256i*>(srcptr));+ auto zmm = unpack_4bits_high(ymm, vmask);+ _mm512_mask_storeu_epi64(dstptr, __mmask8(LoadMask), zmm);+}++static inline void convert_s4_s8_highbits_v32(int8_t* dstptr, int8_t* srcptr, __m512i vmask, int LoadMask) {+ auto xmm = _mm_maskz_loadu_epi32(__mmask8(LoadMask), reinterpret_cast<const __m256i*>(srcptr));+ auto ymm = _mm256_castsi128_si256(xmm);+ auto zmm = unpack_4bits_high(ymm, vmask);+ auto ymm_out = _mm512_castsi512_si256(zmm);+ _mm256_mask_storeu_epi64(dstptr, __mmask8(LoadMask), ymm_out);+}++constexpr void (*pad_fp4)(int8_t* dstptr, int8_t* srcptr, __m512i vmask, int) = &convert_s4_s8_highbits;++template <int N, typename _DST_T, bool _IS_SYM>+static inline void dequant_s8_N(_DST_T* dstptr, int8_t* srcptr, __m512* vscales, __m512i* vzps = nullptr) {+ static_assert(N % 16 == 0);+ int constexpr VLoop = N / 16;+ for (int iv = 0; iv < VLoop; iv += 1) {+ auto src_s8 = _mm_loadu_si128(reinterpret_cast<__m128i*>(srcptr + iv * 16));+ auto zmm = _mm512_cvtepi8_epi32(src_s8);+ if constexpr (!_IS_SYM) zmm = _mm512_sub_epi32(zmm, vzps[iv]);+ auto fzmm = _mm512_cvtepi32_ps(zmm);+ fzmm = _mm512_mul_ps(fzmm, vscales[iv]);+ if constexpr (std::is_same<_DST_T, float>::value) {+ _mm512_storeu_ps(dstptr + iv * 16, fzmm);+ } else if constexpr (std::is_same<_DST_T, utils::bf16>::value) {+ auto bf16_v = zmm_cvt_fp32_bf16(fzmm);+ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstptr + iv * 16), bf16_v);+ } else {+ assert(false);+ }+ }+}++template <bool IsAsym = false>+static inline __m512 dequant_s8_fp(int8_t* srcptr, __m512 vscales, __m512i vzps = __m512i()) {+ auto src_s32_y = load_s8_s32(srcptr);+ if constexpr (IsAsym) src_s32_y = _mm512_sub_epi32(src_s32_y, vzps);+ auto src_fp_y = _mm512_cvtepi32_ps(src_s32_y);+ src_fp_y = _mm512_mul_ps(src_fp_y, vscales);+ return src_fp_y;+}++template <int N, typename _DST_T, BTLA_DTYPE F4_T>+static inline void dequant_f4_N(_DST_T* dstptr, int8_t* srcptr, __m512* vscales, __m512i* vzps = nullptr) {+ static_assert(N % 16 == 0);+ int constexpr VLoop = N / 16;+ float* LUT;+ static_assert(F4_T == BTLA_DTYPE::F4_BNB || F4_T == BTLA_DTYPE::F4_NF4 || F4_T == BTLA_DTYPE::F4_E2M1,+ "Unsupported F4 type");+ if constexpr (F4_T == BTLA_DTYPE::F4_BNB) {+ LUT = fp4_bnb_dequant_fp32_LUT;+ } else if constexpr (F4_T == BTLA_DTYPE::F4_NF4) {+ LUT = nf4_dequant_fp32_LUT;+ } else if constexpr (F4_T == BTLA_DTYPE::F4_E2M1) {+ LUT = fp4_e2m1_dequant_fp32_LUT;+ }+ for (int iv = 0; iv < VLoop; iv += 1) {+ auto idx = _mm_loadu_si128(reinterpret_cast<__m128i*>(srcptr + iv * 16));+ idx = _mm_srli_epi32(idx, 4);+ auto pad_idx = _mm512_cvtepu8_epi32(idx);+ auto lut = _mm512_loadu_si512(LUT);+ auto fp32_dq_v = _mm512_permutexvar_epi32(pad_idx, lut);+ auto fzmm = _mm512_mul_ps(_mm512_castsi512_ps(fp32_dq_v), vscales[iv]);+ if constexpr (std::is_same<_DST_T, float>::value) {+ _mm512_storeu_ps(dstptr + iv * 16, fzmm);+ } else if constexpr (std::is_same<_DST_T, utils::bf16>::value) {+ auto bf16_v = zmm_cvt_fp32_bf16(fzmm);+ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstptr + iv * 16), bf16_v);+ } else {+ assert(false);+ }+ }+}++template <int N, typename _DST_T, BTLA_DTYPE F4_T>+static inline void unpack_f4_N(_DST_T* dstptr, int8_t* srcptr) {+ static_assert(N % 16 == 0);+ int constexpr VLoop = N / 16;+ float* LUT;+ static_assert(F4_T == BTLA_DTYPE::F4_BNB || F4_T == BTLA_DTYPE::F4_NF4 || F4_T == BTLA_DTYPE::F4_E2M1,+ "Unsupported F4 type");+ if constexpr (F4_T == BTLA_DTYPE::F4_BNB) {+ LUT = fp4_bnb_dequant_fp32_LUT;+ } else if constexpr (F4_T == BTLA_DTYPE::F4_NF4) {+ LUT = nf4_dequant_fp32_LUT;+ } else if constexpr (F4_T == BTLA_DTYPE::F4_E2M1) {+ LUT = fp4_e2m1_dequant_fp32_LUT;+ }+ for (int iv = 0; iv < VLoop; iv += 1) {+ auto idx = _mm_loadu_si128(reinterpret_cast<__m128i*>(srcptr + iv * 16));+ idx = _mm_srli_epi32(idx, 4);+ auto pad_idx = _mm512_cvtepu8_epi32(idx);+ auto lut = _mm512_loadu_si512(LUT);+ auto fp32_dq_v = _mm512_permutexvar_epi32(pad_idx, lut);+ auto fzmm = _mm512_castsi512_ps(fp32_dq_v);+ if constexpr (std::is_same<_DST_T, float>::value) {+ _mm512_storeu_ps(dstptr + iv * 16, fzmm);+ } else if constexpr (std::is_same<_DST_T, utils::bf16>::value) {+ auto bf16_v = zmm_cvt_fp32_bf16(fzmm);+ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstptr + iv * 16), bf16_v);+ } else {+ assert(false);+ }+ }+}++static inline void vec_broadcast_epi32_1_2(__m512i* dst2regs, __m512i* src1regs) {+ dst2regs[0] = _mm512_unpacklo_epi32(src1regs[0], src1regs[0]);+ dst2regs[1] = _mm512_unpackhi_epi32(src1regs[0], src1regs[0]);+}++static inline void vec_broadcast_ps_1_2(__m512* dst2regs, __m512* src1regs, __m512i idxreg) {+ auto tmpreg = _mm512_permutexvar_epi64(idxreg, _mm512_castps_si512(src1regs[0]));+ dst2regs[0] = _mm512_castsi512_ps(_mm512_unpacklo_epi32(tmpreg, tmpreg));+ dst2regs[1] = _mm512_castsi512_ps(_mm512_unpackhi_epi32(tmpreg, tmpreg));+}++template <bool LowBits>+static inline __m512 broadcast_ps_1_2(__m512 vsrc_y, const __m512i& vshuf_index_high, const __m512i& vshuf_index_low) {+ __m512 tmp;+ if constexpr (LowBits) {+ tmp = _mm512_permutex2var_ps(vsrc_y, vshuf_index_low, vsrc_y);+ } else {+ tmp = _mm512_permutex2var_ps(vsrc_y, vshuf_index_high, vsrc_y);+ }+ return tmp;+}++template <bool LowBits>+static inline __m512i broadcast_epi32_1_2(__m512i vsrc_y, const __m512i& vshuf_index_high,+ const __m512i& vshuf_index_low) {+ return _mm512_castps_si512(broadcast_ps_1_2<LowBits>(_mm512_castsi512_ps(vsrc_y), vshuf_index_high, vshuf_index_low));+}++static inline void vec_broadcast_epi32_1_2(__m512i* dst2regs, __m512i* src1regs, __m512i idxreg) {+ auto tmpreg = _mm512_permutexvar_epi64(idxreg, src1regs[0]);+ dst2regs[0] = _mm512_unpacklo_epi32(tmpreg, tmpreg);+ dst2regs[1] = _mm512_unpackhi_epi32(tmpreg, tmpreg);+}++static inline void vec_broadcast_pi8_1_2(__m128i* dst2regs, __m128i* src1regs, __m128i idxreg) {+ auto tmpreg = _mm_permutexvar_epi16(idxreg, src1regs[0]);+ dst2regs[0] = _mm_unpacklo_epi8(tmpreg, tmpreg);+ dst2regs[1] = _mm_unpackhi_epi8(tmpreg, tmpreg);+}++static inline void vec_broadcast_epi32_2_4(__m512i* dst4regs, __m512i* src2regs) {+ vec_broadcast_epi32_1_2(dst4regs, src2regs);+ vec_broadcast_epi32_1_2(dst4regs + 2, src2regs + 1);+}++static inline __m512i _mm512_sign_epi8(__m512i a, __m512i b) {+ __m512i zero = _mm512_setzero_si512();+ __mmask64 blt0 = _mm512_movepi8_mask(b);+ return _mm512_mask_sub_epi8(a, blt0, zero, a);+}++template <typename _ST, typename _DT, bool _IS_SYM>+static inline BTLA_CODE decompress_kblock_bit4_packrow1(utils::bit4x2* srcptr, _DT* dstptr, int row, int col,+ int ld_src, int ld_dst, _ST* scales, int8_t* zero_points,+ int k_offset, int kblock, int NPad,+ void (*dequantize)(_DT*, int8_t*, __m512*, __m512i*),+ void (*pad_bit4)(int8_t*, int8_t*, __m512i, int),+ int8_t* tmpbuf, size_t tmpsize) {+ uint32_t mask = 0xf0f0f0f0;+ auto zmm_mask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ if (col == 48) {+ constexpr int ColTile = 48;+ constexpr int NRegs = ColTile / 16;+ constexpr int LoadMask64 = (1 << (64 / 8)) - 1;+ constexpr int LoadMask48 = (1 << (48 / 8)) - 1;+ __m512 vscales[NRegs];+ __m512i vzps[NRegs];+ int constexpr UnrollRow = 4;+ int constexpr Loop64 = ColTile * UnrollRow / 64;+ assert(tmpsize >= (ColTile * UnrollRow));+ int row0 = kblock - k_offset % kblock;+ row0 = row0 == kblock ? 0 : row0;+ row0 = row0 > row ? row : row0;+ int row1 = row - row0;+ int irow = 0;+ if (row0) {+ int rowpad4 = utils::padto_le(row0, UnrollRow);+ for (int iv = 0; iv < 3; iv++) {+ vscales[iv] = load_T_fp32(scales + (k_offset + irow) / kblock * NPad + iv * 16);+ if constexpr (!_IS_SYM) {+ auto tmp =+ _mm_loadu_si128(reinterpret_cast<__m128i*>(zero_points + (k_offset + irow) / kblock * NPad + iv * 16));+ vzps[iv] = _mm512_cvtepi8_epi32(tmp);+ }+ }+ for (; irow < rowpad4; irow += UnrollRow) {+ for (int iter64 = 0; iter64 < Loop64; iter64++) {+ pad_bit4(tmpbuf + iter64 * 64, reinterpret_cast<int8_t*>(srcptr + irow * ld_src / 2 + 32 * iter64), zmm_mask,+ LoadMask64);+ }+ for (int iterr = 0; iterr < UnrollRow; iterr++) {+ if constexpr (_IS_SYM) {+ dequantize(dstptr + (irow + iterr) * ld_dst, tmpbuf + iterr * ColTile, vscales, nullptr);+ } else {+ dequantize(dstptr + (irow + iterr) * ld_dst, tmpbuf + iterr * ColTile, vscales, vzps);+ }+ }+ }+ for (; irow < row0; irow++) {+ pad_bit4(tmpbuf, reinterpret_cast<int8_t*>(srcptr + irow * ld_src / 2), zmm_mask, LoadMask48);+ if constexpr (_IS_SYM) {+ dequantize(dstptr + irow * ld_dst, tmpbuf, vscales, nullptr);+ } else {+ dequantize(dstptr + irow * ld_dst, tmpbuf, vscales, vzps);+ }+ }+ }++ int row1_blk = utils::padto_le(row1, kblock) + row0;+ assert(kblock % UnrollRow == 0);+ assert(ld_src == 48); // no padding for unroll process++ for (; irow < row1_blk; irow += kblock) {+ for (int iv = 0; iv < 3; iv++) {+ vscales[iv] = load_T_fp32(scales + (k_offset + irow) / kblock * NPad + iv * 16);+ if constexpr (!_IS_SYM) {+ auto tmp =+ _mm_loadu_si128(reinterpret_cast<__m128i*>(zero_points + (k_offset + irow) / kblock * NPad + iv * 16));+ vzps[iv] = _mm512_cvtepi8_epi32(tmp);+ }+ }++ for (int irr = 0; irr < kblock; irr += UnrollRow) {+ for (int iter64 = 0; iter64 < Loop64; iter64++) {+ pad_bit4(tmpbuf + iter64 * 64, reinterpret_cast<int8_t*>(srcptr + (irow + irr) * ld_src / 2 + 32 * iter64),+ zmm_mask, LoadMask64);+ }+ for (int iterr = 0; iterr < UnrollRow; iterr++) {+ if constexpr (_IS_SYM) {+ dequantize(dstptr + (irow + irr + iterr) * ld_dst, tmpbuf + iterr * ColTile, vscales, nullptr);+ } else {+ dequantize(dstptr + (irow + irr + iterr) * ld_dst, tmpbuf + iterr * ColTile, vscales, vzps);+ }+ }+ }+ }+ if (irow < row) {+ for (int iv = 0; iv < 3; iv++) {+ vscales[iv] = load_T_fp32(scales + (k_offset + irow) / kblock * NPad + iv * 16);+ if constexpr (!_IS_SYM) {+ auto tmp =+ _mm_loadu_si128(reinterpret_cast<__m128i*>(zero_points + (k_offset + irow) / kblock * NPad + iv * 16));+ vzps[iv] = _mm512_cvtepi8_epi32(tmp);+ }+ }+ auto rowre = row - irow;+ int rowpad4 = utils::padto_le(rowre, UnrollRow) + irow;+ for (; irow < rowpad4; irow += UnrollRow) {+ for (int iter64 = 0; iter64 < Loop64; iter64++) {+ pad_bit4(tmpbuf + iter64 * 64, reinterpret_cast<int8_t*>(srcptr + irow * ld_src / 2 + 32 * iter64), zmm_mask,+ LoadMask64);+ }+ for (int iterr = 0; iterr < UnrollRow; iterr++) {+ if constexpr (_IS_SYM) {+ dequantize(dstptr + (irow + iterr) * ld_dst, tmpbuf + iterr * ColTile, vscales, nullptr);+ } else {+ dequantize(dstptr + (irow + iterr) * ld_dst, tmpbuf + iterr * ColTile, vscales, vzps);+ }+ }+ }+ for (; irow < row; irow++) {+ pad_bit4(tmpbuf, reinterpret_cast<int8_t*>(srcptr + irow * ld_src / 2), zmm_mask, LoadMask48);+ if constexpr (_IS_SYM) {+ dequantize(dstptr + irow * ld_dst, tmpbuf, vscales, nullptr);+ } else {+ dequantize(dstptr + irow * ld_dst, tmpbuf, vscales, vzps);+ }+ }+ }+ return BTLA_CODE::Success;+ }+ return BTLA_CODE::NotSupport;+}++template <BTLA_DTYPE _SRCT, typename _ST, typename _DT, bool _IS_SYM = true>+static inline BTLA_CODE decompress_kblock_bit4_packrow2(utils::bit4x2* srcptr, _DT* dstptr, int row, int col,+ int ld_src, int ld_dst, _ST* scales, int8_t* zero_points,+ int k_offset, int kblock, int NPad, int8_t* tmpbuf,+ size_t tmpsize) {+ uint32_t mask = 0xf0f0f0f0;+ auto zmm_mask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto broadcast_idx = _mm512_setr_epi64(0, 4, 1, 5, 2, 6, 3, 7);+ auto broadcast_idx_128 = _mm_setr_epi16(0, 1, 2, 3, 4, 5, 6, 7);+ auto constexpr SRC_TYPE =+ static_cast<BTLA_DTYPE>(utils::bestla_dtype_get_mask_val(_SRCT, BTLA_DTYPE::TypeMask, BTLA_DTYPE::TypeShift));+ if (col % 64 == 0) {+ constexpr int ColTile = 64;+ constexpr int NRegs = ColTile / 16;+ constexpr int ScaleRegCount = NRegs / 2;+ constexpr int LoadMask64 = (1 << (64 / 8)) - 1;+ for (int icol = 0; icol < col; icol += ColTile) {+ __m512 vscales[NRegs];+ __m512i vzps[NRegs];+ assert(tmpsize >= ColTile);+ int row0 = kblock - k_offset % kblock;+ row0 = row0 == kblock ? 0 : row0;+ row0 = row0 > row ? row : row0;+ int row1 = row - row0;+ int irow = 0;+ if (row0) {+ for (int iv = 0; iv < ScaleRegCount; iv++) {+ auto tmpscale = load_T_fp32(scales + (k_offset + irow) / kblock * NPad + iv * 16 + icol / 2);+ vec_broadcast_ps_1_2(vscales + iv * 2, &tmpscale, broadcast_idx);+ if constexpr (!_IS_SYM) {+ auto tmpzp = _mm_loadu_si128(+ reinterpret_cast<__m128i*>(zero_points + (k_offset + irow) / kblock * NPad + iv * 16 + icol / 2));+ auto vzp = _mm512_cvtepi8_epi32(tmpzp);+ vec_broadcast_epi32_1_2(vzps + iv * 2, &vzp, broadcast_idx);+ }+ }++ for (; irow < row0; irow++) {+ convert_s4_s8_highbits(tmpbuf, reinterpret_cast<int8_t*>(srcptr + irow * ld_src / 2 + icol / 2), zmm_mask,+ LoadMask64);+ if constexpr (SRC_TYPE == BTLA_DTYPE::TypeFloat) {+ dequant_f4_N<ColTile, _DT, _SRCT>(dstptr + irow * ld_dst + icol, tmpbuf, vscales, vzps);+ } else {+ dequant_s8_N<ColTile, _DT, _IS_SYM>(dstptr + irow * ld_dst + icol, tmpbuf, vscales, vzps);+ }+ }+ }++ int row1_blk = utils::padto_le(row1, kblock) + row0;+ for (; irow < row1_blk; irow += kblock) {+ for (int iv = 0; iv < ScaleRegCount; iv++) {+ auto tmpscale = load_T_fp32(scales + (k_offset + irow) / kblock * NPad + iv * 16 + icol / 2);+ vec_broadcast_ps_1_2(vscales + iv * 2, &tmpscale, broadcast_idx);+ if constexpr (!_IS_SYM) {+ auto tmpzp = _mm_loadu_si128(+ reinterpret_cast<__m128i*>(zero_points + (k_offset + irow) / kblock * NPad + iv * 16 + icol / 2));+ auto vzp = _mm512_cvtepi8_epi32(tmpzp);+ vec_broadcast_epi32_1_2(vzps + iv * 2, &vzp, broadcast_idx);+ }+ }++ for (int irr = 0; irr < kblock; irr += 1) {+ convert_s4_s8_highbits(tmpbuf, reinterpret_cast<int8_t*>(srcptr + (irow + irr) * ld_src / 2 + icol / 2),+ zmm_mask, LoadMask64);+ if constexpr (SRC_TYPE == BTLA_DTYPE::TypeFloat) {+ dequant_f4_N<ColTile, _DT, _SRCT>(dstptr + (irow + irr) * ld_dst + icol, tmpbuf, vscales, vzps);+ } else {+ dequant_s8_N<ColTile, _DT, _IS_SYM>(dstptr + (irow + irr) * ld_dst + icol, tmpbuf, vscales, vzps);+ }+ }+ }+ if (irow < row) {+ for (int iv = 0; iv < ScaleRegCount; iv++) {+ auto tmpscale = load_T_fp32(scales + (k_offset + irow) / kblock * NPad + iv * 16 + icol / 2);+ vec_broadcast_ps_1_2(vscales + iv * 2, &tmpscale, broadcast_idx);+ if constexpr (!_IS_SYM) {+ auto tmpzp = _mm_loadu_si128(+ reinterpret_cast<__m128i*>(zero_points + (k_offset + irow) / kblock * NPad + iv * 16 + icol / 2));+ auto vzp = _mm512_cvtepi8_epi32(tmpzp);+ vec_broadcast_epi32_1_2(vzps + iv * 2, &vzp, broadcast_idx);+ }+ }+ }+ for (; irow < row; irow++) {+ convert_s4_s8_highbits(tmpbuf, reinterpret_cast<int8_t*>(srcptr + irow * ld_src / 2 + icol / 2), zmm_mask,+ LoadMask64);+ if constexpr (SRC_TYPE == BTLA_DTYPE::TypeFloat) {+ dequant_f4_N<ColTile, _DT, _SRCT>(dstptr + irow * ld_dst + icol, tmpbuf, vscales, vzps);+ } else {+ dequant_s8_N<ColTile, _DT, _IS_SYM>(dstptr + irow * ld_dst + icol, tmpbuf, vscales, vzps);+ }+ }+ }+ return BTLA_CODE::Success;+ } else if (col % 96 == 0) {+ constexpr int ColTile = 96;+ constexpr int NRegs = ColTile / 16;+ constexpr int ScaleRegCount = NRegs / 2;+ constexpr int LoadMask64 = (1 << (64 / 8)) - 1;+ for (int icol = 0; icol < col; icol += ColTile) {+ __m512 vscales[NRegs];+ __m512i vzps[NRegs];+ assert(tmpsize >= ColTile);+ int row0 = kblock - k_offset % kblock;+ row0 = row0 == kblock ? 0 : row0;+ row0 = row0 > row ? row : row0;+ int row1 = row - row0;+ int irow = 0;+ if (row0) {+ for (int iv = 0; iv < ScaleRegCount; iv++) {+ auto tmpscale = load_T_fp32(scales + (k_offset + irow) / kblock * NPad + iv * 16 + icol / 2);+ vec_broadcast_ps_1_2(vscales + iv * 2, &tmpscale, broadcast_idx);+ if constexpr (!_IS_SYM) {+ auto tmpzp = _mm_loadu_si128(+ reinterpret_cast<__m128i*>(zero_points + (k_offset + irow) / kblock * NPad + iv * 16 + icol / 2));+ auto vzp = _mm512_cvtepi8_epi32(tmpzp);+ vec_broadcast_epi32_1_2(vzps + iv * 2, &vzp, broadcast_idx);+ }+ }++ for (; irow < row0; irow++) {+ convert_s4_s8_highbits(tmpbuf, reinterpret_cast<int8_t*>(srcptr + irow * ld_src / 2 + icol / 2), zmm_mask,+ LoadMask64);+ convert_s4_s8_highbits_v32(tmpbuf + 64, reinterpret_cast<int8_t*>(srcptr + irow * ld_src / 2 + icol / 2 + 32),+ zmm_mask, LoadMask64);+ if constexpr (SRC_TYPE == BTLA_DTYPE::TypeFloat) {+ dequant_f4_N<ColTile, _DT, _SRCT>(dstptr + irow * ld_dst + icol, tmpbuf, vscales, vzps);+ } else {+ dequant_s8_N<ColTile, _DT, _IS_SYM>(dstptr + irow * ld_dst + icol, tmpbuf, vscales, vzps);+ }+ }+ }++ int row1_blk = utils::padto_le(row1, kblock) + row0;+ for (; irow < row1_blk; irow += kblock) {+ for (int iv = 0; iv < ScaleRegCount; iv++) {+ auto tmpscale = load_T_fp32(scales + (k_offset + irow) / kblock * NPad + iv * 16 + icol / 2);+ vec_broadcast_ps_1_2(vscales + iv * 2, &tmpscale, broadcast_idx);+ if constexpr (!_IS_SYM) {+ auto tmpzp = _mm_loadu_si128(+ reinterpret_cast<__m128i*>(zero_points + (k_offset + irow) / kblock * NPad + iv * 16 + icol / 2));+ auto vzp = _mm512_cvtepi8_epi32(tmpzp);+ vec_broadcast_epi32_1_2(vzps + iv * 2, &vzp, broadcast_idx);+ }+ }++ for (int irr = 0; irr < kblock; irr += 1) {+ convert_s4_s8_highbits(tmpbuf, reinterpret_cast<int8_t*>(srcptr + (irow + irr) * ld_src / 2 + icol / 2),+ zmm_mask, LoadMask64);+ convert_s4_s8_highbits_v32(tmpbuf + 64,+ reinterpret_cast<int8_t*>(srcptr + (irow + irr) * ld_src / 2 + icol / 2 + 32),+ zmm_mask, LoadMask64);+ if constexpr (SRC_TYPE == BTLA_DTYPE::TypeFloat) {+ dequant_f4_N<ColTile, _DT, _SRCT>(dstptr + (irow + irr) * ld_dst + icol, tmpbuf, vscales, vzps);+ } else {+ dequant_s8_N<ColTile, _DT, _IS_SYM>(dstptr + (irow + irr) * ld_dst + icol, tmpbuf, vscales, vzps);+ }+ }+ }+ if (irow < row) {+ for (int iv = 0; iv < ScaleRegCount; iv++) {+ auto tmpscale = load_T_fp32(scales + (k_offset + irow) / kblock * NPad + iv * 16 + icol / 2);+ vec_broadcast_ps_1_2(vscales + iv * 2, &tmpscale, broadcast_idx);+ if constexpr (!_IS_SYM) {+ auto tmpzp = _mm_loadu_si128(+ reinterpret_cast<__m128i*>(zero_points + (k_offset + irow) / kblock * NPad + iv * 16 + icol / 2));+ auto vzp = _mm512_cvtepi8_epi32(tmpzp);+ vec_broadcast_epi32_1_2(vzps + iv * 2, &vzp, broadcast_idx);+ }+ }+ }+ for (; irow < row; irow++) {+ convert_s4_s8_highbits(tmpbuf, reinterpret_cast<int8_t*>(srcptr + irow * ld_src / 2 + icol / 2), zmm_mask,+ LoadMask64);+ convert_s4_s8_highbits_v32(tmpbuf + 64, reinterpret_cast<int8_t*>(srcptr + irow * ld_src / 2 + icol / 2 + 32),+ zmm_mask, LoadMask64);+ if constexpr (SRC_TYPE == BTLA_DTYPE::TypeFloat) {+ dequant_f4_N<ColTile, _DT, _SRCT>(dstptr + irow * ld_dst + icol, tmpbuf, vscales, vzps);+ } else {+ dequant_s8_N<ColTile, _DT, _IS_SYM>(dstptr + irow * ld_dst + icol, tmpbuf, vscales, vzps);+ }+ }+ }++ return BTLA_CODE::Success;+ }+ return BTLA_CODE::NotSupport;+}++template <bool WITH_SCALE, typename _DST_T, int _PACK_ROW, typename _S_T>+inline BTLA_CODE decompress_kblock_f8_fp(utils::f8* srcptr, _DST_T* dstptr, int row, int col, int ld_src, int ld_dst,+ _S_T* scales, int k_offset, int kblock, int NPad, BTLA_DTYPE src_f8_type) {+ int align_col = col / 16 * 16;+ int col_tail = col - align_col;+ auto ebits = utils::bestla_dtype_get_f8_ebits(src_f8_type);+ auto mantissabit = 7 - ebits;+ auto sign_revert_and_mask = _mm512_set1_epi32(0x80000000);+ auto e_revert_and_mask = _mm512_set1_epi32(0x0000007f);+ auto e_revert_shift = _mm512_set1_epi32(1);+ e_revert_shift = _mm512_slli_epi32(e_revert_shift, ebits - 1);+ e_revert_shift = _mm512_sub_epi32(e_revert_shift, _mm512_set1_epi32(128));+ auto mantissa_revert_and_mask = _mm512_set1_epi32(0x007fffff);+ auto packrow2_permute_idx = _mm512_setr_epi32(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7);+ for (int i = 0; i < row; i++) {+ int kpos = (k_offset + i) / kblock;+ auto sptr = scales + kpos * NPad;+ int j = 0;+ auto quant = [&](__mmask16 mask) {+ auto sign_revert =+ _mm512_cvtepi8_epi32(_mm_maskz_loadu_epi8(mask, reinterpret_cast<__m128i*>(srcptr + i * ld_src + j)));+ auto e_revert = sign_revert;+ auto mantissa_revert = sign_revert;+ sign_revert = _mm512_slli_epi32(sign_revert, 24);+ sign_revert = _mm512_and_epi32(sign_revert, sign_revert_and_mask);+ e_revert = _mm512_and_epi32(e_revert, e_revert_and_mask);+ e_revert = _mm512_srli_epi32(e_revert, mantissabit);+ e_revert = _mm512_sub_epi32(e_revert, e_revert_shift);+ if constexpr (WITH_SCALE && std::is_same_v<_S_T, utils::f8>) {+ auto scale = _mm512_cvtepi8_epi32(_mm_loadu_si128(reinterpret_cast<__m128i*>(sptr + j / _PACK_ROW)));+ if constexpr (_PACK_ROW == 2) scale = _mm512_permutexvar_epi32(packrow2_permute_idx, scale);+ e_revert = _mm512_add_epi32(e_revert, scale);+ }+ e_revert = _mm512_slli_epi32(e_revert, 23);+ mantissa_revert = _mm512_slli_epi32(mantissa_revert, 23 - mantissabit);+ mantissa_revert = _mm512_and_epi32(mantissa_revert, mantissa_revert_and_mask);+ auto fp_v = _mm512_or_ps(_mm512_castsi512_ps(sign_revert), _mm512_castsi512_ps(e_revert));+ fp_v = _mm512_or_ps(fp_v, _mm512_castsi512_ps(mantissa_revert));+ if constexpr (WITH_SCALE && std::is_same_v<_S_T, float>) {+ auto scale = _mm512_loadu_ps(sptr + j / _PACK_ROW);+ if constexpr (_PACK_ROW == 2) scale = _mm512_permutexvar_ps(packrow2_permute_idx, scale);+ fp_v = _mm512_mul_ps(fp_v, scale);+ }+ if constexpr (std::is_same_v<_DST_T, float>) {+ _mm512_mask_storeu_ps(dstptr + i * ld_dst + j, mask, fp_v);+ } else if constexpr (std::is_same_v<_DST_T, utils::bf16>) {+ auto bf16_v = zmm_cvt_fp32_bf16(fp_v);+ _mm256_mask_storeu_epi16(reinterpret_cast<__m256i*>(dstptr + i * ld_dst + j), mask, bf16_v);+ } else {+ assert(0);+ }+ };+ for (; j < align_col; j += 16) quant(_cvtu32_mask16(0xffff));+ if (col_tail > 0) quant(_cvtu32_mask16(0xffff >> (16 - col_tail)));+ }+ return BTLA_CODE::Success;+}++template <BTLA_DTYPE S4_T, typename _DST_T, int _PACK_ROW, typename _ST>+static inline BTLA_CODE decompress_kblock_s4_fp_Dep(utils::int4x2* srcptr, _DST_T* dstptr, int row, int col, int ld_src,+ int ld_dst, _ST* scales, int8_t* zero_points, int k_offset,+ int kblock, int NPad, int8_t* tmp, size_t tmpsize) {+ if constexpr (_PACK_ROW == 1) {+ if (zero_points == nullptr) {+ return decompress_kblock_bit4_packrow1<_ST, _DST_T, true>(+ srcptr, dstptr, row, col, ld_src, ld_dst, scales, zero_points, k_offset, kblock, NPad,+ &dequant_s8_N<48, _DST_T, true>, &convert_s4_s8_highbits, tmp, tmpsize);+ } else {+ return decompress_kblock_bit4_packrow1<_ST, _DST_T, false>(+ srcptr, dstptr, row, col, ld_src, ld_dst, scales, zero_points, k_offset, kblock, NPad,+ &dequant_s8_N<48, _DST_T, false>, &convert_s4_s8_highbits, tmp, tmpsize);+ }+ } else if constexpr (_PACK_ROW == 2) {+ if (zero_points == nullptr) {+ return decompress_kblock_bit4_packrow2<S4_T, _ST, _DST_T, true>(+ srcptr, dstptr, row, col, ld_src, ld_dst, scales, zero_points, k_offset, kblock, NPad, tmp, tmpsize);+ } else {+ return decompress_kblock_bit4_packrow2<S4_T, _ST, _DST_T, false>(+ srcptr, dstptr, row, col, ld_src, ld_dst, scales, zero_points, k_offset, kblock, NPad, tmp, tmpsize);+ }+ }+ return BTLA_CODE::NotSupport;+}++template <BTLA_DTYPE S3_T, typename _DST_T>+inline BTLA_CODE decompress_kblock_s3_s8fp(utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr, _DST_T* dstptr,+ int interleave_n_offset, int unpack_elt, int8_t* tmp, size_t tmpsize) {+ auto head_ignore_num = interleave_n_offset % 128;+ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ auto zmm_shift = _mm512_set1_epi32(5);++ auto bit3_interleave_decompress_pack128 = [&](utils::bit2x4* src1, utils::bit1x8* src2, int8_t* dst) {+ const __m256i lowMask = _mm256_set1_epi8(0x03);+ const __m256i bit2_data = _mm256_loadu_si256((const __m256i*)src1);+ auto ymm0 = _mm256_and_si256(lowMask, bit2_data); // uop:1 p:015+ auto ymm1 = _mm256_and_si256(lowMask, _mm256_srli_epi16(bit2_data, 2)); // uop:1 p:01+ auto ymm2 = _mm256_and_si256(lowMask, _mm256_srli_epi16(bit2_data, 4));+ auto ymm3 = _mm256_and_si256(lowMask, _mm256_srli_epi16(bit2_data, 6));+ auto zmm1 = _mm512_inserti32x8(_mm512_castsi256_si512(ymm0), ymm1, 0x1); // lat3, tp1 uop1 p:5+ auto zmm2 = _mm512_inserti32x8(_mm512_castsi256_si512(ymm2), ymm3, 0x1);++ unsigned long long* bit1_ptr = reinterpret_cast<unsigned long long*>(src2);+ auto bit1_mask1 = _cvtu64_mask64(*bit1_ptr);+ auto bit1_mask2 = _cvtu64_mask64(*(bit1_ptr + 1));+ auto zmm1_ = _mm512_mask_mov_epi8(zmm_0x00, bit1_mask1, zmm_0x04);+ auto zmm2_ = _mm512_mask_mov_epi8(zmm_0x00, bit1_mask2, zmm_0x04);+ zmm1 = _mm512_add_epi8(zmm1, zmm1_);+ zmm2 = _mm512_add_epi8(zmm2, zmm2_);+ zmm1 = _mm512_sllv_epi32(zmm1, zmm_shift); // int3_clip => int8+ zmm2 = _mm512_sllv_epi32(zmm2, zmm_shift); // int3_clip => int8++ _mm512_storeu_si512((__m512i*)dst, zmm1);+ _mm512_storeu_si512((__m512i*)(dst + 64), zmm2);+ };+ int compress_wei_ptr_offset = 0;+ if (head_ignore_num != 0) {+ assert(head_ignore_num % 8 == 0);++ auto base_bit2ptr = bit2ptr - head_ignore_num / 4;+ auto base_bit1ptr = bit1ptr - head_ignore_num / 8;+ auto head_write_num = 128 - head_ignore_num;+ bit3_interleave_decompress_pack128(base_bit2ptr, base_bit1ptr, tmp);+ for (int i = 0; i < head_write_num; i++) dstptr[i] = tmp[head_ignore_num + i];+ compress_wei_ptr_offset += head_write_num;+ unpack_elt -= head_write_num;+ }+ auto body_loop = unpack_elt / 128;+ auto tail_proc_num = unpack_elt % 128;++ bestla::kernel::jit::DecompressS3::forward_avx512f(bit2ptr + compress_wei_ptr_offset / 4,+ bit1ptr + compress_wei_ptr_offset / 8,+ dstptr + compress_wei_ptr_offset, tmp, body_loop * 128);+ compress_wei_ptr_offset += body_loop * 128;+ if (tail_proc_num > 0) {+ bit3_interleave_decompress_pack128(bit2ptr + compress_wei_ptr_offset / 4, bit1ptr + compress_wei_ptr_offset / 8,+ tmp);+ for (int i = 0; i < tail_proc_num; i++) dstptr[compress_wei_ptr_offset + i] = tmp[i];+ }+ return BTLA_CODE::Success;+}++template <BTLA_DTYPE _S3_T, typename _DST_T, int _PACK_ROW, typename _ST>+static inline BTLA_CODE decompress_kblock_bit3_packrow_fp(utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr,+ _DST_T* dstptr, int interleave_n_offset, int row, int col,+ _ST* scales, int8_t* zero_points, int k_offset, int kblock,+ int NPad, void* tmp, size_t tmpsize) {+ auto unpack_elt = row * col;+ decompress_kblock_s3_s8fp<_S3_T>(bit2ptr, bit1ptr, dstptr, interleave_n_offset, unpack_elt,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ // TODO(zhe): simd version+ for (int i = 0; i < row; i++) {+ int kpos = (k_offset + i) / kblock;+ auto sptr = scales + kpos * NPad;+ for (int j = 0; j < col; j++) {+ float tmp = static_cast<float>(dstptr[i * col + j]);+ if (zero_points != nullptr) tmp -= static_cast<float>(zero_points[kpos * NPad + j / _PACK_ROW]);+ dstptr[i * col + j] = static_cast<_DST_T>(tmp * sptr[j / _PACK_ROW]);+ }+ }++ return BTLA_CODE::Success;+}++template <BTLA_DTYPE _F4_T, typename _DST_T, int _PACK_ROW, typename _ST>+static inline BTLA_CODE decompress_kblock_f4_fp(utils::f4x2* srcptr, _DST_T* dstptr, int row, int col, int ld_src,+ int ld_dst, _ST* scales, int k_offset, int kblock, int NPad,+ int8_t* tmp, size_t tmpsize) {+ if constexpr (_PACK_ROW == 1) {+ return decompress_kblock_bit4_packrow1<_ST, _DST_T, true>(srcptr, dstptr, row, col, ld_src, ld_dst, scales, nullptr,+ k_offset, kblock, NPad, &dequant_f4_N<48, _DST_T, _F4_T>,+ pad_fp4, tmp, tmpsize);+ } else if constexpr (_PACK_ROW == 2) {+ return decompress_kblock_bit4_packrow2<_F4_T, _ST, _DST_T, true>(srcptr, dstptr, row, col, ld_src, ld_dst, scales,+ nullptr, k_offset, kblock, NPad, tmp, tmpsize);+ }+ return BTLA_CODE::NotSupport;+}++template <BTLA_DTYPE F4_T, typename DST_T>+inline BTLA_CODE decompress_kblock_f4_fp_noscale(utils::f4x2* srcptr, DST_T* dstptr, int row, int col, int ld_src,+ int ld_dst, int8_t* tmp, size_t tmpsize) {+ uint32_t mask = 0xf0f0f0f0;+ auto zmm_mask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ if (col == ld_src) {+ size_t elesize = (size_t)row * col;+ size_t ele256 = utils::padto_le(elesize, 256);+ size_t ele64 = utils::padto_le(elesize, 64);+ assert(tmpsize >= 256);+ size_t i = 0;+ constexpr int LoadMask64 = (1 << (64 / 8)) - 1;+ for (; i < ele256; i += 256) {+ pad_fp4(tmp + 0, reinterpret_cast<int8_t*>(srcptr + i / 2 + 0), zmm_mask, LoadMask64);+ pad_fp4(tmp + 64, reinterpret_cast<int8_t*>(srcptr + i / 2 + 32), zmm_mask, LoadMask64);+ pad_fp4(tmp + 128, reinterpret_cast<int8_t*>(srcptr + i / 2 + 64), zmm_mask, LoadMask64);+ pad_fp4(tmp + 192, reinterpret_cast<int8_t*>(srcptr + i / 2 + 96), zmm_mask, LoadMask64);+ for (size_t j = 0; j < 256; j += 64) {+ unpack_f4_N<64, DST_T, F4_T>(dstptr + i + j, tmp + j);+ }+ }+ if (i + 64 <= ele64) {+ for (; i < ele64; i += 64) {+ pad_fp4(tmp, reinterpret_cast<int8_t*>(srcptr + i / 2), zmm_mask, LoadMask64);+ unpack_f4_N<64, DST_T, F4_T>(dstptr + i, tmp);+ }+ }+ for (; i < elesize; i += 2) {+ auto tmp = srcptr[i / 2];+ dstptr[i + 0] = static_cast<DST_T>(ref::f4_unpack<F4_T>(tmp.x));+ dstptr[i + 1] = static_cast<DST_T>(ref::f4_unpack<F4_T>(tmp.y));+ }+ return BTLA_CODE::Success;+ }+ return BTLA_CODE::NotSupport;+}++static inline BTLA_CODE quantize_f32_sign_int_rowblock_sym(const float* srcptr, int8_t* dstptr, int row, int col,+ int ld_src, int ld_dst, float* scales, int blocksize) {+ int constexpr VLen = 16;+ auto v127 = _mm512_set1_ps(127.f);+ int col16 = utils::padto_le(col, 16);+ int i = 0;+ auto align_row = row / blocksize * blocksize;+ for (; i < col16; i += VLen) {+ int j = 0;+ auto simd_process_block = [&](int size) {+ __m512 vscale;+ __m512 vmaxval = _mm512_set1_ps(0.f);+ for (size_t ij = 0; ij < size; ij++) {+ auto vsrc = _mm512_loadu_ps(&srcptr[(j + ij) * ld_src + i]);+ vsrc = _mm512_abs_ps(vsrc);+ vmaxval = _mm512_max_ps(vmaxval, vsrc);+ }+ vscale = _mm512_div_ps(vmaxval, v127);+ auto vrscale = _mm512_div_ps(v127, vmaxval);+ _mm512_storeu_ps(&scales[j / blocksize * ld_dst + i], vscale);+ for (size_t ij = 0; ij < size; ij++) {+ auto vsrc = _mm512_loadu_ps(&srcptr[(j + ij) * ld_src + i]);+ vsrc = _mm512_mul_ps(vsrc, vrscale);+ auto vdsrc = _mm512_cvtps_epi32(vsrc);+ auto vbsrc = _mm512_cvtepi32_epi8(vdsrc);+ _mm_storeu_si128(reinterpret_cast<__m128i*>(&dstptr[(j + ij) * ld_dst + i]), vbsrc);+ }+ };+ for (; j < align_row; j += blocksize) simd_process_block(blocksize);+ if (j < row) simd_process_block(row - align_row);+ }+ for (; i < col; i++) {+ int j = 0;+ auto scalar_process_block = [&](int size) {+ float maxval = std::numeric_limits<float>::min();+ for (size_t ij = 0; ij < size; ij++) {+ maxval = std::max(maxval, std::abs(srcptr[(j + ij) * ld_src + i]));+ }+ float scale = maxval / 127;+ float rscale = 1.f / scale;+ scales[j / blocksize * ld_dst + i] = scale;+ for (size_t ij = 0; ij < size; ij++) {+ dstptr[(j + ij) * ld_dst + i] = utils::cast<float, int8_t>(srcptr[(j + ij) * ld_src + i] * rscale);+ }+ };+ for (; j < align_row; j += blocksize) scalar_process_block(blocksize);+ if (j < row) scalar_process_block(row - align_row);+ }+ return BTLA_CODE::Success;+}+static inline BTLA_CODE quantize_f32_sign_int_rowblock_sym_auto(const float* srcptr, int8_t* dstptr, int row, int col,+ int ld_src, int ld_dst, float* scales, int blocksize,+ BTLA_DTYPE qtype) {+ int constexpr VLen = 16;+ int col16 = utils::padto_le(col, VLen);+ int i = 0;+ auto NBits = utils::bestla_dtype_bits(qtype);+ int FullValue = 1 << (NBits - 1);+ int SymValue = FullValue - 1;+ auto align_row = row / blocksize * blocksize;+ for (; i < col16; i += VLen) {+ int j = 0;+ float tmp_min[VLen];+ float tmp_max[VLen];+ float tmp_abs[VLen];+ auto simd_process_block = [&](int size) {+ __m512 vscale;+ __m512 vmaxval = _mm512_set1_ps(std::numeric_limits<float>::min());+ __m512 vminval = _mm512_set1_ps(std::numeric_limits<float>::max());+ __m512 vabsval = _mm512_set1_ps(0.f);+ for (size_t ij = 0; ij < size; ij++) {+ auto vsrc = _mm512_loadu_ps(&srcptr[(j + ij) * ld_src + i]);+ vmaxval = _mm512_max_ps(vmaxval, vsrc);+ vminval = _mm512_min_ps(vminval, vsrc);+ vsrc = _mm512_abs_ps(vsrc);+ vabsval = _mm512_max_ps(vabsval, vsrc);+ }+ _mm512_storeu_ps(tmp_min, vminval);+ _mm512_storeu_ps(tmp_max, vmaxval);+ _mm512_storeu_ps(tmp_abs, vabsval);++ for (int iv = 0; iv < VLen; iv++) {+ int NVal = SymValue;+ auto sum = tmp_max[iv] + tmp_min[iv];+ if (abs(sum) >= tmp_abs[iv] / FullValue) {+ NVal = sum > 0.f ? -FullValue : FullValue;+ }+ NVal = NVal << (8 - NBits);+ tmp_abs[iv] = static_cast<float>(NVal);+ }+ auto vmag = _mm512_loadu_ps(tmp_abs);+ vscale = _mm512_div_ps(vabsval, vmag);+ auto vrscale = _mm512_div_ps(vmag, vabsval);+ _mm512_storeu_ps(&scales[j / blocksize * ld_dst + i], vscale);+ for (size_t ij = 0; ij < size; ij++) {+ auto vsrc = _mm512_loadu_ps(&srcptr[(j + ij) * ld_src + i]);+ vsrc = _mm512_mul_ps(vsrc, vrscale);+ auto vdsrc = _mm512_cvtps_epi32(vsrc);+ auto vbsrc = _mm512_cvtepi32_epi8(vdsrc);+ _mm_storeu_si128(reinterpret_cast<__m128i*>(&dstptr[(j + ij) * ld_dst + i]), vbsrc);+ }+ };+ for (; j < align_row; j += blocksize) simd_process_block(blocksize);+ if (j < row) simd_process_block(row - align_row);+ }+ kernel::ref::quantize_f32_sign_int_rowblock(srcptr + i, dstptr + i, row, col - i, ld_src, ld_dst, scales + i, nullptr,+ blocksize, qtype);+ return BTLA_CODE::Success;+}++static inline BTLA_CODE quantize_f32_sign_int_rowblock_asym(const float* srcptr, int8_t* dstptr, int row, int col,+ int ld_src, int ld_dst, float* scales, int8_t* zero_points,+ int blocksize) {+ int constexpr VLen = 16;+ auto v255 = _mm512_set1_ps(255.f);+ auto v2 = _mm512_set1_ps(2.f);+ auto v0 = _mm512_set1_ps(0.f);+ int col16 = utils::padto_le(col, 16);+ int i = 0;+ auto align_row = row / blocksize * blocksize;+ for (; i < col16; i += VLen) {+ int j = 0;+ auto simd_process_block = [&](int size) {+ __m512 vscale;+ __m512 vzp;+ __m512 vmaxval = v0;+ __m512 vminval = vmaxval;+ for (size_t ij = 0; ij < size; ij++) {+ auto vsrc = _mm512_loadu_ps(&srcptr[(j + ij) * ld_src + i]);+ vmaxval = _mm512_max_ps(vmaxval, vsrc);+ vminval = _mm512_min_ps(vminval, vsrc);+ }+ auto vsub = _mm512_sub_ps(vmaxval, vminval);+ vscale = _mm512_div_ps(vsub, v255);+ auto vrscale = _mm512_div_ps(v255, vsub);+ _mm512_storeu_ps(&scales[j / blocksize * ld_dst + i], vscale);+ auto vsum = _mm512_add_ps(vmaxval, vminval);+ auto vmedium = _mm512_div_ps(vsum, v2);+ vzp = _mm512_mul_ps(_mm512_sub_ps(v0, vmedium), vrscale);+ auto vbzp = _mm512_cvtsepi32_epi8(_mm512_cvtps_epi32(vzp));+ _mm_storeu_si128(reinterpret_cast<__m128i*>(&zero_points[j / blocksize * ld_dst + i]), vbzp);+ for (size_t ij = 0; ij < size; ij++) {+ auto vsrc = _mm512_loadu_ps(&srcptr[(j + ij) * ld_src + i]);+ vsrc = _mm512_mul_ps(_mm512_sub_ps(vsrc, vmedium), vrscale);+ auto vdsrc = _mm512_cvtps_epi32(vsrc);+ auto vbsrc = _mm512_cvtsepi32_epi8(vdsrc);+ _mm_storeu_si128(reinterpret_cast<__m128i*>(&dstptr[(j + ij) * ld_dst + i]), vbsrc);+ }+ };+ for (; j < align_row; j += blocksize) simd_process_block(blocksize);+ if (j < row) simd_process_block(row - align_row);+ }+ for (; i < col; i++) {+ int j = 0;+ auto scalar_process_block = [&](int size) {+ float maxval = 0;+ float minval = 0;+ for (size_t ij = 0; ij < size; ij++) {+ maxval = std::max(maxval, srcptr[(j + ij) * ld_src + i]);+ minval = std::min(maxval, srcptr[(j + ij) * ld_src + i]);+ }+ float scale = (maxval - minval) / 255.f;+ float rscale = 1.f / scale;+ scales[j / blocksize * ld_dst + i] = scale;+ float fmedium = (maxval + minval) / 2.f;+ int8_t bzp = utils::cast<float, int8_t>((0 - fmedium) * rscale);+ zero_points[j / blocksize * ld_dst + i] = bzp;+ for (size_t ij = 0; ij < size; ij++) {+ dstptr[(j + ij) * ld_dst + i] = utils::cast<float, int8_t>((srcptr[(j + ij) * ld_src + i] - fmedium) * rscale);+ }+ };+ for (; j < align_row; j += blocksize) scalar_process_block(blocksize);+ if (j < row) scalar_process_block(row - align_row);+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE quantize_f32_sign_int_rowblock(const float* srcptr, int8_t* dstptr, int row, int col,+ int ld_src, int ld_dst, float* scales, int8_t* zero_points,+ int blocksize, BTLA_DTYPE qtype) {+ if (zero_points == nullptr)+ if (qtype == BTLA_DTYPE::S4_CLIP || qtype == BTLA_DTYPE::S3_CLIP) {+ return quantize_f32_sign_int_rowblock_sym_auto(srcptr, dstptr, row, col, ld_src, ld_dst, scales, blocksize,+ qtype);+ } else {+ return quantize_f32_sign_int_rowblock_sym(srcptr, dstptr, row, col, ld_src, ld_dst, scales, blocksize);+ }+ else+ return quantize_f32_sign_int_rowblock_asym(srcptr, dstptr, row, col, ld_src, ld_dst, scales, zero_points,+ blocksize);+}++static float F4_NF4_quant_sub_helper[] = {0.f, 0.23746347f, 0.38810113f, 0.50841697f, 0.61348899f, 0.71018467f,+ 0.80257138f, 0.88788655f, 0.96835165f, 1.05161765f, 1.14011017f, 1.23740894f,+ 1.34975982f, 1.49088332f, 1.70957482f, 2.0f};+static float F4_BNB_quant_sub_helper[] = {0.00260417f, 0.0859375f, 0.20833333f, 0.29166667f,+ 0.4166667f, 0.583333f, 0.8333333f, 1.01f};+static float F4_E2M1_quant_sub_helper[] = {0.00520833f, 0.08854167f, 0.20833333f, 0.29166667f,+ 0.41666667f, 0.58333333f, 0.83333333f, 1.01f};+constexpr static int8_t F4_NF4_simd_quant_v[] = {0b0111, 0b0001, 0b0010, 0b0011, 0b0100, 0b0101, 0b0110, 0b0000,+ 0b1000, 0b1001, 0b1010, 0b1011, 0b1100, 0b1101, 0b1110, 0b1111};+constexpr static int8_t F4_BNB_simd_quant_v[] = {0b0000, 0b0001, 0b0110, 0b0111, 0b0100, 0b0101, 0b0010, 0b0011};+constexpr static int8_t F4_E2M1_simd_quant_v[] = {0b0000, 0b0001, 0b0010, 0b0011, 0b0100, 0b0101, 0b0110, 0b0111};++template <std::size_t N, std::size_t... I>+constexpr auto broadcast_N_2_Nx16(const int8_t* arr, std::index_sequence<I...>) {+ return std::array<int8_t, N * 16>{(arr[I / 16])...};+}++template <std::size_t N>+constexpr auto broadcast_N_2_Nx16(const int8_t* arr) {+ return broadcast_N_2_Nx16<N>(arr, std::make_index_sequence<N * 16>{});+}++template <BTLA_DTYPE F4_T>+inline void f32_f4_quantize_4x16(const float* srcptr, int8_t* dstptr, int ld_src, int ld_dst,+ const int8_t* broadcast_f4_v, float* scales, __mmask16 ls_mask) {+ __m128i xmm0{}, xmm1{}, xmm2{}, xmm3{};+ __m512 zmm0{}, zmm1{}, zmm2{}, zmm3{}, zmm4, zmm5, zmm6, zmm7, zmm_scale{};+ __mmask16 mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7;+ zmm_scale = _mm512_rcp14_ps(_mm512_mask_loadu_ps(zmm_scale, ls_mask, scales));+ auto avoid_double_cmp = _mm512_set1_ps(100.f);+ auto zmm_v0 = _mm512_set1_ps(0.f);+ zmm0 = _mm512_mask_loadu_ps(zmm0, ls_mask, srcptr);+ zmm1 = _mm512_mask_loadu_ps(zmm1, ls_mask, srcptr + 1 * ld_src);+ zmm2 = _mm512_mask_loadu_ps(zmm2, ls_mask, srcptr + 2 * ld_src);+ zmm3 = _mm512_mask_loadu_ps(zmm3, ls_mask, srcptr + 3 * ld_src);+ zmm0 = _mm512_mul_ps(zmm0, zmm_scale);+ zmm1 = _mm512_mul_ps(zmm1, zmm_scale);+ zmm2 = _mm512_mul_ps(zmm2, zmm_scale);+ zmm3 = _mm512_mul_ps(zmm3, zmm_scale);+ if constexpr (F4_T == BTLA_DTYPE::F4_NF4) {+ auto zmm_zp = _mm512_set1_ps(0.8480964004993439f);+ zmm0 = _mm512_add_ps(zmm0, zmm_zp);+ zmm1 = _mm512_add_ps(zmm1, zmm_zp);+ zmm2 = _mm512_add_ps(zmm2, zmm_zp);+ zmm3 = _mm512_add_ps(zmm3, zmm_zp);+ } else {+ mask4 = _mm512_cmp_ps_mask(zmm0, zmm_v0, 1);+ mask5 = _mm512_cmp_ps_mask(zmm1, zmm_v0, 1);+ mask6 = _mm512_cmp_ps_mask(zmm2, zmm_v0, 1);+ mask7 = _mm512_cmp_ps_mask(zmm3, zmm_v0, 1);++ zmm0 = _mm512_abs_ps(zmm0);+ zmm1 = _mm512_abs_ps(zmm1);+ zmm2 = _mm512_abs_ps(zmm2);+ zmm3 = _mm512_abs_ps(zmm3);+ }+ constexpr int loop_num = F4_T == BTLA_DTYPE::F4_NF4 ? 16 : 8;+ for (int i = 0; i < loop_num; i++) {+ __m512 sub_v;+ if constexpr (F4_T == BTLA_DTYPE::F4_NF4) sub_v = _mm512_set1_ps(F4_NF4_quant_sub_helper[i]);+ if constexpr (F4_T == BTLA_DTYPE::F4_BNB) sub_v = _mm512_set1_ps(F4_BNB_quant_sub_helper[i]);+ if constexpr (F4_T == BTLA_DTYPE::F4_E2M1) sub_v = _mm512_set1_ps(F4_E2M1_quant_sub_helper[i]);+ zmm4 = _mm512_sub_ps(zmm0, sub_v);+ zmm5 = _mm512_sub_ps(zmm1, sub_v);+ zmm6 = _mm512_sub_ps(zmm2, sub_v);+ zmm7 = _mm512_sub_ps(zmm3, sub_v);+ mask0 = _mm512_cmp_ps_mask(zmm4, zmm_v0, 2);+ mask1 = _mm512_cmp_ps_mask(zmm5, zmm_v0, 2);+ mask2 = _mm512_cmp_ps_mask(zmm6, zmm_v0, 2);+ mask3 = _mm512_cmp_ps_mask(zmm7, zmm_v0, 2);+ xmm0 = _mm_mask_blend_epi8(mask0, xmm0, _mm_loadu_si128(reinterpret_cast<const __m128i*>(broadcast_f4_v + i * 16)));+ xmm1 = _mm_mask_blend_epi8(mask1, xmm1, _mm_loadu_si128(reinterpret_cast<const __m128i*>(broadcast_f4_v + i * 16)));+ xmm2 = _mm_mask_blend_epi8(mask2, xmm2, _mm_loadu_si128(reinterpret_cast<const __m128i*>(broadcast_f4_v + i * 16)));+ xmm3 = _mm_mask_blend_epi8(mask3, xmm3, _mm_loadu_si128(reinterpret_cast<const __m128i*>(broadcast_f4_v + i * 16)));+ zmm0 = _mm512_mask_add_ps(zmm0, mask0, zmm0, avoid_double_cmp);+ zmm1 = _mm512_mask_add_ps(zmm1, mask1, zmm1, avoid_double_cmp);+ zmm2 = _mm512_mask_add_ps(zmm2, mask2, zmm2, avoid_double_cmp);+ zmm3 = _mm512_mask_add_ps(zmm3, mask3, zmm3, avoid_double_cmp);+ }+ if constexpr (F4_T != BTLA_DTYPE::F4_NF4) {+ auto xmm_bias = _mm_set1_epi8(0x08);+ xmm0 = _mm_mask_add_epi8(xmm0, mask4, xmm0, xmm_bias);+ xmm1 = _mm_mask_add_epi8(xmm1, mask5, xmm1, xmm_bias);+ xmm2 = _mm_mask_add_epi8(xmm2, mask6, xmm2, xmm_bias);+ xmm3 = _mm_mask_add_epi8(xmm3, mask7, xmm3, xmm_bias);+ }+ _mm_mask_storeu_epi8(dstptr, ls_mask, xmm0);+ _mm_mask_storeu_epi8(dstptr + 1 * ld_dst, ls_mask, xmm1);+ _mm_mask_storeu_epi8(dstptr + 2 * ld_dst, ls_mask, xmm2);+ _mm_mask_storeu_epi8(dstptr + 3 * ld_dst, ls_mask, xmm3);+}++template <BTLA_DTYPE F4_T>+inline void f32_f4_quantize_1x16(const float* srcptr, int8_t* dstptr, int ld_src, int ld_dst,+ const int8_t* broadcast_f4_v, float* scales, __mmask16 ls_mask) {+ __m512 zmm0{}, zmm1, zmm_scale{};+ zmm_scale = _mm512_rcp14_ps(_mm512_mask_loadu_ps(zmm_scale, ls_mask, scales));+ auto avoid_double_cmp = _mm512_set1_ps(100.f);+ auto zmm_v0 = _mm512_set1_ps(0.f);+ __m128i xmm0{};+ __mmask16 mask0, mask1;+ zmm0 = _mm512_mask_loadu_ps(zmm0, ls_mask, srcptr);+ zmm0 = _mm512_mul_ps(zmm0, zmm_scale);+ if constexpr (F4_T == BTLA_DTYPE::F4_NF4) {+ auto zp = _mm512_set1_ps(0.8480964004993439f);+ zmm0 = _mm512_add_ps(zmm0, zp);+ } else {+ mask1 = _mm512_cmp_ps_mask(zmm0, zmm_v0, 1);+ zmm0 = _mm512_abs_ps(zmm0);+ }+ constexpr int loop_num = F4_T == BTLA_DTYPE::F4_NF4 ? 16 : 8;+ for (int i = 0; i < loop_num; i++) {+ __m512 sub_v;+ if constexpr (F4_T == BTLA_DTYPE::F4_NF4) sub_v = _mm512_set1_ps(F4_NF4_quant_sub_helper[i]);+ if constexpr (F4_T == BTLA_DTYPE::F4_BNB) sub_v = _mm512_set1_ps(F4_BNB_quant_sub_helper[i]);+ if constexpr (F4_T == BTLA_DTYPE::F4_E2M1) sub_v = _mm512_set1_ps(F4_E2M1_quant_sub_helper[i]);+ zmm1 = _mm512_sub_ps(zmm0, sub_v);+ mask0 = _mm512_cmp_ps_mask(zmm1, zmm_v0, 2);+ xmm0 = _mm_mask_blend_epi8(mask0, xmm0, _mm_loadu_si128(reinterpret_cast<const __m128i*>(broadcast_f4_v + i * 16)));+ zmm0 = _mm512_mask_add_ps(zmm0, mask0, zmm0, avoid_double_cmp);+ }+ if constexpr (F4_T != BTLA_DTYPE::F4_NF4) {+ auto xmm_bias = _mm_set1_epi8(0x08);+ xmm0 = _mm_mask_add_epi8(xmm0, mask1, xmm0, xmm_bias);+ }+ _mm_mask_storeu_epi8(dstptr, ls_mask, xmm0);+}++inline void calc_blkx16_scale(const float* srcptr, int blocksize, int ld_src, float* scales, __mmask16 ls_mask) {+ auto absmax = _mm512_set1_ps(0.f);+ __m512 tmp{};+ for (int i = 0; i < blocksize; i++) {+ absmax = _mm512_range_ps(absmax, _mm512_mask_loadu_ps(tmp, ls_mask, srcptr + i * ld_src), 7);+ }+ _mm512_mask_storeu_ps(scales, ls_mask, absmax);+}++constexpr auto broadcast_F4_NF4_quantv = broadcast_N_2_Nx16<16>(F4_NF4_simd_quant_v);+constexpr auto broadcast_F4_BNB_quantv = broadcast_N_2_Nx16<8>(F4_BNB_simd_quant_v);+constexpr auto broadcast_F4_E2M1_quantv = broadcast_N_2_Nx16<8>(F4_E2M1_simd_quant_v);++template <BTLA_DTYPE F4_T>+inline BTLA_CODE quantize_f32_f4_rowblock(const float* srcptr, int8_t* dstptr, int row, int col, int ld_src, int ld_dst,+ float* scales, int blocksize) {+ // assert(col % 16 == 0);+ auto align_row = row / blocksize * blocksize;+ auto align_blk = blocksize / 4 * 4;+ int8_t* broadcast_f4_quantv;+ if constexpr (F4_T == BTLA_DTYPE::F4_NF4) broadcast_f4_quantv = const_cast<int8_t*>(broadcast_F4_NF4_quantv.data());+ if constexpr (F4_T == BTLA_DTYPE::F4_BNB) broadcast_f4_quantv = const_cast<int8_t*>(broadcast_F4_BNB_quantv.data());+ if constexpr (F4_T == BTLA_DTYPE::F4_E2M1) broadcast_f4_quantv = const_cast<int8_t*>(broadcast_F4_E2M1_quantv.data());+ int i = 0;+ int align_col = col / 16 * 16;++ auto process_row_blk = [&](int i, int col_size) {+ int j = 0;+ __mmask16 ls_mask = _cvtu32_mask16(0xffff >> (16 - col_size));+ for (; j < align_row; j += blocksize) {+ calc_blkx16_scale(srcptr + j * ld_src + i, blocksize, ld_src, scales + j / blocksize * ld_dst + i, ls_mask);+ int k = 0;+ for (; k < align_blk; k += 4) {+ f32_f4_quantize_4x16<F4_T>(srcptr + (j + k) * ld_src + i, dstptr + (j + k) * ld_dst + i, ld_src, ld_dst,+ broadcast_f4_quantv, scales + j / blocksize * ld_dst + i, ls_mask);+ }+ for (; k < blocksize; k++) {+ f32_f4_quantize_1x16<F4_T>(srcptr + (j + k) * ld_src + i, dstptr + (j + k) * ld_dst + i, ld_src, ld_dst,+ broadcast_f4_quantv, scales + j / blocksize * ld_dst + i, ls_mask);+ }+ }+ if (j < row) {+ auto fin_row = row - align_row;+ calc_blkx16_scale(srcptr + j * ld_src + i, fin_row, ld_src, scales + j / blocksize * ld_dst + i, ls_mask);+ int k = 0;+ auto align_fin_blk = fin_row / 4 * 4;+ for (; k < align_fin_blk; k += 4) {+ f32_f4_quantize_4x16<F4_T>(srcptr + (j + k) * ld_src + i, dstptr + (j + k) * ld_dst + i, ld_src, ld_dst,+ broadcast_f4_quantv, scales + j / blocksize * ld_dst + i, ls_mask);+ }+ for (; k < fin_row; k++) {+ f32_f4_quantize_1x16<F4_T>(srcptr + (j + k) * ld_src + i, dstptr + (j + k) * ld_dst + i, ld_src, ld_dst,+ broadcast_f4_quantv, scales + j / blocksize * ld_dst + i, ls_mask);+ }+ }+ };++ for (; i < align_col; i += 16) process_row_blk(i, 16);+ if (i < col) process_row_blk(i, col - i);++ return BTLA_CODE::Success;+}++template <typename SRC_T>+static inline BTLA_CODE quantize_fp_u8_colblock(int row, int col, const SRC_T* srcptr, int ld_src, uint8_t* dstptr,+ int ld_dst, float* scales, int ld_scale, uint8_t* zps, int blocksize,+ float* blkreduce) {+ int constexpr VLen = 16;+ int constexpr Unroll = 2;+ auto vff = _mm512_set1_epi32(255);+ auto v0 = _mm512_set1_epi32(0);+ int vblocksize = utils::padto_le(blocksize, VLen);+ int vblocksize_un = utils::padto_le(blocksize, VLen * Unroll);+ int colblk = utils::padto_le(col, blocksize);+ for (size_t i = 0; i < row; i += 1) {+ size_t j = 0;+ for (; j < colblk; j += blocksize) {+ __m512 vmaxval = _mm512_set1_ps(0.f);+ __m512 vminval = _mm512_set1_ps(0.f);+ size_t ij = 0;+ for (; ij < vblocksize_un; ij += VLen * Unroll) {+ for (size_t iu = 0; iu < Unroll; iu++) {+ __m512 vsrc = load_T_fp32(srcptr + j + ij + i * ld_src + iu * VLen);+ vmaxval = _mm512_max_ps(vmaxval, vsrc);+ vminval = _mm512_min_ps(vminval, vsrc);+ }+ }+ if (ij + VLen < vblocksize) {+ for (; ij < vblocksize; ij += VLen) {+ __m512 vsrc = load_T_fp32(srcptr + j + ij + i * ld_src);+ vmaxval = _mm512_max_ps(vmaxval, vsrc);+ vminval = _mm512_min_ps(vminval, vsrc);+ }+ }++ auto maxval = _mm512_reduce_max_ps(vmaxval);+ auto minval = _mm512_reduce_min_ps(vminval);+ if (ij < blocksize) {+ for (; ij < blocksize; ij++) {+ auto srcval = static_cast<float>(srcptr[(j + ij) + i * ld_src]);+ maxval = std::max(maxval, srcval);+ minval = std::min(minval, srcval);+ }+ }+ float scale = (maxval - minval) / 255;+ uint8_t zp = utils::cast<float, uint8_t>((0 - minval) / scale);+ scales[j / blocksize + i * ld_scale] = scale;+ zps[j / blocksize + i * ld_scale] = zp;+ float rscale = 1.f / scale;+ auto vrscale = _mm512_set1_ps(rscale);+ auto vdzp = _mm512_set1_epi32(zp);+ int sum = 0;+ ij = 0;+ for (; ij < vblocksize_un; ij += VLen * Unroll) {+ for (size_t iu = 0; iu < Unroll; iu++) {+ __m512 vsrc = load_T_fp32(srcptr + j + ij + i * ld_src + iu * VLen);+ vsrc = _mm512_mul_ps(vsrc, vrscale);+ auto vdsrc = _mm512_cvtps_epi32(vsrc);+ if (blkreduce) {+ sum += _mm512_reduce_add_epi32(vdsrc);+ }+ vdsrc = _mm512_add_epi32(vdsrc, vdzp);+ vdsrc = _mm512_min_epi32(vdsrc, vff);+ vdsrc = _mm512_max_epi32(vdsrc, v0);+ auto vbsrc = _mm512_cvtepi32_epi8(vdsrc);+ _mm_storeu_si128(reinterpret_cast<__m128i*>(&dstptr[(j + ij) + i * ld_dst + iu * VLen]), vbsrc);+ }+ }+ if (ij + VLen < vblocksize) {+ for (; ij < vblocksize; ij += VLen) {+ __m512 vsrc = load_T_fp32(srcptr + j + ij + i * ld_src);+ vsrc = _mm512_mul_ps(vsrc, vrscale);+ auto vdsrc = _mm512_cvtps_epi32(vsrc);+ if (blkreduce) {+ sum += _mm512_reduce_add_epi32(vdsrc);+ }+ vdsrc = _mm512_add_epi32(vdsrc, vdzp);+ vdsrc = _mm512_min_epi32(vdsrc, vff);+ vdsrc = _mm512_max_epi32(vdsrc, v0);+ auto vbsrc = _mm512_cvtepi32_epi8(vdsrc);+ _mm_storeu_si128(reinterpret_cast<__m128i*>(&dstptr[(j + ij) + i * ld_dst]), vbsrc);+ }+ }+ for (; ij < blocksize; ij++) {+ auto srcval = static_cast<float>(srcptr[(j + ij) + i * ld_src]);+ srcval = srcval * rscale;+ auto srcint = utils::cast<float, int>(srcval);+ sum += srcint;+ srcint += zp;+ srcint = std::min(srcint, 0xff);+ srcint = std::max(srcint, 0);+ dstptr[(j + ij) + i * ld_dst] = static_cast<uint8_t>(srcint);+ }+ if (blkreduce) {+ blkreduce[j / blocksize + i * ld_scale] = sum * scale;+ }+ }++ if (j < col) {+ float maxval = 0.f;+ float minval = 0.f;+ for (size_t ij = j; ij < col; ij++) {+ auto fsrc = static_cast<float>(srcptr[ij + i * ld_src]);+ maxval = std::max(fsrc, maxval);+ minval = std::min(fsrc, minval);+ }+ float scale = (maxval - minval) / 255;+ uint8_t zp = utils::cast<float, uint8_t>((0 - minval) / scale);+ float rscale = 1.f / scale;+ scales[j / blocksize + i * ld_scale] = scale;+ zps[j / blocksize + i * ld_scale] = zp;+ int sum = 0;+ for (size_t ij = j; ij < col; ij++) {+ auto fsrc = static_cast<float>(srcptr[ij + i * ld_src]);+ auto srcint = utils::cast<float, int>(fsrc * rscale);+ sum += srcint;+ srcint += zp;+ srcint = srcint <= 255 ? srcint : 255;+ srcint = srcint >= 0 ? srcint : 0;+ dstptr[ij + i * ld_dst] = srcint;+ }+ if (blkreduce) {+ blkreduce[j / blocksize + i * ld_scale] = sum * scale;+ }+ }+ }+ return BTLA_CODE::Success;+}++template <typename SRC_T>+static inline BTLA_CODE quantize_fp_s8_colblock(int row, int col, const SRC_T* srcptr, int ld_src, int8_t* dstptr,+ int ld_dst, float* scales, int ld_scale, int blocksize, float* reduce) {+ int constexpr VLen = 16;+ auto vpos = _mm512_set1_epi32(127);+ auto vneg = _mm512_set1_epi32(-128);+ int VBlockSizeU3 = utils::padto_le(blocksize, VLen * 3);+ int VBlockSize = utils::padto_le(blocksize, VLen);+ int colblk = utils::padto_le(col, blocksize);+ for (int i = 0; i < row; i += 1) {+ size_t j = 0;+ for (; j < colblk; j += blocksize) {+ __m512 vmaxval = _mm512_set1_ps(std::numeric_limits<float>::min());+ size_t ij = 0;+ for (; ij < VBlockSizeU3; ij += VLen * 3) {+ for (int iu = 0; iu < 3; iu++) {+ __m512 vsrc = load_T_fp32(srcptr + j + ij + i * ld_src + iu * VLen);+ vsrc = _mm512_abs_ps(vsrc);+ vmaxval = _mm512_max_ps(vmaxval, vsrc);+ }+ }+ if (ij + VLen < VBlockSize) {+ for (; ij < VBlockSize; ij += VLen) {+ __m512 vsrc = load_T_fp32(srcptr + j + ij + i * ld_src);+ vsrc = _mm512_abs_ps(vsrc);+ vmaxval = _mm512_max_ps(vmaxval, vsrc);+ }+ }++ auto maxval = _mm512_reduce_max_ps(vmaxval);+ if (ij < blocksize) {+ for (; ij < blocksize; ij++) {+ auto srcval = std::abs(static_cast<float>(srcptr[(j + ij) + i * ld_src]));+ maxval = std::max(maxval, srcval);+ }+ }+ float scale = maxval / 127;+ scales[j / blocksize + i * ld_scale] = scale;+ float rscale = 1.f / scale;+ auto vrscale = _mm512_set1_ps(rscale);+ ij = 0;+ int sum = 0;+ for (; ij < VBlockSizeU3; ij += VLen * 3) {+ for (int iu = 0; iu < 3; iu++) {+ __m512 vsrc = load_T_fp32(srcptr + j + ij + i * ld_src + iu * VLen);+ vsrc = _mm512_mul_ps(vsrc, vrscale);+ auto vdsrc = _mm512_cvtps_epi32(vsrc);+ sum += _mm512_reduce_add_epi32(vdsrc);+ vdsrc = _mm512_min_epi32(vdsrc, vpos);+ vdsrc = _mm512_max_epi32(vdsrc, vneg);+ auto vbsrc = _mm512_cvtepi32_epi8(vdsrc);+ _mm_storeu_si128(reinterpret_cast<__m128i*>(&dstptr[(j + ij) + i * ld_dst + iu * VLen]), vbsrc);+ }+ }+ if (ij + VLen < VBlockSize) {+ for (; ij < VBlockSize; ij += VLen) {+ __m512 vsrc = load_T_fp32(srcptr + j + ij + i * ld_src);+ vsrc = _mm512_mul_ps(vsrc, vrscale);+ auto vdsrc = _mm512_cvtps_epi32(vsrc);+ sum += _mm512_reduce_add_epi32(vdsrc);+ vdsrc = _mm512_min_epi32(vdsrc, vpos);+ vdsrc = _mm512_max_epi32(vdsrc, vneg);+ auto vbsrc = _mm512_cvtepi32_epi8(vdsrc);+ _mm_storeu_si128(reinterpret_cast<__m128i*>(&dstptr[(j + ij) + i * ld_dst]), vbsrc);+ }+ }+ if (ij < blocksize) {+ for (; ij < blocksize; ij++) {+ auto srcval = static_cast<float>(srcptr[(j + ij) + i * ld_src]);+ srcval = srcval * rscale;+ auto srcint = int(roundf(srcval));+ sum += srcint;+ srcint = std::min(srcint, 127);+ srcint = std::max(srcint, -127);+ dstptr[(j + ij) + i * ld_dst] = static_cast<uint8_t>(srcint);+ }+ }+ if (reduce) reduce[j / blocksize + i * ld_scale] = sum * scale;+ }+ if (j < col) {+ float absmaxval = std::numeric_limits<float>::min();+ for (size_t ij = j; ij < col; ij++) {+ absmaxval = std::max(std::abs((float)srcptr[(ij) + i * ld_src]), absmaxval);+ }+ float scale = absmaxval / 127;+ float rscale = 1.f / scale;+ scales[j / blocksize + i * ld_scale] = scale;+ int sum = 0;+ for (size_t ij = j; ij < col; ij++) {+ dstptr[(ij) + i * ld_dst] = utils::cast<float, int8_t>((float)srcptr[(ij) + i * ld_src] * rscale);+ sum += dstptr[(ij) + i * ld_dst];+ }+ if (reduce) reduce[j / blocksize + i * ld_scale] = sum * scale;+ }+ }+ return BTLA_CODE::Success;+}++inline BTLA_CODE dq8_get_fp_scale(uint8_t* src, float* dst, int row, int col, int scale_offset, int dq_blk,+ int dq_offset_idx, float* dq_scale, int src_stride, int dst_stride, bool zeropadding,+ int mN) {+ auto head_proc_num = utils::updiv(scale_offset, 16) * 16 - scale_offset;+ auto zmm_dq_offset = _mm512_set1_ps(dq_scale[dq_offset_idx]);++ auto get_fp_scale = [&](int proc_src_num, __mmask16 mask, int scale_offset, uint8_t* src, float* dst) {+ auto dq_s_idx = scale_offset / dq_blk;+ auto zmm_dq_scale = _mm512_set1_ps(dq_scale[dq_s_idx]);+ float tmp[16];+ for (int i = 0; i < proc_src_num; i++) tmp[i] = dq8_bnb_LUT[src[i]];+ __m512 fp32_dq_zmm = _mm512_loadu_ps(tmp);+ auto fzmm = _mm512_mul_ps(fp32_dq_zmm, zmm_dq_scale);+ fzmm = _mm512_add_ps(fzmm, zmm_dq_offset);+ _mm512_mask_storeu_ps(dst, mask, fzmm);+ };++ for (int i = 0; i < row; i++) {+ if (head_proc_num > col) {+ auto mask = _cvtu32_mask16(0xffff >> (16 - col));+ get_fp_scale(col, mask, scale_offset + i * mN, src + i * src_stride, dst + i * dst_stride);+ } else {+ // TODO(zhe): consider head_proc_num==0 case.+ auto head_mask = _cvtu32_mask16(0xffff >> (16 - head_proc_num));+ auto body_mask = _cvtu32_mask16(0xffff);+ get_fp_scale(head_proc_num, head_mask, scale_offset + i * mN, src + i * src_stride, dst + i * dst_stride);+ auto scale_offset_iter = scale_offset + i * mN + head_proc_num;+ uint8_t* src_iter_ptr = src + head_proc_num;+ float* dst_iter_ptr = dst + head_proc_num;+ auto body_loop = (col - head_proc_num) / 16;+ auto tail_proc_num = (col - head_proc_num) % 16;+ int ii = 0;+ for (; ii < body_loop; ii++) {+ get_fp_scale(16, body_mask, scale_offset_iter + ii * 16, src_iter_ptr + i * src_stride + ii * 16,+ dst_iter_ptr + i * dst_stride + ii * 16);+ }+ if (tail_proc_num > 0) {+ auto tail_mask = _cvtu32_mask16(0xffff >> (16 - tail_proc_num));+ get_fp_scale(tail_proc_num, tail_mask, scale_offset_iter + ii * 16, src_iter_ptr + i * src_stride + ii * 16,+ dst_iter_ptr + i * dst_stride + ii * 16);+ }+ }+ }+ if (zeropadding) assert(0);+ return BTLA_CODE::Success;+}++static inline BTLA_CODE alphabeta_f32_f32(const float alpha, const float* srcptr, const int srcstep, const float beta,+ const float* src1ptr, const int src1step, float* dstptr, const int dststep,+ const int M, const int N) {+ int constexpr Vlen = 16;+ auto vN = utils::padto_le(N, Vlen);+ auto valpha = _mm512_set1_ps(alpha);+ auto vbeta = _mm512_set1_ps(beta);++ for (int i = 0; i < M; i++) {+ int j = 0;+ if (beta != 0.f) {+ for (; j < vN; j += Vlen) {+ auto vsrc = _mm512_loadu_ps(srcptr + i * srcstep + j);+ auto vsrc1 = _mm512_loadu_ps(src1ptr + i * src1step + j);+ auto vdst = _mm512_mul_ps(valpha, vsrc);+ vdst = _mm512_fmadd_ps(vbeta, vsrc1, vdst);+ _mm512_storeu_ps(dstptr + i * dststep + j, vdst);+ }+ for (; j < N; j += 1) {+ dstptr[i * dststep + j] = alpha * srcptr[i * srcstep + j] + beta * src1ptr[i * src1step + j];+ }+ } else {+ for (; j < vN; j += Vlen) {+ auto vsrc = _mm512_loadu_ps(srcptr + i * srcstep + j);+ auto vdst = _mm512_mul_ps(valpha, vsrc);+ _mm512_storeu_ps(dstptr + i * dststep + j, vdst);+ }+ for (; j < N; j += 1) {+ dstptr[i * dststep + j] = alpha * srcptr[i * srcstep + j];+ }+ }+ }+ return BTLA_CODE::Success;+}++template <typename SCA_T>+static inline BTLA_CODE accum_alphaN_f32_f32(const SCA_T* alpha, const float* srcptr, const int srcstep, float* dstptr,+ const int dststep, const int M, const int N) {+ int constexpr Vlen = 16;+ auto vN = utils::padto_le(N, Vlen);+ int j = 0;+ for (; j < vN; j += Vlen) {+ __m512 valpha;+ if constexpr (std::is_same_v<SCA_T, float>) {+ valpha = _mm512_loadu_ps(alpha + j);+ } else if constexpr (std::is_same_v<SCA_T, utils::bf16>) {+ auto tmp = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(alpha + j));+ valpha = zmm_cvt_bf16_fp32(tmp);+ } else if constexpr (std::is_same_v<SCA_T, utils::f8>) {+ valpha = _mm512_scalef_ps(+ _mm512_set1_ps(1),+ _mm512_cvtepi32_ps(_mm512_cvtepi8_epi32(_mm_loadu_si128(reinterpret_cast<const __m128i*>(alpha + j)))));+ }+ for (size_t i = 0; i < M; i++) {+ auto vsrc = _mm512_loadu_ps(srcptr + i * srcstep + j);+ auto vsrc1 = _mm512_loadu_ps(dstptr + i * dststep + j);+ auto vdst = _mm512_fmadd_ps(valpha, vsrc, vsrc1);+ _mm512_storeu_ps(dstptr + i * dststep + j, vdst);+ }+ }+ for (; j < N; j += 1) {+ for (size_t i = 0; i < M; i++) {+ if constexpr (!std::is_same_v<SCA_T, utils::f8>) {+ dstptr[i * dststep + j] += static_cast<float>(alpha[j]) * srcptr[i * srcstep + j];+ } else {+ dstptr[i * dststep + j] += alpha[j].mul(srcptr[i * srcstep + j]);+ }+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE accum_f32_f32(const float* srcptr, const int srcstep, float* dstptr, const int dststep,+ const int M, const int N) {+ int constexpr Vlen = 16;+ auto vN = utils::padto_le(N, Vlen);+ int j = 0;+ for (; j < vN; j += Vlen) {+ for (size_t i = 0; i < M; i++) {+ auto vsrc = _mm512_loadu_ps(srcptr + i * srcstep + j);+ auto vsrc1 = _mm512_loadu_ps(dstptr + i * dststep + j);+ auto vdst = _mm512_add_ps(vsrc, vsrc1);+ _mm512_storeu_ps(dstptr + i * dststep + j, vdst);+ }+ }+ for (; j < N; j += 1) {+ for (size_t i = 0; i < M; i++) {+ dstptr[i * dststep + j] += srcptr[i * srcstep + j];+ }+ }+ return BTLA_CODE::Success;+}++static inline void vec_quanout_s32_u32_v16(const int32_t* srcptr, __m512& vfactor, __m512i& vzp, __m512i& vzeros,+ __m512i& v255, uint8_t* dstptr) {+ auto vsrcd = _mm512_loadu_si512(srcptr);+ auto vsrcf = _mm512_mul_ps(vfactor, _mm512_cvtepi32_ps(vsrcd));+ vsrcd = _mm512_cvtps_epi32(vsrcf);+ vsrcd = _mm512_add_epi32(vsrcd, vzp);+ vsrcd = _mm512_max_epi32(vsrcd, vzeros);+ vsrcd = _mm512_min_epi32(vsrcd, v255);+ auto vdstb = _mm512_cvtepi32_epi8(vsrcd);+ _mm_storeu_si128(reinterpret_cast<__m128i*>(dstptr), vdstb);+}++static inline BTLA_CODE quanout_s32_u32(const float alpha, const int32_t* srcptr, const int srcstep, uint8_t* dstptr,+ const int dststep, const int M, const int N, float scaleSrc, float scaleDst,+ int zpDst) {+ float factor = alpha * scaleSrc / scaleDst;+ auto vfactor = _mm512_set1_ps(factor);+ auto vzp = _mm512_set1_epi32(zpDst);+ auto vzeros = _mm512_set1_epi32(0);+ auto v255 = _mm512_set1_epi32(255);+ int N64 = utils::padto_le(N, 64);+ int N48 = utils::padto_le(N, 48);+ int N16 = utils::padto_le(N, 16);+ for (int i = 0; i < M; i++) {+ int j = 0;+ for (; j < N64; j += 64) {+ for (int iv = 0; iv < 4; iv++) {+ vec_quanout_s32_u32_v16(&srcptr[i * srcstep + j + iv * 16], vfactor, vzp, vzeros, v255,+ &dstptr[i * dststep + j + iv * 16]);+ }+ }+ if (N48 - j >= 48) {+ for (; j < N48; j += 48) {+ for (int iv = 0; iv < 3; iv++) {+ vec_quanout_s32_u32_v16(&srcptr[i * srcstep + j + iv * 16], vfactor, vzp, vzeros, v255,+ &dstptr[i * dststep + j + iv * 16]);+ }+ }+ }+ if (N16 - j >= 16) {+ for (; j < N16; j += 16) {+ vec_quanout_s32_u32_v16(&srcptr[i * srcstep + j], vfactor, vzp, vzeros, v255, &dstptr[i * dststep + j]);+ }+ }+ for (; j < N; j++) {+ float fsrc = static_cast<float>(srcptr[i * srcstep + j]) * factor;+ dstptr[i * dststep + j] = utils::cast<float, uint8_t>(fsrc + static_cast<float>(zpDst));+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE accumulate_dequantize_s32_f32(const int32_t* srcptr, float* dstptr, float alpha, float beta,+ int row, int col, int ld_src, int ld_dst, float* ascales,+ int ldas, float* wscales) {+ auto vbeta = _mm512_set1_ps(beta);+ int col16 = utils::padto_le(col, 16);+ for (int irow = 0; irow < row; irow++) {+ auto scale = ascales[irow * ldas] * alpha;+ auto valpha = _mm512_set1_ps(scale);+ int icol = 0;+ for (; icol < col16; icol += 16) {+ auto vwscale = _mm512_loadu_ps(wscales + icol);+ auto vscale = _mm512_mul_ps(valpha, vwscale);+ auto vdst = _mm512_loadu_ps(dstptr + irow * ld_dst + icol);+ vdst = _mm512_mul_ps(vdst, vbeta);+ auto vsrcd = _mm512_loadu_si512(srcptr + irow * ld_src + icol);+ auto vsrc = _mm512_cvtepi32_ps(vsrcd);+ vsrc = _mm512_fmadd_ps(vsrc, vscale, vdst);+ _mm512_storeu_ps(dstptr + irow * ld_dst + icol, vsrc);+ }+ for (; icol < col; icol += 1) {+ dstptr[irow * ld_dst + icol] =+ scale * wscales[icol] * srcptr[irow * ld_src + icol] + beta * dstptr[irow * ld_dst + icol];+ }+ }+ return BTLA_CODE::Success;+}++template <typename SCAB_T, int NTILE>+static inline BTLA_CODE dequant_s32_fp32_N(const int32_t* srcptr, const int srcstep, float* dstptr, const int dststep,+ const int row, const float* scaleA, const int ldsa, const SCAB_T* scaleB) {+ int constexpr VLen = 16;+ static_assert(NTILE % VLen == 0);+ int constexpr NRegs = NTILE / VLen;+ __m512 vwscale[NRegs];+ for (int ic = 0; ic < NRegs; ic++) {+ vwscale[ic] = load_T_fp32(scaleB + ic * VLen);+ }+ for (int irow = 0; irow < row; irow++) {+ auto scale = scaleA[irow * ldsa];+ auto valpha = _mm512_set1_ps(scale);+ for (int ic = 0; ic < NRegs; ic++) {+ auto vscale = _mm512_mul_ps(valpha, vwscale[ic]);+ auto vsrcd = _mm512_loadu_si512(srcptr + irow * srcstep + ic * VLen);+ auto vsrc = _mm512_cvtepi32_ps(vsrcd);+ vsrc = _mm512_mul_ps(vsrc, vscale);+ _mm512_storeu_ps(dstptr + irow * dststep + ic * VLen, vsrc);+ }+ }+ return BTLA_CODE::Success;+}++template <typename SCAB_T>+static inline BTLA_CODE dequant_s32_fp32(const int32_t* srcptr, const int srcstep, float* dstptr, const int dststep,+ const int row, const int col, const float* scaleA, const int ldsa,+ const SCAB_T* scaleB) {+ int col16 = utils::padto_le(col, 16);+ int col48 = utils::padto_le(col, 48);+ int col64 = utils::padto_le(col, 64);+ int icol = 0;+ for (; icol < col64; icol += 64) {+ dequant_s32_fp32_N<SCAB_T, 64>(srcptr + icol, srcstep, dstptr + icol, dststep, row, scaleA, ldsa, scaleB + icol);+ }+ if (icol + 48 <= col48) {+ for (; icol < col48; icol += 48) {+ dequant_s32_fp32_N<SCAB_T, 48>(srcptr + icol, srcstep, dstptr + icol, dststep, row, scaleA, ldsa, scaleB + icol);+ }+ }+ if (icol + 16 <= col16) {+ for (; icol < col16; icol += 16) {+ dequant_s32_fp32_N<SCAB_T, 16>(srcptr + icol, srcstep, dstptr + icol, dststep, row, scaleA, ldsa, scaleB + icol);+ }+ }+ for (; icol < col; icol += 1) {+ for (int irow = 0; irow < row; irow++) {+ auto scale = scaleA[irow * ldsa];+ dstptr[irow * dststep + icol] = scale * float(scaleB[icol]) * srcptr[irow * srcstep + icol];+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE broadcast_u8(int num, const uint8_t& srcval, uint8_t* dstptr) {+ int i = 0;+ int constexpr VN = 64 / sizeof(srcval);+ int numv = utils::padto_le(num, VN);+ auto vsrc = _mm512_set1_epi8(srcval);+ for (; i < numv; i += VN) {+ _mm512_storeu_si512(dstptr + i, vsrc);+ }+ int num32 = utils::padto_le(num, 32);+ if (i + 32 <= num32) {+ for (; i < num32; i += 32) {+ _mm256_storeu_si256(reinterpret_cast<__m256i*>(dstptr + i), _mm512_castsi512_si256(vsrc));+ }+ }+ for (; i < num; i++) {+ dstptr[i] = srcval;+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE remove_act_zeropoint_bias(float* accptr, int ldacc, int row, int col, uint8_t* zps,+ float* scales, int lds, const float* reduce) {+ int constexpr VLen = 16;+ auto col16 = utils::padto_le(col, VLen);+ for (int i = 0; i < row; i++) {+ auto zpf = static_cast<float>(zps[i * lds]) * scales[i * lds];+ int j = 0;+ auto vzp = _mm512_set1_ps(-zpf);+ for (; j < col16; j += VLen) {+ auto vreduce = _mm512_loadu_ps(reduce + j);+ auto vacc = _mm512_loadu_ps(&accptr[i * ldacc + j]);+ vacc = _mm512_fmadd_ps(vzp, vreduce, vacc);+ _mm512_storeu_ps(&accptr[i * ldacc + j], vacc);+ }+ if (j < col) {+ for (; j < col; j++) {+ accptr[i * ldacc + j] -= zpf * reduce[j];+ }+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE remove_wei_zeropoint_bias(float* accptr, int ldacc, int row, int col, int8_t* zps,+ float* scales, int lds, const float* reduce) {+ int constexpr VLen = 16;+ auto col16 = utils::padto_le(col, VLen);+ for (int i = 0; i < row; i++) {+ auto vreduce = _mm512_set1_ps(-reduce[i * lds]);+ int j = 0;+ for (; j < col16; j += VLen) {+ auto vzp_s32 = _mm512_cvtepi8_epi32(_mm_loadu_si128(reinterpret_cast<__m128i*>(zps + j)));+ auto vzp_f32 = _mm512_cvtepi32_ps(vzp_s32);+ auto vzp = _mm512_mul_ps(vzp_f32, _mm512_loadu_ps(scales + j));+ auto vacc = _mm512_loadu_ps(&accptr[i * ldacc + j]);+ vacc = _mm512_fmadd_ps(vzp, vreduce, vacc);+ _mm512_storeu_ps(&accptr[i * ldacc + j], vacc);+ }+ if (j < col) {+ for (; j < col; j++) {+ accptr[i * ldacc + j] -= static_cast<float>(zps[j]) * scales[j] * reduce[i * lds];+ }+ }+ }+ return BTLA_CODE::Success;+}++template <bool ZPA, bool ZPB>+static inline BTLA_CODE remove_zeropoint_bias(int32_t* accptr, int ldacc, int row, int col, uint8_t* zpa, int8_t* zpb,+ int lda, int k, const int32_t* reducea, const int32_t* reduceb) {+ int constexpr VLen = 16;+ auto col16 = utils::padto_le(col, VLen);+ auto vk = _mm512_set1_epi32(k);+ for (int i = 0; i < row; i++) {+ auto vreducea = ZPB ? _mm512_set1_epi32(reducea[i * lda]) : _mm512_set1_epi32(0);+ auto vzpa = ZPA ? _mm512_set1_epi32(int(zpa[i * lda])) : _mm512_set1_epi32(0);+ int j = 0;+ for (; j < col16; j += VLen) {+ auto vzpb = ZPB ? load_s8_s32(zpb + j) : _mm512_set1_epi32(0);+ auto vreduceb = ZPA ? _mm512_loadu_epi32(reduceb + j) : _mm512_set1_epi32(0);+ auto vacc = _mm512_loadu_epi32(&accptr[i * ldacc + j]);+ __m512i tmp;+ if constexpr (ZPA) {+ tmp = _mm512_mullo_epi32(vzpa, vreduceb);+ vacc = _mm512_sub_epi32(vacc, tmp);+ }+ if constexpr (ZPB) {+ tmp = _mm512_mullo_epi32(vzpb, vreducea);+ vacc = _mm512_sub_epi32(vacc, tmp);+ }+ if constexpr (ZPA && ZPB) {+ tmp = _mm512_mullo_epi32(vzpb, vk);+ tmp = _mm512_mullo_epi32(tmp, vzpa);+ vacc = _mm512_sub_epi32(vacc, tmp);+ }+ _mm512_storeu_epi32(&accptr[i * ldacc + j], vacc);+ }+ if (j < col) {+ int32_t vreducea = ZPB ? reducea[i * lda] : 0;+ uint8_t vzpa = ZPA ? zpa[i * lda] : 0;+ for (; j < col; j++) {+ auto vzpb = ZPB ? zpb[j] : 0;+ accptr[i * ldacc + j] -= ZPB ? vzpb * vreducea : 0;+ accptr[i * ldacc + j] -= ZPA ? vzpa * reduceb[j] : 0;+ accptr[i * ldacc + j] -= ZPB && ZPA ? vzpa * vzpb * k : 0;+ }+ }+ }+ return BTLA_CODE::Success;+}++template <typename T>+static inline BTLA_CODE cvt_fp32_T_2D(const float* src_ptr, T* dst_ptr, int row, int col, int src_step, int dst_step,+ bool zeropadding) {+ const int npadding = (dst_step - col) * sizeof(T);+ constexpr int simd_proc_elt = 16;+ float tmpbuf[simd_proc_elt];+ auto col_body = col / simd_proc_elt * simd_proc_elt;+ auto col_tail = col % simd_proc_elt;+ for (int i = 0; i < row; i++) {+ const auto src = src_ptr + i * src_step;+ const auto dst = dst_ptr + i * dst_step;+ int j = 0;+ for (; j < col_body; j += simd_proc_elt) {+ store_fp32_T(_mm512_loadu_ps(src + j), dst + j);+ }+ if (col_tail > 0) {+ memcpy(tmpbuf, src + j, col_tail * sizeof(src_ptr[0]));+ auto vf32 = _mm512_loadu_ps(tmpbuf);+ store_fp32_T(vf32, (T*)tmpbuf);+ memcpy(dst + j, tmpbuf, col_tail * sizeof(dst[0]));+ }+ if (zeropadding && npadding) std::memset(dst + col, 0, npadding);+ }+ return BTLA_CODE::Success;+}++template <typename T>+static inline BTLA_CODE cvt_T_fp32_2D(const T* src_ptr, float* dst_ptr, int row, int col, int src_step, int dst_step,+ bool zeropadding) {+ const int npadding = (dst_step - col) * sizeof(float);+ constexpr int simd_proc_elt = 16;+ float* tmpbuf = (float*)alloca(simd_proc_elt * sizeof(float));+ auto col_body = col / simd_proc_elt * simd_proc_elt;+ auto col_tail = col % simd_proc_elt;+ for (int i = 0; i < row; i++) {+ const auto src = src_ptr + i * src_step;+ const auto dst = dst_ptr + i * dst_step;+ int j = 0;+ for (; j < col_body; j += simd_proc_elt) {+ auto vf32 = load_T_fp32(src + j);+ _mm512_storeu_ps(dst + j, vf32);+ }+ if (col_tail > 0) {+ memcpy(tmpbuf, src + j, col_tail * sizeof(src_ptr[0]));+ auto vf32 = load_T_fp32((T*)tmpbuf);+ _mm512_storeu_ps(tmpbuf, vf32);+ memcpy(dst + j, tmpbuf, col_tail * sizeof(dst[0]));+ }+ if (zeropadding && npadding) std::memset(dst + col, 0, npadding);+ }+ return BTLA_CODE::Success;+}++template <typename SRC_T>+static inline BTLA_CODE col_block_reduce_sum(const SRC_T* srcptr, int ldsrc, int row, int col, int blocksize,+ float* reduce, int ldr) {+ int constexpr VLen = 16;+ auto vblock2_ = utils::padto_le(blocksize, VLen * 2);+ auto vblock_ = utils::padto_le(blocksize, VLen);+ for (int i = 0; i < row; i++) {+ for (int j = 0; j < col; j += blocksize) {+ auto tmp = 0.f;+ auto vsum = _mm512_set1_ps(0.f);+ int jj = 0;+ auto vblock2 = j + vblock2_ <= col ? vblock2_ : 0;+ auto vblock = j + vblock_ <= col ? vblock_ : 0;+ for (; jj < vblock2; jj += VLen * 2) {+ auto vtmp = _mm512_loadu_ps(srcptr + i * ldsrc + j + jj);+ auto vtmp1 = _mm512_loadu_ps(srcptr + i * ldsrc + j + jj + VLen);+ auto s0 = _mm512_reduce_add_ps(vtmp);+ auto s1 = _mm512_reduce_add_ps(vtmp1);+ tmp += s0;+ tmp += s1;+ }+ if (jj + VLen <= vblock) {+ for (; jj < vblock; jj += VLen) {+ auto vtmp = _mm512_loadu_ps(srcptr + i * ldsrc + j + jj);+ auto s0 = _mm512_reduce_add_ps(vtmp);+ tmp += s0;+ }+ }+ for (; jj < blocksize; jj++) {+ tmp += *(srcptr + i * ldsrc + j + jj);+ }+ reduce[i * ldr + j / blocksize] = tmp;+ }+ }+ return BTLA_CODE::Success;+}+#ifdef __GNUC__+#pragma GCC diagnostic push+#pragma GCC diagnostic ignored "-Wignored-attributes" // https://stackoverflow.com/a/49216021+#endif+// Interleave 2 bf16 zmm vectors inplace+static inline void interleave_word(std::array<__m512i, 2>& dst) { // NOLINT [runtime/references]+ static constexpr uint32_t perm_idx_a[16]{+ 0 | 0, 1 | 0, 2 | 0, 3 | 0, //+ 0 | 16, 1 | 16, 2 | 16, 3 | 16, //+ 4 | 0, 5 | 0, 6 | 0, 7 | 0, //+ 4 | 16, 5 | 16, 6 | 16, 7 | 16, //+ };+ static constexpr uint32_t perm_idx_b[16]{+ 8 | 0, 9 | 0, 10 | 0, 11 | 0, //+ 8 | 16, 9 | 16, 10 | 16, 11 | 16, //+ 12 | 0, 13 | 0, 14 | 0, 15 | 0, //+ 12 | 16, 13 | 16, 14 | 16, 15 | 16, //+ };+ static const auto v_perm_idx_a = _mm512_loadu_si512(perm_idx_a);+ static const auto v_perm_idx_b = _mm512_loadu_si512(perm_idx_b);++ __m512i tmp[2];+ tmp[0] = _mm512_unpacklo_epi16(dst[0], dst[1]);+ tmp[1] = _mm512_unpackhi_epi16(dst[0], dst[1]);+ dst[0] = _mm512_permutex2var_epi32(tmp[0], v_perm_idx_a, tmp[1]);+ dst[1] = _mm512_permutex2var_epi32(tmp[0], v_perm_idx_b, tmp[1]);+}++// Interleave 16 zmm vectors of dwords inplace+static inline void tr_x16_dword(std::array<__m512i, 16>& dst) { // NOLINT [runtime/references]+ __m512i tmp[16];++ for (int i = 0; i < 8; ++i) {+ tmp[2 * i] = _mm512_unpacklo_epi32(dst[2 * i], dst[2 * i + 1]);+ tmp[2 * i + 1] = _mm512_unpackhi_epi32(dst[2 * i], dst[2 * i + 1]);+ }++ for (int i = 0; i < 4; ++i) {+ dst[4 * i] = _mm512_unpacklo_epi64(tmp[4 * i], tmp[4 * i + 2]);+ dst[4 * i + 1] = _mm512_unpackhi_epi64(tmp[4 * i], tmp[4 * i + 2]);+ dst[4 * i + 2] = _mm512_unpacklo_epi64(tmp[4 * i + 1], tmp[4 * i + 3]);+ dst[4 * i + 3] = _mm512_unpackhi_epi64(tmp[4 * i + 1], tmp[4 * i + 3]);+ }++ for (int i = 0; i < 2; ++i) {+ tmp[8 * i + 0] = _mm512_shuffle_i32x4(dst[8 * i + 0], dst[8 * i + 4], 0x88);+ tmp[8 * i + 1] = _mm512_shuffle_i32x4(dst[8 * i + 1], dst[8 * i + 5], 0x88);+ tmp[8 * i + 2] = _mm512_shuffle_i32x4(dst[8 * i + 2], dst[8 * i + 6], 0x88);+ tmp[8 * i + 3] = _mm512_shuffle_i32x4(dst[8 * i + 3], dst[8 * i + 7], 0x88);+ tmp[8 * i + 4] = _mm512_shuffle_i32x4(dst[8 * i + 0], dst[8 * i + 4], 0xdd);+ tmp[8 * i + 5] = _mm512_shuffle_i32x4(dst[8 * i + 1], dst[8 * i + 5], 0xdd);+ tmp[8 * i + 6] = _mm512_shuffle_i32x4(dst[8 * i + 2], dst[8 * i + 6], 0xdd);+ tmp[8 * i + 7] = _mm512_shuffle_i32x4(dst[8 * i + 3], dst[8 * i + 7], 0xdd);+ }++ dst[0] = _mm512_shuffle_i32x4(tmp[0], tmp[8], 0x88);+ dst[1] = _mm512_shuffle_i32x4(tmp[1], tmp[9], 0x88);+ dst[2] = _mm512_shuffle_i32x4(tmp[2], tmp[10], 0x88);+ dst[3] = _mm512_shuffle_i32x4(tmp[3], tmp[11], 0x88);+ dst[4] = _mm512_shuffle_i32x4(tmp[4], tmp[12], 0x88);+ dst[5] = _mm512_shuffle_i32x4(tmp[5], tmp[13], 0x88);+ dst[6] = _mm512_shuffle_i32x4(tmp[6], tmp[14], 0x88);+ dst[7] = _mm512_shuffle_i32x4(tmp[7], tmp[15], 0x88);+ dst[8] = _mm512_shuffle_i32x4(tmp[0], tmp[8], 0xdd);+ dst[9] = _mm512_shuffle_i32x4(tmp[1], tmp[9], 0xdd);+ dst[10] = _mm512_shuffle_i32x4(tmp[2], tmp[10], 0xdd);+ dst[11] = _mm512_shuffle_i32x4(tmp[3], tmp[11], 0xdd);+ dst[12] = _mm512_shuffle_i32x4(tmp[4], tmp[12], 0xdd);+ dst[13] = _mm512_shuffle_i32x4(tmp[5], tmp[13], 0xdd);+ dst[14] = _mm512_shuffle_i32x4(tmp[6], tmp[14], 0xdd);+ dst[15] = _mm512_shuffle_i32x4(tmp[7], tmp[15], 0xdd);+}++// Load 2 fp16 vectors; convert them to bf16 and interleave them+template <int tail>+static inline std::array<__m512i, 2> load_fp16_bf16_interleave_word(const utils::fp16* a, size_t lda) {+ static_assert(tail > 0 && tail <= 2, "Unexpected tail value.");+ std::array<__m512i, 2> dst;+ for (int i = 0; i < tail; ++i) {+ dst[i] = zmm_cvt_fp32_bf16_x2(load_T_fp32(a + i * lda + 16), load_T_fp32(a + i * lda + 0));+ }+ for (int i = tail; i < 2; ++i) dst[i] = _mm512_setzero_epi32();+ interleave_word(dst);+ return dst;+}++// load_fp16_bf16_interleave_word with maskz+template <int tail>+static inline std::array<__m512i, 2> load_maskz_fp16_bf16_interleave_word(const utils::fp16* a, size_t lda,+ uint32_t mask) {+ static_assert(tail > 0 && tail <= 2, "Unexpected tail value.");++ const auto mask_lo = mask;+ const auto mask_hi = mask >> 16;+ std::array<__m512i, 2> dst;+ for (int i = 0; i < tail; ++i) {+ dst[i] = zmm_cvt_fp32_bf16_x2( //+ zmm_cvt_fp16_fp32(_mm256_maskz_loadu_epi16(mask_hi, a + i * lda + 16)), //+ zmm_cvt_fp16_fp32(_mm256_maskz_loadu_epi16(mask_lo, a + i * lda + 0)));+ }+ for (int i = tail; i < 2; ++i) dst[i] = _mm512_setzero_epi32();+ interleave_word(dst);+ return dst;+}++template <int tail>+static inline std::array<__m512i, 16> load_fp16_bf16_tr_x16_dword(const utils::fp16* a, size_t lda) {+ static_assert(tail > 0 && tail <= 16, "Unexpected tail value.");+ std::array<__m512i, 16> dst;+ for (int i = 0; i < tail; ++i) {+ dst[i] = zmm_cvt_fp32_bf16_x2(zmm_cvt_fp16_fp32(_mm256_loadu_epi16(a + i * lda + 16)),+ zmm_cvt_fp16_fp32(_mm256_loadu_epi16(a + i * lda + 0)));+ }+ for (int i = tail; i < 16; ++i) dst[i] = _mm512_setzero_epi32();+ tr_x16_dword(dst);+ return dst;+}+static constexpr decltype(load_fp16_bf16_tr_x16_dword<1>)* load_fp16_bf16_tr_x16_dword_tbl[17]{+ load_fp16_bf16_tr_x16_dword<1>, load_fp16_bf16_tr_x16_dword<1>, load_fp16_bf16_tr_x16_dword<2>,+ load_fp16_bf16_tr_x16_dword<3>, load_fp16_bf16_tr_x16_dword<4>, load_fp16_bf16_tr_x16_dword<5>,+ load_fp16_bf16_tr_x16_dword<6>, load_fp16_bf16_tr_x16_dword<7>, load_fp16_bf16_tr_x16_dword<8>,+ load_fp16_bf16_tr_x16_dword<9>, load_fp16_bf16_tr_x16_dword<10>, load_fp16_bf16_tr_x16_dword<11>,+ load_fp16_bf16_tr_x16_dword<12>, load_fp16_bf16_tr_x16_dword<13>, load_fp16_bf16_tr_x16_dword<14>,+ load_fp16_bf16_tr_x16_dword<15>, load_fp16_bf16_tr_x16_dword<16>,+};++template <int tail>+static inline std::array<__m512i, 16> load_maskz_fp16_bf16_tr_x16_dword(const utils::fp16* a, size_t lda,+ uint32_t mask) {+ static_assert(tail > 0 && tail <= 16, "Unexpected tail value.");+ std::array<__m512i, 16> dst;++ const auto mask_lo = mask;+ const auto mask_hi = mask >> 16;+ for (int i = 0; i < tail; ++i) {+ dst[i] = zmm_cvt_fp32_bf16_x2(zmm_cvt_fp16_fp32(_mm256_maskz_loadu_epi16(mask_hi, a + i * lda + 16)),+ zmm_cvt_fp16_fp32(_mm256_maskz_loadu_epi16(mask_lo, a + i * lda + 0)));+ }+ for (int i = tail; i < 16; ++i) dst[i] = _mm512_setzero_epi32();+ tr_x16_dword(dst);+ return dst;+}+static constexpr decltype(load_maskz_fp16_bf16_tr_x16_dword<1>)* load_maskz_fp16_bf16_tr_x16_dword_tbl[17]{+ load_maskz_fp16_bf16_tr_x16_dword<1>, load_maskz_fp16_bf16_tr_x16_dword<1>, load_maskz_fp16_bf16_tr_x16_dword<2>,+ load_maskz_fp16_bf16_tr_x16_dword<3>, load_maskz_fp16_bf16_tr_x16_dword<4>, load_maskz_fp16_bf16_tr_x16_dword<5>,+ load_maskz_fp16_bf16_tr_x16_dword<6>, load_maskz_fp16_bf16_tr_x16_dword<7>, load_maskz_fp16_bf16_tr_x16_dword<8>,+ load_maskz_fp16_bf16_tr_x16_dword<9>, load_maskz_fp16_bf16_tr_x16_dword<10>, load_maskz_fp16_bf16_tr_x16_dword<11>,+ load_maskz_fp16_bf16_tr_x16_dword<12>, load_maskz_fp16_bf16_tr_x16_dword<13>, load_maskz_fp16_bf16_tr_x16_dword<14>,+ load_maskz_fp16_bf16_tr_x16_dword<15>, load_maskz_fp16_bf16_tr_x16_dword<16>,+};+#ifdef __GNUC__+#pragma GCC diagnostic pop+#endif++template <typename T_SRC, typename T_DST = T_SRC, int RowPack = 4 / sizeof(T_DST)>+struct padding_interleave_cvt {+ padding_interleave_cvt() = delete;+ static BTLA_CODE forward(const T_SRC* src, T_DST* dst, int NTile, int row, int col, int row_pad, int col_pad,+ int src_step, int dst_step) {+ return BTLA_CODE::NotSupport;+ }+};++template <>+struct padding_interleave_cvt<utils::fp16, utils::bf16, 2> {+ static constexpr int RowPack = 2;+ padding_interleave_cvt() = delete;++ // M x N ===> N/NTile x M/RowPack x NTile x RowPack (leading dim stride = NTile * dststride)+ static BTLA_CODE forward(const utils::fp16* src, utils::bf16* dst, int NTile, int row, int col, int row_pad,+ int col_pad, int src_step, int dst_step) {+ int i = 0;+ for (; i < row / RowPack * RowPack; i += RowPack) {+ int j = 0;+ for (; j < col / NTile * NTile; j += NTile) {+ assert(NTile % 32 == 0);+ for (int jj = 0; jj < NTile; jj += 32) {+ const auto xss = load_fp16_bf16_interleave_word<2>(src + i * src_step + j + jj, src_step);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 0) * RowPack, xss[0]);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 16) * RowPack, xss[1]);+ }+ }+ if (j < col) { // j: tail processing+ int jj = 0;+ for (; j + jj < col / 32 * 32; jj += 32) {+ const auto xss = load_fp16_bf16_interleave_word<2>(src + i * src_step + j + jj, src_step);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 0) * RowPack, xss[0]);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 16) * RowPack, xss[1]);+ }+ if (j + jj < col) { // jj: tail processing+ const uint32_t mask = (1U << (col - j - jj)) - 1;+ const auto xss = load_maskz_fp16_bf16_interleave_word<2>(src + i * src_step + j + jj, src_step, mask);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 0) * RowPack, xss[0]);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 16) * RowPack, xss[1]);+ jj += 32;+ }+ for (; jj < NTile; jj += 32) { // jj: padding zero+ memset(dst + i * NTile + j * dst_step + jj * RowPack, 0, sizeof(utils::bf16) * 32 * RowPack);+ }+ j += NTile;+ }+ for (; j < col_pad; j += NTile) { // j: padding zero+ memset(dst + i * NTile + j * dst_step, 0, sizeof(utils::bf16) * NTile * RowPack);+ }+ }+ if (i < row) { // i: tail processing+ static constexpr int tail_m = 1; // must be 1+ int j = 0;+ for (; j < col / NTile * NTile; j += NTile) {+ assert(NTile % 32 == 0);+ for (int jj = 0; jj < NTile; jj += 32) {+ const auto xss = load_fp16_bf16_interleave_word<tail_m>(src + i * src_step + j + jj, src_step);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 0) * RowPack, xss[0]);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 16) * RowPack, xss[1]);+ }+ }+ if (j < col) { // j: tail processing+ int jj = 0;+ for (; j + jj < col / 32 * 32; jj += 32) {+ const auto xss = load_fp16_bf16_interleave_word<tail_m>(src + i * src_step + j + jj, src_step);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 0) * RowPack, xss[0]);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 16) * RowPack, xss[1]);+ }+ if (j + jj < col) { // jj: tail processing+ const uint32_t mask = (1U << (col - j - jj)) - 1;+ const auto xss = load_maskz_fp16_bf16_interleave_word<tail_m>(src + i * src_step + j + jj, src_step, mask);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 0) * RowPack, xss[0]);+ _mm512_storeu_si512(dst + i * NTile + j * dst_step + (jj + 16) * RowPack, xss[1]);+ jj += 32;+ }+ for (; jj < NTile; jj += 32) { // jj: padding zero+ memset(dst + i * NTile + j * dst_step + jj * RowPack, 0, sizeof(utils::bf16) * 32 * RowPack);+ }+ j += NTile;+ }+ for (; j < col_pad; j += NTile) { // j: padding zero+ memset(dst + i * NTile + j * dst_step, 0, sizeof(utils::bf16) * NTile * RowPack);+ }+ i += RowPack;+ }+ for (; i < row_pad; i += RowPack) { // i: padding zero+ for (int j = 0; j < col_pad; j += NTile) {+ memset(dst + i * NTile + j * dst_step, 0, sizeof(utils::bf16) * NTile * RowPack);+ }+ }+ return BTLA_CODE::Success;+ }+};++template <typename T_SRC, typename T_DST = T_SRC, int ColPack = 4 / sizeof(T_DST)>+struct padding_trans_interleave_cvt {+ padding_trans_interleave_cvt() = delete;+ static BTLA_CODE forward(const T_SRC* src, T_DST* dst, int MTile, int row, int col, int row_pad, int col_pad,+ int src_step, int dst_step) {+ return BTLA_CODE::NotSupport;+ }+};++template <>+struct padding_trans_interleave_cvt<utils::fp16, utils::bf16, 2> {+ static constexpr int ColPack = 2;+ padding_trans_interleave_cvt() = delete;++ static BTLA_CODE forward(const utils::fp16* src, utils::bf16* dst, int MTile, int row, int col, int row_pad,+ int col_pad, int src_step, int dst_step) {+ assert(row_pad % 16 == 0 && col_pad % 32 == 0);+ int i = 0;+ for (; i < row / MTile * MTile; i += MTile) {+ assert(MTile % 16 == 0);+ int j = 0;+ for (; j < col / 32 * 32; j += 32) {+ for (int ii = 0; ii < MTile; ii += 16) {+ assert(MTile % 16 == 0);+ const auto xss = load_fp16_bf16_tr_x16_dword<16>(src + (i + ii) * src_step + j, src_step);+ for (int jj = 0; jj < 32; jj += 2) {+ _mm512_storeu_si512(dst + i * dst_step + ii * ColPack + (j + jj) * MTile, xss[jj / 2]);+ }+ }+ }+ if (j < col) { // j: tail processing+ for (int ii = 0; ii < MTile; ii += 16) {+ assert(MTile % 16 == 0);+ const uint32_t mask = (1U << (col - j)) - 1;+ const auto xss = load_maskz_fp16_bf16_tr_x16_dword<16>(src + (i + ii) * src_step + j, src_step, mask);+ for (int jj = 0; jj < 32; jj += 2) {+ _mm512_storeu_si512(dst + i * dst_step + ii * ColPack + (j + jj) * MTile, xss[jj / 2]);+ }+ }+ j += 32;+ }+ for (; j < col_pad; j += 2) { // j: padding zero+ memset(dst + i * dst_step + j * MTile, 0, 2 * sizeof(utils::bf16) * MTile);+ }+ }+ if (i < row) { // i: tail processing+ int ii = 0;+ for (; i + ii < row / 16 * 16; ii += 16) {+ int j = 0;+ for (; j < col / 32 * 32; j += 32) {+ assert(MTile % 16 == 0);+ const auto xss = load_fp16_bf16_tr_x16_dword<16>(src + (i + ii) * src_step + j, src_step);+ for (int jj = 0; jj < 32; jj += 2) {+ _mm512_storeu_si512(dst + i * dst_step + ii * ColPack + (j + jj) * MTile, xss[jj / 2]);+ }+ }+ if (j < col) { // j: tail processing+ assert(MTile % 16 == 0);+ const uint32_t mask = (1U << (col - j)) - 1;+ const auto xss = load_maskz_fp16_bf16_tr_x16_dword<16>(src + (i + ii) * src_step + j, src_step, mask);+ for (int jj = 0; jj < 32; jj += 2) {+ _mm512_storeu_si512(dst + i * dst_step + ii * ColPack + (j + jj) * MTile, xss[jj / 2]);+ }+ j += 32;+ }+ for (; j < col_pad; j += 2) { // j: padding zero+ memset(dst + i * dst_step + ii * ColPack + j * MTile, 0, 2 * sizeof(utils::bf16) * 16);+ }+ }+ if (i + ii < row) { // ii: tail processing+ const int tbl_idx = row - i - ii;+ int j = 0;+ for (; j < col / 32 * 32; j += 32) {+ assert(MTile % 16 == 0);+ const auto xss = load_fp16_bf16_tr_x16_dword_tbl[tbl_idx](src + (i + ii) * src_step + j, src_step);+ for (int jj = 0; jj < 32; jj += 2) {+ _mm512_storeu_si512(dst + i * dst_step + ii * ColPack + (j + jj) * MTile, xss[jj / 2]);+ }+ }+ if (j < col) { // j: tail processing+ assert(MTile % 16 == 0);+ const uint32_t mask = (1U << (col - j)) - 1;+ const auto xss =+ load_maskz_fp16_bf16_tr_x16_dword_tbl[tbl_idx](src + (i + ii) * src_step + j, src_step, mask);+ for (int jj = 0; jj < 32; jj += 2) {+ _mm512_storeu_si512(dst + i * dst_step + ii * ColPack + (j + jj) * MTile, xss[jj / 2]);+ }+ j += 32;+ }+ for (; j < col_pad; j += 2) { // j: padding zero+ memset(dst + i * dst_step + ii * ColPack + j * MTile, 0, 2 * sizeof(utils::bf16) * 16);+ }+ ii += 16;+ }+ for (; ii < MTile; ii += 16) { // ii: padding zero+ for (int j = 0; j < col_pad; j += 2) {+ memset(dst + i * dst_step + ii * ColPack + j * MTile, 0, 2 * sizeof(utils::bf16) * 16);+ }+ }+ assert(ii == MTile);+ i += MTile;+ }+ assert(row_pad % MTile == 0);+ for (; i < row_pad; i += MTile) { // i: padding zero+ for (int j = 0; j < col_pad; j += 2) {+ memset(dst + i * dst_step + j * MTile, 0, 2 * sizeof(utils::bf16) * MTile);+ }+ }+ return BTLA_CODE::Success;+ }+};++static inline BTLA_CODE layernorm(const float* srcptr, const float* scaleptr, const float* biasptr, float epsilon,+ int norm_size, float* dstptr, float* mean_out, float* mean_square_out,+ bool simplified) {+ int constexpr VLen = 16;+ int norm_size16 = utils::padto_le(norm_size, VLen);+ int h = 0;+ __m512 vmean = _mm512_setzero_ps(), vmeansq = _mm512_setzero_ps();+ for (; h < norm_size16; h += VLen) {+ auto tmp = _mm512_loadu_ps(srcptr + h);+ vmean = _mm512_add_ps(vmean, tmp);+ tmp = _mm512_mul_ps(tmp, tmp);+ vmeansq = _mm512_add_ps(vmeansq, tmp);+ }+ float mean = _mm512_reduce_add_ps(vmean);+ float mean_square = _mm512_reduce_add_ps(vmeansq);+ for (; h < norm_size; h++) {+ mean += srcptr[h];+ mean_square += srcptr[h] * srcptr[h];+ }+ mean = mean / norm_size;+ if (simplified) {+ mean_square = std::sqrt(mean_square / norm_size + epsilon);+ } else {+ mean_square = std::sqrt(mean_square / norm_size - mean * mean + epsilon);+ }+ auto vm = _mm512_set1_ps(mean);+ float inv_meansq = 1.f / mean_square;+ auto vms = _mm512_set1_ps(inv_meansq);+ h = 0;+ if (simplified) {+ if (scaleptr) {+ for (; h < norm_size16; h += VLen) {+ auto inp = _mm512_loadu_ps(srcptr + h);+ auto scale = _mm512_loadu_ps(scaleptr + h);+ inp = _mm512_mul_ps(inp, vms);+ inp = _mm512_mul_ps(inp, scale);+ _mm512_storeu_ps(dstptr + h, inp);+ }+ for (; h < norm_size; h++) {+ dstptr[h] = srcptr[h] * inv_meansq * scaleptr[h];+ }+ } else {+ for (; h < norm_size16; h += VLen) {+ auto inp = _mm512_loadu_ps(srcptr + h);+ inp = _mm512_mul_ps(inp, vms);+ _mm512_storeu_ps(dstptr + h, inp);+ }+ for (; h < norm_size; h++) {+ dstptr[h] = srcptr[h] * inv_meansq;+ }+ }++ } else {+ if (scaleptr) {+ if (biasptr == nullptr) {+ for (; h < norm_size16; h += VLen) {+ auto inp = _mm512_loadu_ps(srcptr + h);+ auto scale = _mm512_loadu_ps(scaleptr + h);+ inp = _mm512_sub_ps(inp, vm);+ inp = _mm512_mul_ps(inp, vms);+ inp = _mm512_mul_ps(inp, scale);+ _mm512_storeu_ps(dstptr + h, inp);+ }+ for (; h < norm_size; h++) {+ dstptr[h] = (srcptr[h] - mean) * inv_meansq * scaleptr[h];+ }+ } else {+ for (; h < norm_size16; h += VLen) {+ auto inp = _mm512_loadu_ps(srcptr + h);+ auto scale = _mm512_loadu_ps(scaleptr + h);+ inp = _mm512_sub_ps(inp, vm);+ inp = _mm512_mul_ps(inp, vms);+ inp = _mm512_mul_ps(inp, scale);+ auto bias = _mm512_loadu_ps(biasptr + h);+ inp = _mm512_add_ps(inp, bias);+ _mm512_storeu_ps(dstptr + h, inp);+ }+ for (; h < norm_size; h++) {+ dstptr[h] = (srcptr[h] - mean) * inv_meansq * scaleptr[h] + biasptr[h];+ }+ }+ } else {+ for (; h < norm_size16; h += VLen) {+ auto inp = _mm512_loadu_ps(srcptr + h);+ inp = _mm512_sub_ps(inp, vm);+ inp = _mm512_mul_ps(inp, vms);+ _mm512_storeu_ps(dstptr + h, inp);+ }+ for (; h < norm_size; h++) {+ dstptr[h] = (srcptr[h] - mean) * inv_meansq;+ }+ }+ }++ if (mean_out) {+ *mean_out = mean;+ }+ if (mean_square_out) {+ *mean_square_out = mean_square;+ }+ return BTLA_CODE::Success;+}++inline __m512 poly_scale_2nd_ps(const __m512 z, const __m512 f, const __m512 c0, const __m512 c1, const __m512 c2) {+ const auto y = _mm512_fmadd_ps(_mm512_fmadd_ps(f, c0, c1), f, c2); // auto y = (f * c0 + c1) * f + c2;+ const auto exp = _mm512_scalef_ps(y, z);+ return exp;+}++inline __m512 exp_ps_0_1(const __m512 x) {+ static const auto c0 = _mm512_set1_ps(0.240226507f);+ static const auto c1 = _mm512_set1_ps(0.452920674f);+ static const auto c2 = _mm512_set1_ps(0.713483036f);+ static const float v_log2e = std::log2(std::exp(1.f));+ static const auto log2e = _mm512_set1_ps(v_log2e);+ static const auto half = _mm512_set1_ps(.5f);++ const auto x1 = _mm512_fmadd_ps(x, log2e, half); // auto x1 = x * log2e + _mm512_set1_ps(.5f);+ const auto z = _mm512_floor_ps(x1);+ const auto f = _mm512_sub_ps(x1, z); // auto f = x1 - z;++ return poly_scale_2nd_ps(z, f, c0, c1, c2);+}++static inline __m512i load_zp_epi8_broadcast_epi16(int8_t* zpptr, const __m512i& vindex) {+ auto v_zp_x = _mm256_loadu_si256((const __m256i*)zpptr);+ auto v_zp_y = _mm512_cvtepi8_epi16(v_zp_x);+ auto v_zp_y_cast = _mm512_shuffle_epi8(v_zp_y, vindex); // TODO(Yu) AVX512F only+ return v_zp_y_cast;+}++static inline __m512i load_zp_epi8_broadcast_epi32(int8_t* zpptr, const __m512i& vindex) {+ auto v_zp_x = _mm_loadu_si128((const __m128i*)zpptr);+ auto v_zp_y = _mm512_cvtepi8_epi32(v_zp_x);+ auto v_zp_y_cast = _mm512_shuffle_epi8(v_zp_y, vindex); // TODO(Yu) AVX512F only+ return v_zp_y_cast;+}++static inline BTLA_CODE decompress_s4_s8(utils::int4x2* srcptr, int8_t* dstptr, size_t elesize, int8_t* tmp,+ size_t tmpsize) {+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ size_t velt = utils::padto_le(elesize, 64);+ size_t i = 0;+ auto vbias = _mm512_set1_epi8(8);+ for (; i < velt; i += 64) {+ auto vout_y = unpack_4bits(reinterpret_cast<int8_t*>(srcptr + i / 2), vmask);+ vout_y = _mm512_sub_epi8(vout_y, vbias);+ _mm512_storeu_si512((__m512i*)(dstptr + i), vout_y);+ }+ if (velt < elesize) {+ if (elesize >= 64) {+ i = elesize - 64;+ auto vout_y = unpack_4bits(reinterpret_cast<int8_t*>(srcptr + i / 2), vmask);+ vout_y = _mm512_sub_epi8(vout_y, vbias);+ _mm512_storeu_si512((__m512i*)(dstptr + i), vout_y);+ } else {+ ref::decompress_kblock_s4_s8<1, 1>(srcptr + i / 2, nullptr, dstptr + i, 0, 0, 0, 0, 1,+ static_cast<int>(elesize - i), nullptr, 0);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s4_s8_pack1_row(utils::int4x2* srcptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 16;+ static_assert((NTILE % 16) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ int constexpr UnpackLoop = Unroll * NTILE / 64;+ __m512i v_zp_y[UnpackLoop];+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(8);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, zptr, NTILE * sizeof(int8_t));+ }+ for (int i = 0; i < UnpackLoop; i++) {+ v_zp_y[i] = _mm512_loadu_si512((const __m512i*)(tmp + i * 64));+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += Unroll) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ for (int i = 0; i < UnpackLoop; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 32, vmask);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }++ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ memcpy(tmp, srcptr + (ir + ib) * NTILE / 2, k_tail * NTILE / 2);+ auto tmpout = tmp + Unroll * NTILE / 2;+ for (int i = 0; i < UnpackLoop; i++) {+ auto v_s8_y = unpack_4bits(tmp + i * 32, vmask);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(tmpout + i * 64), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s4_s8_pack2_row(utils::int4x2* srcptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 16;+ static_assert((NTILE % 16) == 0);+ int constexpr PackRow = 2;+ int constexpr Unroll = 2;+ __m512i v_zp_y[NReg];+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(8);+ const auto vindex = _mm512_set_epi8(14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8,+ 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0,+ 14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ memcpy(tmp, zptr, NTILE * sizeof(int8_t));+ memcpy(tmp + NTILE, zptr, NTILE * sizeof(int8_t));+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi16(tmp + i * 32, vindex);+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, PackRow * Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += PackRow * Unroll) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 32, vmask);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }++ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ memcpy(tmp, srcptr + (ir + ib) * NTILE / 2, k_tail * NTILE / 2);+ auto tmpout = tmp + Unroll * PackRow * NTILE / 2;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(tmp + i * 32, vmask);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(tmpout + i * 64), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s4_s8_pack4_row(utils::int4x2* srcptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 16;+ static_assert((NTILE % 16) == 0);+ int constexpr PackRow = 4;+ __m512i v_zp_y[NReg];+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(8);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi32(zptr + i * 16, vindex);+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 32, vmask);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+inline BTLA_CODE decompress_kblock_s4_s8(utils::int4x2* srcptr, int8_t* zpptr, int8_t* dstptr, int blocksize, int ldzp,+ int n_offset, int k_offset, int row, int col, int8_t* tmp, size_t tmpsize) {+ if (zpptr) {+ typedef BTLA_CODE (*decompfunc)(utils::int4x2* srcptr, int8_t* zpptr, int8_t* dstptr, int blocksize, int ldzp,+ int n_offset, int k_offset, int row, int8_t* tmp, size_t tmpsize);+ decompfunc func = nullptr;+ if (col == NTILE) {+ if constexpr (PackRow == 1) {+ func = &decompress_kblock_s4_s8_pack1_row<NTILE>;+ }+ if constexpr (PackRow == 2) {+ func = &decompress_kblock_s4_s8_pack2_row<NTILE>;+ }+ if constexpr (PackRow == 4) {+ func = &decompress_kblock_s4_s8_pack4_row<NTILE>;+ }+ if (func) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ (*func)(srcptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, head_size, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ (*func)(srcptr + head_size * NTILE / 2, zpptr, dstptr + head_size * NTILE, blocksize, ldzp, n_offset,+ head_end, body_size, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ }+ assert(0);+ return BTLA_CODE::NotSupport;+ } else {+ size_t elesize = static_cast<size_t>(row) * col;+ return decompress_s4_s8(srcptr, dstptr, elesize, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s2_s8(utils::bit2x4* bit2ptr, int8_t* dstptr, size_t unpack_elt, int8_t* tmp,+ size_t tmpsize) {+ int constexpr VBits = 512;+ int constexpr VElt = VBits / 8;+ size_t i = 0;+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vbias = _mm512_set1_epi8(2);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ size_t elt_pad = utils::padto_le(unpack_elt, VElt);+ for (; i < elt_pad; i += VElt) {+ auto vout = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vout = _mm512_sub_epi8(vout, vbias);+ _mm512_storeu_si512((__m512i*)(dstptr + i), vout);+ }+ if (elt_pad < unpack_elt) {+ if (unpack_elt >= VElt) {+ i = unpack_elt - VElt;+ auto vout = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vout = _mm512_sub_epi8(vout, vbias);+ _mm512_storeu_si512((__m512i*)(dstptr + i), vout);+ } else {+ ref::decompress_s2_s8(bit2ptr + i / 4, dstptr + i, unpack_elt - i, tmp, tmpsize);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s2_s8_pack4_row(utils::bit2x4* srcptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 4;+ __m512i v_zp_y[NReg];+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vbias = _mm512_set1_epi8(2);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi32(zptr + i * 16, vindex);+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b2ptr = srcptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_2bits(b2ptr + i * 16, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s2_s8_pack2_row(utils::bit2x4* srcptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 2;+ int constexpr Unroll = 2;+ __m512i v_zp_y[NReg];+ const auto vindex = _mm512_set_epi8(14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8,+ 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0,+ 14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0);+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vbias = _mm512_set1_epi8(2);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ memcpy(tmp, zptr, NTILE * sizeof(int8_t));+ memcpy(tmp + NTILE, zptr, NTILE * sizeof(int8_t));+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi16(tmp + i * 32, vindex);+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, PackRow * Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += PackRow * Unroll) {+ auto b2ptr = srcptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_2bits(b2ptr + i * 16, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ memcpy(tmp, srcptr + (ir + ib) * NTILE / 4, k_tail * NTILE / 4);+ auto tmpout = tmp + Unroll * PackRow * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_2bits((utils::bit2x4*)(tmp + i * 16), vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(tmpout + i * 64), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s2_s8_pack1_row(utils::bit2x4* srcptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ __m512i v_zp_y[NReg];+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vbias = _mm512_set1_epi8(2);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, zptr, NTILE * sizeof(int8_t));+ }+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = _mm512_loadu_si512((const __m512i*)(tmp + i * 64));+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += Unroll) {+ auto b2ptr = srcptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_2bits(b2ptr + i * 16, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }++ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ memcpy(tmp, srcptr + (ir + ib) * NTILE / 4, k_tail * NTILE / 4);+ auto tmpout = tmp + Unroll * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_2bits((utils::bit2x4*)(tmp + i * 16), vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(tmpout + i * 64), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s2_s8(utils::bit2x4* bit2ptr, int8_t* zpptr, int8_t* dstptr, int blocksize,+ int ldzp, int n_offset, int k_offset, int row, int col, int8_t* tmp,+ size_t tmpsize) {+ if (zpptr) {+ typedef BTLA_CODE (*decompfunc)(utils::bit2x4* srcptr, int8_t* zpptr, int8_t* dstptr, int blocksize, int ldzp,+ int n_offset, int k_offset, int row, int8_t* tmp, size_t tmpsize);+ decompfunc func = nullptr;+ if (col == NTILE) {+ if constexpr (PackRow == 1) {+ func = &decompress_kblock_s2_s8_pack1_row<NTILE>;+ }+ if constexpr (PackRow == 2) {+ func = &decompress_kblock_s2_s8_pack2_row<NTILE>;+ }+ if constexpr (PackRow == 4) {+ func = &decompress_kblock_s2_s8_pack4_row<NTILE>;+ }+ if (func) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ (*func)(bit2ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, head_size, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ (*func)(bit2ptr + head_size * NTILE / 4, zpptr, dstptr + head_size * NTILE, blocksize, ldzp, n_offset,+ head_end, body_size, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ }+ assert(0);+ return BTLA_CODE::NotSupport;+ } else {+ size_t elesize = static_cast<size_t>(row) * col;+ return decompress_s2_s8(bit2ptr, dstptr, elesize, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s3_s8(utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr, int8_t* dstptr,+ size_t unpack_elt, int8_t* tmp, size_t tmpsize) {+ int constexpr VBits = 512;+ int constexpr VElt = VBits / 8;+ size_t i = 0;+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vbias = _mm512_set1_epi8(4);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ size_t elt_pad = utils::padto_le(unpack_elt, VElt);+ for (; i < elt_pad; i += VElt) {+ auto vout = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(bit1ptr + i / 8, zmm_0x00, zmm_0x04);+ vout = _mm512_or_si512(vout, vb1);+ vout = _mm512_sub_epi8(vout, vbias);+ _mm512_storeu_si512((__m512i*)(dstptr + i), vout);+ }+ if (elt_pad < unpack_elt) {+ if (unpack_elt >= VElt) {+ i = unpack_elt - VElt;+ auto vout = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(bit1ptr + i / 8, zmm_0x00, zmm_0x04);+ vout = _mm512_or_si512(vout, vb1);+ vout = _mm512_sub_epi8(vout, vbias);+ _mm512_storeu_si512((__m512i*)(dstptr + i), vout);+ } else {+ ref::decompress_s3_s8(bit2ptr + i / 4, bit1ptr + i / 8, dstptr + i, unpack_elt - i, tmp, tmpsize);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s3_s8_pack1_row(utils::bit2x4* srcptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ __m512i v_zp_y[NReg];+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vbias = _mm512_set1_epi8(4);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, zptr, NTILE * sizeof(int8_t));+ }+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = _mm512_loadu_si512((const __m512i*)(tmp + i * 64));+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += Unroll) {+ auto b2ptr = srcptr + (ir + ib) * NTILE / 4;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_2bits(b2ptr + i * 16, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr + i * 8, zmm_0x00, zmm_0x04);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }++ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb2ptr = tmp;+ memcpy(tmpb2ptr, srcptr + (ir + ib) * NTILE / 4, k_tail * NTILE / 4);+ auto tmpb1ptr = tmp + Unroll * NTILE / 2;+ memcpy(tmpb1ptr, bit1ptr + (ir + ib) * NTILE / 8, k_tail * NTILE / 8);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_2bits((utils::bit2x4*)(tmpb2ptr + i * 16), vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits((utils::bit1x8*)(tmpb1ptr + i * 8), zmm_0x00, zmm_0x04);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(tmpout + i * 64), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s3_s8_pack2_row(utils::bit2x4* srcptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ __m512i v_zp_y[NReg];+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vbias = _mm512_set1_epi8(4);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);++ const auto vindex = _mm512_set_epi8(14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8,+ 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0,+ 14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ memcpy(tmp, zptr, NTILE * sizeof(int8_t));+ memcpy(tmp + NTILE, zptr, NTILE * sizeof(int8_t));+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi16(tmp + i * 32, vindex);+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, PackRow * Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += PackRow * Unroll) {+ auto b2ptr = srcptr + (ir + ib) * NTILE / 4;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_2bits(b2ptr + i * 16, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr + i * 8, zmm_0x00, zmm_0x04);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb2ptr = tmp;+ memcpy(tmpb2ptr, srcptr + (ir + ib) * NTILE / 4, k_tail * NTILE / 4);+ auto tmpb1ptr = tmp + Unroll * NTILE / 2;+ memcpy(tmpb1ptr, bit1ptr + (ir + ib) * NTILE / 8, k_tail * NTILE / 8);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_2bits((utils::bit2x4*)(tmpb2ptr + i * 16), vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits((utils::bit1x8*)(tmpb1ptr + i * 8), zmm_0x00, zmm_0x04);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(tmpout + i * 64), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s3_s8_pack4_row(utils::bit2x4* srcptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 4;+ __m512i v_zp_y[NReg];+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vbias = _mm512_set1_epi8(4);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi32(zptr + i * 16, vindex);+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b2ptr = srcptr + (ir + ib) * NTILE / 4;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_2bits(b2ptr + i * 16, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr + i * 8, zmm_0x00, zmm_0x04);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s3_s8(utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset, int k_offset,+ int row, int col, int8_t* tmp, size_t tmpsize) {+ if (zpptr) {+ typedef BTLA_CODE (*decompfunc)(utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row, int8_t* tmp,+ size_t tmpsize);+ decompfunc func = nullptr;+ if (col == NTILE) {+ if constexpr (PackRow == 1) {+ func = &decompress_kblock_s3_s8_pack1_row<NTILE>;+ }+ if constexpr (PackRow == 2) {+ func = &decompress_kblock_s3_s8_pack2_row<NTILE>;+ }+ if constexpr (PackRow == 4) {+ func = &decompress_kblock_s3_s8_pack4_row<NTILE>;+ }++ if (func) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ (*func)(bit2ptr, bit1ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, head_size, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ (*func)(bit2ptr + head_size * NTILE / 4, bit1ptr + head_size * NTILE / 8, zpptr, dstptr + head_size * NTILE,+ blocksize, ldzp, n_offset, head_end, body_size, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ }+ assert(0);+ return BTLA_CODE::NotSupport;+ } else {+ size_t elesize = static_cast<size_t>(row) * col;+ return decompress_s3_s8(bit2ptr, bit1ptr, dstptr, elesize, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s1_s8(utils::bit1x8* bit1ptr, int8_t* dstptr, size_t unpack_elt, int8_t* tmp,+ size_t tmpsize) {+ int constexpr VBits = 512;+ int constexpr VElt = VBits / 8;+ size_t i = 0;+ int constexpr FullRange = 1 << (1 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ size_t elt_pad = utils::padto_le(unpack_elt, VElt);+ for (; i < elt_pad; i += VElt) {+ auto vb1 = unpack_1bits(bit1ptr + i / 8, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, vbias);+ _mm512_storeu_si512((__m512i*)(dstptr + i), vb1);+ }+ if (elt_pad < unpack_elt) {+ if (unpack_elt >= VElt) {+ i = unpack_elt - VElt;+ auto vb1 = unpack_1bits(bit1ptr + i / 8, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, vbias);+ _mm512_storeu_si512((__m512i*)(dstptr + i), vb1);+ } else {+ ref::decompress_s1_s8(bit1ptr + i / 8, dstptr + i, unpack_elt - i, tmp, tmpsize);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s1_s8_pack1_row(utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ __m512i v_zp_y[NReg];+ int constexpr FullRange = 1 << (1 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, zptr, NTILE * sizeof(int8_t));+ }+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = _mm512_loadu_si512((const __m512i*)(tmp + i * 64));+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += Unroll) {+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr + i * 8, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), vb1);+ }+ }++ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb1ptr = tmp;+ memcpy(tmpb1ptr, bit1ptr + (ir + ib) * NTILE / 8, k_tail * NTILE / 8);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits((utils::bit1x8*)(tmpb1ptr + i * 8), zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(tmpout + i * 64), vb1);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s1_s8_pack2_row(utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ __m512i v_zp_y[NReg];+ int constexpr FullRange = 1 << (1 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);++ const auto vindex = _mm512_set_epi8(14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8,+ 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0,+ 14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ memcpy(tmp, zptr, NTILE * sizeof(int8_t));+ memcpy(tmp + NTILE, zptr, NTILE * sizeof(int8_t));+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi16(tmp + i * 32, vindex);+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, PackRow * Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += PackRow * Unroll) {+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr + i * 8, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), vb1);+ }+ }+ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb1ptr = tmp;+ memcpy(tmpb1ptr, bit1ptr + (ir + ib) * NTILE / 8, k_tail * NTILE / 8);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits((utils::bit1x8*)(tmpb1ptr + i * 8), zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(tmpout + i * 64), vb1);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s1_s8_pack4_row(utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 4;+ __m512i v_zp_y[NReg];+ int constexpr FullRange = 1 << (1 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi32(zptr + i * 16, vindex);+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr + i * 8, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), vb1);+ }+ }+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s1_s8(utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr, int blocksize,+ int ldzp, int n_offset, int k_offset, int row, int col, int8_t* tmp,+ size_t tmpsize) {+ if (zpptr) {+ typedef BTLA_CODE (*decompfunc)(utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr, int blocksize, int ldzp,+ int n_offset, int k_offset, int row, int8_t* tmp, size_t tmpsize);+ decompfunc func = nullptr;+ if (col == NTILE) {+ if constexpr (PackRow == 1) {+ func = &decompress_kblock_s1_s8_pack1_row<NTILE>;+ }+ if constexpr (PackRow == 2) {+ func = &decompress_kblock_s1_s8_pack2_row<NTILE>;+ }+ if constexpr (PackRow == 4) {+ func = &decompress_kblock_s1_s8_pack4_row<NTILE>;+ }+ if (func) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ (*func)(bit1ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, head_size, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ (*func)(bit1ptr + head_size * NTILE / 8, zpptr, dstptr + head_size * NTILE, blocksize, ldzp, n_offset,+ head_end, body_size, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ }+ assert(0);+ return BTLA_CODE::NotSupport;+ } else {+ size_t elesize = static_cast<size_t>(row) * col;+ return decompress_s1_s8(bit1ptr, dstptr, elesize, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s5_s8(utils::bit4x2* bit4ptr, utils::bit1x8* bit1ptr, int8_t* dstptr,+ size_t unpack_elt, int8_t* tmp, size_t tmpsize) {+ int constexpr VBits = 512;+ int constexpr VElt = VBits / 8;+ size_t i = 0;+ int constexpr FullRange = 1 << (5 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ size_t elt_pad = utils::padto_le(unpack_elt, VElt);+ for (; i < elt_pad; i += VElt) {+ auto vout = unpack_4bits(bit4ptr + i / 2, vmask);+ auto vb1 = unpack_1bits(bit1ptr + i / 8, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ vout = _mm512_or_si512(vout, vb1);+ vout = _mm512_sub_epi8(vout, vbias);+ _mm512_storeu_si512((__m512i*)(dstptr + i), vout);+ }+ if (elt_pad < unpack_elt) {+ if (unpack_elt >= VElt) {+ i = unpack_elt - VElt;+ auto vout = unpack_4bits(bit4ptr + i / 2, vmask);+ auto vb1 = unpack_1bits(bit1ptr + i / 8, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ vout = _mm512_or_si512(vout, vb1);+ vout = _mm512_sub_epi8(vout, vbias);+ _mm512_storeu_si512((__m512i*)(dstptr + i), vout);+ } else {+ ref::decompress_s5_s8(bit4ptr + i / 2, bit1ptr + i / 8, dstptr + i, unpack_elt - i, tmp, tmpsize);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s5_s8_pack1_row(utils::bit4x2* srcptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ __m512i v_zp_y[NReg];+ int constexpr FullRange = 1 << (5 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, zptr, NTILE * sizeof(int8_t));+ }+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = _mm512_loadu_si512((const __m512i*)(tmp + i * 64));+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += Unroll) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 32, vmask);+ auto vb1 = unpack_1bits(b1ptr + i * 8, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }++ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb4ptr = tmp;+ memcpy(tmpb4ptr, srcptr + (ir + ib) * NTILE / 2, k_tail * NTILE / 2);+ auto tmpb1ptr = tmp + Unroll * NTILE / 2;+ memcpy(tmpb1ptr, bit1ptr + (ir + ib) * NTILE / 8, k_tail * NTILE / 8);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits((utils::bit2x4*)(tmpb4ptr + i * 32), vmask);+ auto vb1 = unpack_1bits((utils::bit1x8*)(tmpb1ptr + i * 8), zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(tmpout + i * 64), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s5_s8_pack2_row(utils::bit4x2* srcptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ __m512i v_zp_y[NReg];+ int constexpr FullRange = 1 << (5 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);++ const auto vindex = _mm512_set_epi8(14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8,+ 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0,+ 14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ memcpy(tmp, zptr, NTILE * sizeof(int8_t));+ memcpy(tmp + NTILE, zptr, NTILE * sizeof(int8_t));+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi16(tmp + i * 32, vindex);+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, PackRow * Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += PackRow * Unroll) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 32, vmask);+ auto vb1 = unpack_1bits(b1ptr + i * 8, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb4ptr = tmp;+ memcpy(tmpb4ptr, srcptr + (ir + ib) * NTILE / 2, k_tail * NTILE / 2);+ auto tmpb1ptr = tmp + Unroll * NTILE / 2;+ memcpy(tmpb1ptr, bit1ptr + (ir + ib) * NTILE / 8, k_tail * NTILE / 8);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits((utils::bit2x4*)(tmpb4ptr + i * 32), vmask);+ auto vb1 = unpack_1bits((utils::bit1x8*)(tmpb1ptr + i * 8), zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(tmpout + i * 64), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s5_s8_pack4_row(utils::bit4x2* srcptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 4;+ __m512i v_zp_y[NReg];+ int constexpr FullRange = 1 << (5 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi32(zptr + i * 16, vindex);+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 32, vmask);+ auto vb1 = unpack_1bits(b1ptr + i * 8, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s5_s8(utils::bit4x2* bit4ptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset, int k_offset,+ int row, int col, int8_t* tmp, size_t tmpsize) {+ if (zpptr) {+ typedef BTLA_CODE (*decompfunc)(utils::bit4x2* bit4ptr, utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row, int8_t* tmp,+ size_t tmpsize);+ decompfunc func = nullptr;+ if (col == NTILE) {+ if constexpr (PackRow == 1) {+ func = &decompress_kblock_s5_s8_pack1_row<NTILE>;+ }+ if constexpr (PackRow == 2) {+ func = &decompress_kblock_s5_s8_pack2_row<NTILE>;+ }+ if constexpr (PackRow == 4) {+ func = &decompress_kblock_s5_s8_pack4_row<NTILE>;+ }+ if (func) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ (*func)(bit4ptr, bit1ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, head_size, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ (*func)(bit4ptr + head_size * NTILE / 2, bit1ptr + head_size * NTILE / 8, zpptr, dstptr + head_size * NTILE,+ blocksize, ldzp, n_offset, head_end, body_size, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ }+ assert(0);+ return BTLA_CODE::NotSupport;+ } else {+ size_t elesize = static_cast<size_t>(row) * col;+ return decompress_s5_s8(bit4ptr, bit1ptr, dstptr, elesize, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s7_s8(utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr,+ int8_t* dstptr, size_t unpack_elt, int8_t* tmp, size_t tmpsize) {+ int constexpr VBits = 512;+ int constexpr VElt = VBits / 8;+ size_t i = 0;+ int constexpr FullRange = 1 << (7 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ size_t elt_pad = utils::padto_le(unpack_elt, VElt);+ for (; i < elt_pad; i += VElt) {+ auto vout = unpack_4bits(bit4ptr + i / 2, vmask);+ auto vb1 = unpack_1bits(bit1ptr + i / 8, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ vout = _mm512_or_si512(vout, vb1);+ vout = _mm512_or_si512(vout, vb2);+ vout = _mm512_sub_epi8(vout, vbias);+ _mm512_storeu_si512((__m512i*)(dstptr + i), vout);+ }+ if (elt_pad < unpack_elt) {+ if (unpack_elt >= VElt) {+ i = unpack_elt - VElt;+ auto vout = unpack_4bits(bit4ptr + i / 2, vmask);+ auto vb1 = unpack_1bits(bit1ptr + i / 8, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ vout = _mm512_or_si512(vout, vb1);+ vout = _mm512_or_si512(vout, vb2);+ vout = _mm512_sub_epi8(vout, vbias);+ _mm512_storeu_si512((__m512i*)(dstptr + i), vout);+ } else {+ ref::decompress_s7_s8(bit4ptr + i / 2, bit2ptr + i / 4, bit1ptr + i / 8, dstptr + i, unpack_elt - i, tmp,+ tmpsize);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s7_s8_pack1_row(utils::bit4x2* srcptr, utils::bit2x4* bit2ptr,+ utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ __m512i v_zp_y[NReg];+ int constexpr FullRange = 1 << (7 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, zptr, NTILE * sizeof(int8_t));+ }+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = _mm512_loadu_si512((const __m512i*)(tmp + i * 64));+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += Unroll) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ auto b2ptr = bit2ptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 32, vmask);+ auto vb1 = unpack_1bits(b1ptr + i * 8, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(b2ptr + i * 16, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_or_si512(v_s8_y, vb2);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }++ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb4ptr = tmp;+ memcpy(tmpb4ptr, srcptr + (ir + ib) * NTILE / 2, k_tail * NTILE / 2);+ auto tmpb1ptr = tmp + Unroll * NTILE / 2;+ memcpy(tmpb1ptr, bit1ptr + (ir + ib) * NTILE / 8, k_tail * NTILE / 8);+ auto tmpb2ptr = tmp + Unroll * NTILE * 3 / 4;+ memcpy(tmpb2ptr, bit2ptr + (ir + ib) * NTILE / 4, k_tail * NTILE / 4);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits((utils::bit2x4*)(tmpb4ptr + i * 32), vmask);+ auto vb1 = unpack_1bits((utils::bit1x8*)(tmpb1ptr + i * 8), zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits((utils::bit2x4*)(tmpb2ptr + i * 16), vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_or_si512(v_s8_y, vb2);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(tmpout + i * 64), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s7_s8_pack2_row(utils::bit4x2* srcptr, utils::bit2x4* bit2ptr,+ utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ __m512i v_zp_y[NReg];+ int constexpr FullRange = 1 << (7 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);++ const auto vindex = _mm512_set_epi8(14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8,+ 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0,+ 14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ memcpy(tmp, zptr, NTILE * sizeof(int8_t));+ memcpy(tmp + NTILE, zptr, NTILE * sizeof(int8_t));+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi16(tmp + i * 32, vindex);+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, PackRow * Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += PackRow * Unroll) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ auto b2ptr = bit2ptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 32, vmask);+ auto vb1 = unpack_1bits(b1ptr + i * 8, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(b2ptr + i * 16, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_or_si512(v_s8_y, vb2);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb4ptr = tmp;+ memcpy(tmpb4ptr, srcptr + (ir + ib) * NTILE / 2, k_tail * NTILE / 2);+ auto tmpb1ptr = tmp + Unroll * NTILE / 2;+ memcpy(tmpb1ptr, bit1ptr + (ir + ib) * NTILE / 8, k_tail * NTILE / 8);+ auto tmpb2ptr = tmp + Unroll * NTILE * 3 / 4;+ memcpy(tmpb2ptr, bit2ptr + (ir + ib) * NTILE / 4, k_tail * NTILE / 4);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits((utils::bit2x4*)(tmpb4ptr + i * 32), vmask);+ auto vb1 = unpack_1bits((utils::bit1x8*)(tmpb1ptr + i * 8), zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits((utils::bit2x4*)(tmpb2ptr + i * 16), vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_or_si512(v_s8_y, vb2);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(tmpout + i * 64), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s7_s8_pack4_row(utils::bit4x2* srcptr, utils::bit2x4* bit2ptr,+ utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row,+ int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 4;+ __m512i v_zp_y[NReg];+ int constexpr FullRange = 1 << (7 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi32(zptr + i * 16, vindex);+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b1ptr = bit1ptr + (ir + ib) * NTILE / 8;+ auto b2ptr = bit2ptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 32, vmask);+ auto vb1 = unpack_1bits(b1ptr + i * 8, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(b2ptr + i * 16, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_or_si512(v_s8_y, vb2);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s7_s8(utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr,+ int8_t* zpptr, int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int col, int8_t* tmp, size_t tmpsize) {+ if (zpptr) {+ typedef BTLA_CODE (*decompfunc)(utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr,+ int8_t* zpptr, int8_t* dstptr, int blocksize, int ldzp, int n_offset, int k_offset,+ int row, int8_t* tmp, size_t tmpsize);+ decompfunc func = nullptr;+ if (col == NTILE) {+ if constexpr (PackRow == 1) {+ func = &decompress_kblock_s7_s8_pack1_row<NTILE>;+ }+ if constexpr (PackRow == 2) {+ func = &decompress_kblock_s7_s8_pack2_row<NTILE>;+ }+ if constexpr (PackRow == 4) {+ func = &decompress_kblock_s7_s8_pack4_row<NTILE>;+ }+ if (func) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ (*func)(bit4ptr, bit2ptr, bit1ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, head_size, tmp,+ tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ (*func)(bit4ptr + head_size * NTILE / 2, bit2ptr + head_size * NTILE / 4, bit1ptr + head_size * NTILE / 8,+ zpptr, dstptr + head_size * NTILE, blocksize, ldzp, n_offset, head_end, body_size, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ }+ assert(0);+ return BTLA_CODE::NotSupport;+ } else {+ size_t elesize = static_cast<size_t>(row) * col;+ return decompress_s7_s8(bit4ptr, bit2ptr, bit1ptr, dstptr, elesize, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s6_s8(utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, int8_t* dstptr,+ size_t unpack_elt, int8_t* tmp, size_t tmpsize) {+ int constexpr VBits = 512;+ int constexpr VElt = VBits / 8;+ size_t i = 0;+ int constexpr FullRange = 1 << (6 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ size_t elt_pad = utils::padto_le(unpack_elt, VElt);+ for (; i < elt_pad; i += VElt) {+ auto vout = unpack_4bits(bit4ptr + i / 2, vmask);+ auto vb1 = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vout = _mm512_or_si512(vout, vb1);+ vout = _mm512_sub_epi8(vout, vbias);+ _mm512_storeu_si512((__m512i*)(dstptr + i), vout);+ }+ if (elt_pad < unpack_elt) {+ if (unpack_elt >= VElt) {+ i = unpack_elt - VElt;+ auto vout = unpack_4bits(bit4ptr + i / 2, vmask);+ auto vb1 = unpack_2bits(bit2ptr + i / 4, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vout = _mm512_or_si512(vout, vb1);+ vout = _mm512_sub_epi8(vout, vbias);+ _mm512_storeu_si512((__m512i*)(dstptr + i), vout);+ } else {+ ref::decompress_s6_s8(bit4ptr + i / 2, bit2ptr + i / 4, dstptr + i, unpack_elt - i, tmp, tmpsize);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s6_s8_pack1_row(utils::bit4x2* srcptr, utils::bit2x4* bit2ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ __m512i v_zp_y[NReg];+ int constexpr FullRange = 1 << (6 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, zptr, NTILE * sizeof(int8_t));+ }+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = _mm512_loadu_si512((const __m512i*)(tmp + i * 64));+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += Unroll) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b2ptr = bit2ptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 32, vmask);+ auto vb1 = unpack_2bits(b2ptr + i * 16, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }++ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb4ptr = tmp;+ memcpy(tmpb4ptr, srcptr + (ir + ib) * NTILE / 2, k_tail * NTILE / 2);+ auto tmpb2ptr = tmp + Unroll * NTILE / 2;+ memcpy(tmpb2ptr, bit2ptr + (ir + ib) * NTILE / 4, k_tail * NTILE / 4);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits((utils::bit2x4*)(tmpb4ptr + i * 32), vmask);+ auto vb1 = unpack_2bits((utils::bit2x4*)(tmpb2ptr + i * 16), vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(tmpout + i * 64), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s6_s8_pack2_row(utils::bit4x2* srcptr, utils::bit2x4* bit2ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 1;+ int constexpr Unroll = 4;+ __m512i v_zp_y[NReg];+ int constexpr FullRange = 1 << (6 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);++ const auto vindex = _mm512_set_epi8(14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8,+ 8, 6, 6, 4, 4, 2, 2, 0, 0, 14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0,+ 14, 14, 12, 12, 10, 10, 8, 8, 6, 6, 4, 4, 2, 2, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ memcpy(tmp, zptr, NTILE * sizeof(int8_t));+ memcpy(tmp + NTILE, zptr, NTILE * sizeof(int8_t));+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi16(tmp + i * 32, vindex);+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ int k_remain_unrll = utils::padto_le(k_remain, PackRow * Unroll);+ int ib = 0;+ for (; ib < k_remain_unrll; ib += PackRow * Unroll) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b2ptr = bit2ptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 32, vmask);+ auto vb1 = unpack_2bits(b2ptr + i * 16, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ int k_tail = k_remain - k_remain_unrll;+ if (k_tail > 0) {+ auto tmpb4ptr = tmp;+ memcpy(tmpb4ptr, srcptr + (ir + ib) * NTILE / 2, k_tail * NTILE / 2);+ auto tmpb2ptr = tmp + Unroll * NTILE / 2;+ memcpy(tmpb2ptr, bit2ptr + (ir + ib) * NTILE / 4, k_tail * NTILE / 4);+ auto tmpout = tmp + Unroll * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits((utils::bit2x4*)(tmpb4ptr + i * 32), vmask);+ auto vb1 = unpack_2bits((utils::bit2x4*)(tmpb2ptr + i * 16), vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(tmpout + i * 64), v_s8_y);+ }+ memcpy(dstptr + (ir + ib) * NTILE, tmpout, k_tail * NTILE);+ }+ }+ return BTLA_CODE::Success;+}++template <int NTILE>+static inline BTLA_CODE decompress_kblock_s6_s8_pack4_row(utils::bit4x2* srcptr, utils::bit2x4* bit2ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ static_assert((NTILE % VLen) == 0);+ int constexpr PackRow = 4;+ __m512i v_zp_y[NReg];+ int constexpr FullRange = 1 << (6 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ for (int ir = 0; ir < row; ir += blocksize) {+ auto zptr = zpptr + (k_offset + ir) / blocksize * ldzp + n_offset;+ for (int i = 0; i < NReg; i++) {+ v_zp_y[i] = load_zp_epi8_broadcast_epi32(zptr + i * 16, vindex);+ v_zp_y[i] = _mm512_add_epi8(v_zp_y[i], vbias);+ }+ int k_remain = utils::remainsize(ir, row, blocksize);+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b4ptr = srcptr + (ir + ib) * NTILE / 2;+ auto b2ptr = bit2ptr + (ir + ib) * NTILE / 4;+ for (int i = 0; i < NReg; i++) {+ auto v_s8_y = unpack_4bits(b4ptr + i * 32, vmask);+ auto vb1 = unpack_2bits(b2ptr + i * 16, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ v_s8_y = _mm512_or_si512(v_s8_y, vb1);+ v_s8_y = _mm512_sub_epi8(v_s8_y, v_zp_y[i]);+ _mm512_storeu_si512((__m512i*)(dstptr + i * 64 + (ir + ib) * NTILE), v_s8_y);+ }+ }+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s6_s8(utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset, int k_offset,+ int row, int col, int8_t* tmp, size_t tmpsize) {+ if (zpptr) {+ typedef BTLA_CODE (*decompfunc)(utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row, int8_t* tmp,+ size_t tmpsize);+ decompfunc func = nullptr;+ if (col == NTILE) {+ if constexpr (PackRow == 1) {+ func = &decompress_kblock_s6_s8_pack1_row<NTILE>;+ }+ if constexpr (PackRow == 2) {+ func = &decompress_kblock_s6_s8_pack2_row<NTILE>;+ }+ if constexpr (PackRow == 4) {+ func = &decompress_kblock_s6_s8_pack4_row<NTILE>;+ }+ if (func) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ (*func)(bit4ptr, bit2ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, head_size, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ (*func)(bit4ptr + head_size * NTILE / 2, bit2ptr + head_size * NTILE / 4, zpptr, dstptr + head_size * NTILE,+ blocksize, ldzp, n_offset, head_end, body_size, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ }+ assert(0);+ return BTLA_CODE::NotSupport;+ } else {+ size_t elesize = static_cast<size_t>(row) * col;+ return decompress_s6_s8(bit4ptr, bit2ptr, dstptr, elesize, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s8_fp_row(int8_t* srcptr, DST_T* dstptr, int row, void* scales_, BTLA_DTYPE sdtype,+ int8_t* zero_points, int k_offset, int n_offset, int blocksize, int ldzp,+ int8_t* tmp, size_t tmpsize) {+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ const auto vshuf_index_low = _mm512_set_epi32(7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0);+ const auto vshuf_index_high = _mm512_set_epi32(15, 15, 14, 14, 13, 13, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8);+ if (zero_points == nullptr) {+ for (int ir = 0; ir < row; ir += blocksize) {+ int k_remain = utils::remainsize(ir, row, blocksize);+ int ele_off = (k_offset + ir) / blocksize * ldzp + n_offset;+ if constexpr (PackRow == 1) {+ __m512 vscale_y[NReg];+ if (sdtype == BTLA_DTYPE::F32) {+ auto sptr = (float*)scales_ + ele_off;+ for (int i = 0; i < NReg; i++) vscale_y[i] = _mm512_loadu_ps(sptr + i * VLen);+ } else if (sdtype == BTLA_DTYPE::BF16) {+ auto sptr = (utils::bf16*)scales_ + ele_off;+ for (int i = 0; i < NReg; i++) vscale_y[i] = load_bf16_fp32(sptr + i * VLen);+ } else if (sdtype == BTLA_DTYPE::F16) {+ auto sptr = (utils::fp16*)scales_ + ele_off;+ for (int i = 0; i < NReg; i++) vscale_y[i] = load_fp16_fp32(sptr + i * VLen);+ }+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b8ptr = srcptr + (ir + ib) * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto vdeq_y = dequant_s8_fp(b8ptr + i * VLen, vscale_y[i]);+ store_fp32_T(vdeq_y, dstptr + (ir + ib) * NTILE + i * VLen);+ }+ }+ } else if constexpr (PackRow == 4) {+ __m512 vscale_y[PackRow * NReg];+ for (int i = 0; i < NReg; i++) {+ __m512 vraw;+ if (sdtype == BTLA_DTYPE::F32) {+ auto sptr = (float*)scales_ + ele_off;+ vraw = _mm512_loadu_ps(sptr + i * VLen);+ } else if (sdtype == BTLA_DTYPE::BF16) {+ auto sptr = (utils::bf16*)scales_ + ele_off;+ vraw = load_bf16_fp32(sptr + i * VLen);+ } else if (sdtype == BTLA_DTYPE::F16) {+ auto sptr = (utils::fp16*)scales_ + ele_off;+ vraw = load_fp16_fp32(sptr + i * VLen);+ } else {+ assert(0);+ }+ auto vcast_y = broadcast_ps_1_2<true>(vraw, vshuf_index_high, vshuf_index_low);+ vscale_y[i * PackRow + 0] = broadcast_ps_1_2<true>(vcast_y, vshuf_index_high, vshuf_index_low);+ vscale_y[i * PackRow + 1] = broadcast_ps_1_2<false>(vcast_y, vshuf_index_high, vshuf_index_low);+ vcast_y = broadcast_ps_1_2<false>(vraw, vshuf_index_high, vshuf_index_low);+ vscale_y[i * PackRow + 2] = broadcast_ps_1_2<true>(vcast_y, vshuf_index_high, vshuf_index_low);+ vscale_y[i * PackRow + 3] = broadcast_ps_1_2<false>(vcast_y, vshuf_index_high, vshuf_index_low);+ }+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b8ptr = srcptr + (ir + ib) * NTILE;+ for (int i = 0; i < NReg; i++) {+ for (int ip = 0; ip < PackRow; ip++) {+ auto vdeq_y = dequant_s8_fp(b8ptr + i * VLen * PackRow + ip * VLen, vscale_y[i * PackRow + ip]);+ store_fp32_T(vdeq_y, dstptr + (ir + ib) * NTILE + i * VLen * PackRow + ip * VLen);+ }+ }+ }+ } else if constexpr (PackRow == 2) {+ __m512 vscale_y[PackRow * NReg];+ for (int i = 0; i < NReg; i++) {+ __m512 vraw;+ if (sdtype == BTLA_DTYPE::F32) {+ auto sptr = (float*)scales_ + ele_off;+ vraw = _mm512_loadu_ps(sptr + i * VLen);+ } else if (sdtype == BTLA_DTYPE::BF16) {+ auto sptr = (utils::bf16*)scales_ + ele_off;+ vraw = load_bf16_fp32(sptr + i * VLen);+ } else if (sdtype == BTLA_DTYPE::F16) {+ auto sptr = (utils::fp16*)scales_ + ele_off;+ vraw = load_fp16_fp32(sptr + i * VLen);+ }+ vscale_y[i * PackRow + 0] = broadcast_ps_1_2<true>(vraw, vshuf_index_high, vshuf_index_low);+ vscale_y[i * PackRow + 1] = broadcast_ps_1_2<false>(vraw, vshuf_index_high, vshuf_index_low);+ }+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b8ptr = srcptr + (ir + ib) * NTILE;+ for (int i = 0; i < NReg; i++) {+ for (int ip = 0; ip < PackRow; ip++) {+ auto vdeq_y = dequant_s8_fp(b8ptr + i * VLen * PackRow + ip * VLen, vscale_y[i * PackRow + ip]);+ store_fp32_T(vdeq_y, dstptr + (ir + ib) * NTILE + i * VLen * PackRow + ip * VLen);+ }+ }+ }+ } else {+ assert(0);+ }+ }+ return BTLA_CODE::Success;+ } else {+ for (int ir = 0; ir < row; ir += blocksize) {+ int k_remain = utils::remainsize(ir, row, blocksize);+ int ele_off = (k_offset + ir) / blocksize * ldzp + n_offset;+ if constexpr (PackRow == 1) {+ __m512 vscale_y[NReg];+ if (sdtype == BTLA_DTYPE::F32) {+ auto sptr = (float*)scales_ + ele_off;+ for (int i = 0; i < NReg; i++) vscale_y[i] = _mm512_loadu_ps(sptr + i * VLen);+ } else if (sdtype == BTLA_DTYPE::BF16) {+ auto sptr = (utils::bf16*)scales_ + ele_off;+ for (int i = 0; i < NReg; i++) vscale_y[i] = load_bf16_fp32(sptr + i * VLen);+ } else if (sdtype == BTLA_DTYPE::F16) {+ auto sptr = (utils::fp16*)scales_ + ele_off;+ for (int i = 0; i < NReg; i++) vscale_y[i] = load_fp16_fp32(sptr + i * VLen);+ }+ __m512i vzp_y[NReg];+ for (int i = 0; i < NReg; i++) vzp_y[i] = load_s8_s32(zero_points + ele_off + i * VLen);+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b8ptr = srcptr + (ir + ib) * NTILE;+ for (int i = 0; i < NReg; i++) {+ auto vdeq_y = dequant_s8_fp<true>(b8ptr + i * VLen, vscale_y[i], vzp_y[i]);+ store_fp32_T(vdeq_y, dstptr + (ir + ib) * NTILE + i * VLen);+ }+ }+ } else if constexpr (PackRow == 4) {+ __m512 vscale_y[PackRow * NReg];+ __m512i vzp_y[PackRow * NReg];+ for (int i = 0; i < NReg; i++) {+ __m512 vraw;+ if (sdtype == BTLA_DTYPE::F32) {+ auto sptr = (float*)scales_ + ele_off;+ vraw = _mm512_loadu_ps(sptr + i * VLen);+ } else if (sdtype == BTLA_DTYPE::BF16) {+ auto sptr = (utils::bf16*)scales_ + ele_off;+ vraw = load_bf16_fp32(sptr + i * VLen);+ } else if (sdtype == BTLA_DTYPE::F16) {+ auto sptr = (utils::fp16*)scales_ + ele_off;+ vraw = load_fp16_fp32(sptr + i * VLen);+ } else {+ assert(0);+ }+ auto vcast_y = broadcast_ps_1_2<true>(vraw, vshuf_index_high, vshuf_index_low);+ vscale_y[i * PackRow + 0] = broadcast_ps_1_2<true>(vcast_y, vshuf_index_high, vshuf_index_low);+ vscale_y[i * PackRow + 1] = broadcast_ps_1_2<false>(vcast_y, vshuf_index_high, vshuf_index_low);+ vcast_y = broadcast_ps_1_2<false>(vraw, vshuf_index_high, vshuf_index_low);+ vscale_y[i * PackRow + 2] = broadcast_ps_1_2<true>(vcast_y, vshuf_index_high, vshuf_index_low);+ vscale_y[i * PackRow + 3] = broadcast_ps_1_2<false>(vcast_y, vshuf_index_high, vshuf_index_low);++ auto tmp = load_s8_s32(zero_points + ele_off + i * VLen);+ auto vcasti_y = broadcast_epi32_1_2<true>(tmp, vshuf_index_high, vshuf_index_low);+ vzp_y[i * PackRow + 0] = broadcast_epi32_1_2<true>(vcasti_y, vshuf_index_high, vshuf_index_low);+ vzp_y[i * PackRow + 1] = broadcast_epi32_1_2<false>(vcasti_y, vshuf_index_high, vshuf_index_low);+ vcasti_y = broadcast_epi32_1_2<false>(tmp, vshuf_index_high, vshuf_index_low);+ vzp_y[i * PackRow + 2] = broadcast_epi32_1_2<true>(vcasti_y, vshuf_index_high, vshuf_index_low);+ vzp_y[i * PackRow + 3] = broadcast_epi32_1_2<false>(vcasti_y, vshuf_index_high, vshuf_index_low);+ }+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b8ptr = srcptr + (ir + ib) * NTILE;+ for (int i = 0; i < NReg; i++) {+ for (int ip = 0; ip < PackRow; ip++) {+ auto vdeq_y = dequant_s8_fp<true>(b8ptr + i * VLen * PackRow + ip * VLen, vscale_y[i * PackRow + ip],+ vzp_y[i * PackRow + ip]);+ store_fp32_T(vdeq_y, dstptr + (ir + ib) * NTILE + i * VLen * PackRow + ip * VLen);+ }+ }+ }+ } else if constexpr (PackRow == 2) {+ __m512 vscale_y[PackRow * NReg];+ __m512i vzp_y[PackRow * NReg];+ for (int i = 0; i < NReg; i++) {+ __m512 vraw;+ if (sdtype == BTLA_DTYPE::F32) {+ auto sptr = (float*)scales_ + ele_off;+ vraw = _mm512_loadu_ps(sptr + i * VLen);+ } else if (sdtype == BTLA_DTYPE::BF16) {+ auto sptr = (utils::bf16*)scales_ + ele_off;+ vraw = load_bf16_fp32(sptr + i * VLen);+ } else if (sdtype == BTLA_DTYPE::F16) {+ auto sptr = (utils::fp16*)scales_ + ele_off;+ vraw = load_fp16_fp32(sptr + i * VLen);+ }+ vscale_y[i * PackRow + 0] = broadcast_ps_1_2<true>(vraw, vshuf_index_high, vshuf_index_low);+ vscale_y[i * PackRow + 1] = broadcast_ps_1_2<false>(vraw, vshuf_index_high, vshuf_index_low);+ auto tmp = load_s8_s32(zero_points + ele_off + i * VLen);+ vzp_y[i * PackRow + 0] = broadcast_epi32_1_2<true>(tmp, vshuf_index_high, vshuf_index_low);+ vzp_y[i * PackRow + 1] = broadcast_epi32_1_2<false>(tmp, vshuf_index_high, vshuf_index_low);+ }+ for (int ib = 0; ib < k_remain; ib += PackRow) {+ auto b8ptr = srcptr + (ir + ib) * NTILE;+ for (int i = 0; i < NReg; i++) {+ for (int ip = 0; ip < PackRow; ip++) {+ auto vdeq_y = dequant_s8_fp<true>(b8ptr + i * VLen * PackRow + ip * VLen, vscale_y[i * PackRow + ip],+ vzp_y[i * PackRow + ip]);+ store_fp32_T(vdeq_y, dstptr + (ir + ib) * NTILE + i * VLen * PackRow + ip * VLen);+ }+ }+ }+ } else {+ assert(0);+ }+ }+ return BTLA_CODE::Success;+ }+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s8_fp(int8_t* srcptr, DST_T* dstptr, int row, int col, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ auto ret = BTLA_CODE::NotSupport;+ if (col == NTILE) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(srcptr, dstptr, head_size, scales_, sdtype, zero_points,+ k_offset, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(srcptr + head_size * NTILE, dstptr + head_size * NTILE,+ body_size, scales_, sdtype, zero_points, head_end, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ return ret;+}+template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s4_fp_row(utils::int4x2* srcptr, DST_T* dstptr, int row, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ auto ret = decompress_kblock_s4_s8<PackRow, NTILE>(srcptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset, k_offset,+ row, NTILE, tmp, tmpsize);+ assert(ret == BTLA_CODE::Success);+ return decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(tmps8ptr, dstptr, row, scales_, sdtype, nullptr, k_offset,+ n_offset, blocksize, ldzp, tmp, tmpsize);+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s4_fp(utils::int4x2* srcptr, DST_T* dstptr, int row, int col, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ auto ret = BTLA_CODE::NotSupport;+ if (col == NTILE) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ decompress_kblock_s4_fp_row<PackRow, NTILE, DST_T>(srcptr, dstptr, head_size, scales_, sdtype, zero_points,+ k_offset, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ decompress_kblock_s4_fp_row<PackRow, NTILE, DST_T>(srcptr + head_size * NTILE / 2, dstptr + head_size * NTILE,+ body_size, scales_, sdtype, zero_points, head_end, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ return ret;+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s2_fp_row(utils::bit2x4* b2ptr, DST_T* dstptr, int row, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ auto ret = decompress_kblock_s2_s8<PackRow, NTILE>(b2ptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset, k_offset,+ row, NTILE, tmp, tmpsize);+ assert(ret == BTLA_CODE::Success);+ return decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(tmps8ptr, dstptr, row, scales_, sdtype, nullptr, k_offset,+ n_offset, blocksize, ldzp, tmp, tmpsize);+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s2_fp(utils::bit2x4* b2ptr, DST_T* dstptr, int row, int col, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ auto ret = BTLA_CODE::NotSupport;+ if (col == NTILE) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ decompress_kblock_s2_fp_row<PackRow, NTILE, DST_T>(b2ptr, dstptr, head_size, scales_, sdtype, zero_points,+ k_offset, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ decompress_kblock_s2_fp_row<PackRow, NTILE, DST_T>(b2ptr + head_size * NTILE / 4, dstptr + head_size * NTILE,+ body_size, scales_, sdtype, zero_points, head_end, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ return ret;+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s3_fp_row(utils::bit2x4* b2ptr, utils::bit1x8* b1ptr, DST_T* dstptr, int row,+ void* scales_, BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset,+ int n_offset, int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ auto ret = decompress_kblock_s3_s8<PackRow, NTILE>(b2ptr, b1ptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset,+ k_offset, row, NTILE, tmp, tmpsize);+ assert(ret == BTLA_CODE::Success);+ return decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(tmps8ptr, dstptr, row, scales_, sdtype, nullptr, k_offset,+ n_offset, blocksize, ldzp, tmp, tmpsize);+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s3_fp(utils::bit2x4* b2ptr, utils::bit1x8* b1ptr, DST_T* dstptr, int row, int col,+ void* scales_, BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset,+ int n_offset, int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ auto ret = BTLA_CODE::NotSupport;+ if (col == NTILE) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ decompress_kblock_s3_fp_row<PackRow, NTILE, DST_T>(b2ptr, b1ptr, dstptr, head_size, scales_, sdtype, zero_points,+ k_offset, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ decompress_kblock_s3_fp_row<PackRow, NTILE, DST_T>(+ b2ptr + head_size * NTILE / 4, b1ptr + head_size * NTILE / 8, dstptr + head_size * NTILE, body_size, scales_,+ sdtype, zero_points, head_end, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ return ret;+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s1_fp_row(utils::bit1x8* b1ptr, DST_T* dstptr, int row, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ auto ret = decompress_kblock_s1_s8<PackRow, NTILE>(b1ptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset, k_offset,+ row, NTILE, tmp, tmpsize);+ assert(ret == BTLA_CODE::Success);+ return decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(tmps8ptr, dstptr, row, scales_, sdtype, nullptr, k_offset,+ n_offset, blocksize, ldzp, tmp, tmpsize);+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s1_fp(utils::bit1x8* b1ptr, DST_T* dstptr, int row, int col, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ auto ret = BTLA_CODE::NotSupport;+ if (col == NTILE) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ decompress_kblock_s1_fp_row<PackRow, NTILE, DST_T>(b1ptr, dstptr, head_size, scales_, sdtype, zero_points,+ k_offset, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ decompress_kblock_s1_fp_row<PackRow, NTILE, DST_T>(b1ptr + head_size * NTILE / 8, dstptr + head_size * NTILE,+ body_size, scales_, sdtype, zero_points, head_end, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ return ret;+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s5_fp_row(utils::bit4x2* b4ptr, utils::bit1x8* b1ptr, DST_T* dstptr, int row,+ void* scales_, BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset,+ int n_offset, int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ auto ret = decompress_kblock_s5_s8<PackRow, NTILE>(b4ptr, b1ptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset,+ k_offset, row, NTILE, tmp, tmpsize);+ assert(ret == BTLA_CODE::Success);+ return decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(tmps8ptr, dstptr, row, scales_, sdtype, nullptr, k_offset,+ n_offset, blocksize, ldzp, tmp, tmpsize);+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s5_fp(utils::bit4x2* b4ptr, utils::bit1x8* b1ptr, DST_T* dstptr, int row, int col,+ void* scales_, BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset,+ int n_offset, int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ auto ret = BTLA_CODE::NotSupport;+ if (col == NTILE) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ decompress_kblock_s5_fp_row<PackRow, NTILE, DST_T>(b4ptr, b1ptr, dstptr, head_size, scales_, sdtype, zero_points,+ k_offset, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ decompress_kblock_s5_fp_row<PackRow, NTILE, DST_T>(+ b4ptr + head_size * NTILE / 2, b1ptr + head_size * NTILE / 8, dstptr + head_size * NTILE, body_size, scales_,+ sdtype, zero_points, head_end, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ return ret;+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s6_fp_row(utils::bit4x2* b4ptr, utils::bit2x4* b2ptr, DST_T* dstptr, int row,+ void* scales_, BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset,+ int n_offset, int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ auto ret = decompress_kblock_s6_s8<PackRow, NTILE>(b4ptr, b2ptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset,+ k_offset, row, NTILE, tmp, tmpsize);+ assert(ret == BTLA_CODE::Success);+ return decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(tmps8ptr, dstptr, row, scales_, sdtype, nullptr, k_offset,+ n_offset, blocksize, ldzp, tmp, tmpsize);+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s6_fp(utils::bit4x2* b4ptr, utils::bit2x4* b2ptr, DST_T* dstptr, int row, int col,+ void* scales_, BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset,+ int n_offset, int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ auto ret = BTLA_CODE::NotSupport;+ if (col == NTILE) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ decompress_kblock_s6_fp_row<PackRow, NTILE, DST_T>(b4ptr, b2ptr, dstptr, head_size, scales_, sdtype, zero_points,+ k_offset, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ decompress_kblock_s6_fp_row<PackRow, NTILE, DST_T>(+ b4ptr + head_size * NTILE / 2, b2ptr + head_size * NTILE / 4, dstptr + head_size * NTILE, body_size, scales_,+ sdtype, zero_points, head_end, n_offset, blocksize, ldzp, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ return ret;+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s7_fp_row(utils::bit4x2* b4ptr, utils::bit2x4* b2ptr, utils::bit1x8* b1ptr,+ DST_T* dstptr, int row, void* scales_, BTLA_DTYPE sdtype,+ int8_t* zero_points, int k_offset, int n_offset, int blocksize, int ldzp,+ int8_t* tmp, size_t tmpsize) {+ int constexpr NReg = NTILE / 8;+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ auto ret = decompress_kblock_s7_s8<PackRow, NTILE>(b4ptr, b2ptr, b1ptr, zero_points, tmps8ptr, blocksize, ldzp,+ n_offset, k_offset, row, NTILE, tmp, tmpsize);+ assert(ret == BTLA_CODE::Success);+ return decompress_kblock_s8_fp_row<PackRow, NTILE, DST_T>(tmps8ptr, dstptr, row, scales_, sdtype, nullptr, k_offset,+ n_offset, blocksize, ldzp, tmp, tmpsize);+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s7_fp(utils::bit4x2* b4ptr, utils::bit2x4* b2ptr, utils::bit1x8* b1ptr,+ DST_T* dstptr, int row, int col, void* scales_, BTLA_DTYPE sdtype,+ int8_t* zero_points, int k_offset, int n_offset, int blocksize, int ldzp,+ int8_t* tmp, size_t tmpsize) {+ auto ret = BTLA_CODE::NotSupport;+ if (col == NTILE) {+ int head_end = utils::padto(k_offset, blocksize);+ head_end = std::min(head_end, k_offset + row);+ int head_size = head_end - k_offset;+ if (head_size > 0) {+ decompress_kblock_s7_fp_row<PackRow, NTILE, DST_T>(b4ptr, b2ptr, b1ptr, dstptr, head_size, scales_, sdtype,+ zero_points, k_offset, n_offset, blocksize, ldzp, tmp,+ tmpsize);+ }+ int body_size = row - head_size;+ if (body_size > 0) {+ decompress_kblock_s7_fp_row<PackRow, NTILE, DST_T>(b4ptr + head_size * NTILE / 2, b2ptr + head_size * NTILE / 4,+ b1ptr + head_size * NTILE / 8, dstptr + head_size * NTILE,+ body_size, scales_, sdtype, zero_points, head_end, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+ }+ return ret;+}++template <typename ScaleT, int NReg, int MTILE>+static inline void gemv_dequant_s32fp32(const float* asptr, int ldzp, const ScaleT* bsptr, __m512i* iacc,+ __m512* facc) {+ __m512 v_a_scale[MTILE];+ for (int im = 0; im < MTILE; im++) {+ v_a_scale[im] = _mm512_set1_ps(*(asptr + im * ldzp));+ }++ for (int i = 0; i < NReg; i++) {+ __m512 v_b_scale = load_T_fp32(bsptr + i * 16);+ for (int im = 0; im < MTILE; im++) {+ auto vtmp = _mm512_mul_ps(v_a_scale[im], v_b_scale);+ auto tmp = _mm512_cvtepi32_ps(iacc[im * NReg + i]);+ facc[im * NReg + i] = _mm512_fmadd_ps(tmp, vtmp, facc[im * NReg + i]);+ }+ }+}++template <int NReg, int MReg>+static inline void gemv_remove_zp(const uint8_t* azptr, int ldzp, __m512i* iacc, __m512i* bacc) {+ if constexpr (MReg == 1) {+ auto zp = int(azptr[0]);+ __m512i v_a_zp = _mm512_set1_epi32(zp);+ for (int in = 0; in < NReg; in++) {+ auto vtmp = _mm512_mullo_epi32(v_a_zp, bacc[in]);+ iacc[in] = _mm512_sub_epi32(iacc[in], vtmp);+ }+ } else {+ __m512i v_a_zp[MReg];+ for (int im = 0; im < MReg; im++) {+ auto zp = int(azptr[im * ldzp]);+ v_a_zp[im] = _mm512_set1_epi32(zp);+ for (int in = 0; in < NReg; in++) {+ auto vtmp = _mm512_mullo_epi32(v_a_zp[im], bacc[in]);+ iacc[im * NReg + in] = _mm512_sub_epi32(iacc[im * NReg + in], vtmp);+ }+ }+ }+}++template <int MTILE, int NReg, int Unroll>+static inline void accumulate_fp32_s8_fp32(const float* Aptr, int lda, int8_t* Bptr, __m512* vacc, __m512* vsca) {+ if constexpr (MTILE == 1) {+ for (int ikk = 0; ikk < Unroll; ikk++) {+ __m512 va = _mm512_set1_ps(*(Aptr + ikk));+ for (int i = 0; i < NReg; i++) {+ auto ftmp = load_s8_fp32(Bptr + i * 16 + ikk * NReg * 16);+ ftmp = _mm512_mul_ps(ftmp, vsca[i]);+ vacc[i] = _mm512_fmadd_ps(va, ftmp, vacc[i]);+ }+ }+ } else {+ for (int ikk = 0; ikk < Unroll; ikk++) {+ __m512 va[MTILE];+ for (int i = 0; i < NReg; i++) {+ auto ftmp = load_s8_fp32(Bptr + i * 16 + ikk * NReg * 16);+ ftmp = _mm512_mul_ps(ftmp, vsca[i]);+ for (int im = 0; im < MTILE; im++) {+ if (i == 0) {+ va[im] = _mm512_set1_ps(*(Aptr + ikk + im * lda));+ }+ vacc[im * NReg + i] = _mm512_fmadd_ps(va[im], ftmp, vacc[im * NReg + i]);+ }+ }+ }+ }+}++template <int MTILE, int NReg, int Unroll>+static inline void accumulate_fp32_s8_fp32(const float* Aptr, int lda, int8_t* Bptr, __m512* vacc_loc) {+ if constexpr (MTILE == 1) {+ for (int ikk = 0; ikk < Unroll; ikk++) {+ __m512 va = _mm512_set1_ps(*(Aptr + ikk));+ for (int i = 0; i < NReg; i++) {+ auto ftmp = load_s8_fp32(Bptr + i * 16 + ikk * NReg * 16);+ vacc_loc[i] = _mm512_fmadd_ps(va, ftmp, vacc_loc[i]);+ }+ }+ } else {+ for (int ikk = 0; ikk < Unroll; ikk++) {+ __m512 va[MTILE];+ for (int i = 0; i < NReg; i++) {+ auto ftmp = load_s8_fp32(Bptr + i * 16 + ikk * NReg * 16);+ for (int im = 0; im < MTILE; im++) {+ if (i == 0) {+ va[im] = _mm512_set1_ps(*(Aptr + ikk + im * lda));+ }+ vacc_loc[im * NReg + i] = _mm512_fmadd_ps(va[im], ftmp, vacc_loc[im * NReg + i]);+ }+ }+ }+ }+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_4bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto& b4ptr = B.b4ptr;+ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ // Initialize accumulator with zeros+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (4 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ __m512 v_b_scale[NReg];+ for (int i = 0; i < NReg; i++) {+ v_b_scale[i] = load_T_fp32(bsptr + i * VLen);+ }++ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);++ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;++ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, bzptr, NTILE);+ }+ for (int i = 0; i < NReg; i++) {+ bzp[i] = _mm512_loadu_si512((const __m512i*)(tmp + i * 64));+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits((void*)(b4ptr + i * 32 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ _mm512_storeu_si512((__m512i*)(tmp + 64 * i), vb);+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc, v_b_scale);+ }++ } else {+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits((void*)(b4ptr + i * 32 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm512_sub_epi8(vb, vbias);+ _mm512_storeu_si512((__m512i*)(tmp + 64 * i), vb);+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc, v_b_scale);+ }+ }+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_2bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b2ptr = (utils::bit2x4*)B.b2ptr;+ int constexpr VLen = 16;+ int blks = k / blocksize;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (2 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);++ int constexpr KTILE = 1;+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;++ __m512 acc_loc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc_loc[i] = _mm512_setzero_ps();+ }+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);++ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, bzptr, NTILE);+ }+ for (int i = 0; i < NReg; i++) {+ bzp[i] = _mm512_loadu_si512((const __m512i*)(tmp + i * 64));+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ _mm512_storeu_si512((__m512i*)(tmp + 64 * i), vb);+ b2ptr += VLen * Unroll / 4;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }++ } else {+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb = _mm512_sub_epi8(vb, vbias);+ _mm512_storeu_si512((__m512i*)(tmp + 64 * i), vb);+ b2ptr += VLen * Unroll / 4;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }+ }++ __m512 v_b_scale[NReg];+ for (int i = 0; i < NReg; i++) {+ v_b_scale[i] = load_T_fp32(bsptr + i * VLen);+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NReg; in++) {+ acc[im * NReg + in] = _mm512_fmadd_ps(acc_loc[im * NReg + in], v_b_scale[in], acc[im * NReg + in]);+ }+ }+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_3bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b2ptr = (utils::bit2x4*)B.b2ptr;+ auto b1ptr = (utils::bit1x8*)B.b1ptr;++ int constexpr VLen = 16;+ int blks = k / blocksize;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (3 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ int constexpr KTILE = 1;+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;++ __m512 acc_loc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc_loc[i] = _mm512_setzero_ps();+ }+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);++ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, bzptr, NTILE);+ }+ for (int i = 0; i < NReg; i++) {+ bzp[i] = _mm512_loadu_si512((const __m512i*)(tmp + i * 64));+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ _mm512_storeu_si512((__m512i*)(tmp + 64 * i), vb);+ b2ptr += VLen * Unroll / 4;+ b1ptr += VLen * Unroll / 8;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }++ } else {+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ _mm512_storeu_si512((__m512i*)(tmp + 64 * i), vb);+ b2ptr += VLen * Unroll / 4;+ b1ptr += VLen * Unroll / 8;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }+ }++ __m512 v_b_scale[NReg];+ for (int i = 0; i < NReg; i++) {+ v_b_scale[i] = load_T_fp32(bsptr + i * VLen);+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NReg; in++) {+ acc[im * NReg + in] = _mm512_fmadd_ps(acc_loc[im * NReg + in], v_b_scale[in], acc[im * NReg + in]);+ }+ }+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_1bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b1ptr = (utils::bit1x8*)B.b1ptr;++ int constexpr VLen = 16;+ int blks = k / blocksize;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (1 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ int constexpr KTILE = 1;+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;++ __m512 acc_loc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc_loc[i] = _mm512_setzero_ps();+ }+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);++ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, bzptr, NTILE);+ }+ for (int i = 0; i < NReg; i++) {+ bzp[i] = _mm512_loadu_si512((const __m512i*)(tmp + i * 64));+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, bzp[i]);+ _mm512_storeu_si512((__m512i*)(tmp + 64 * i), vb1);+ b1ptr += VLen * Unroll / 8;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }++ } else {+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, vbias);+ _mm512_storeu_si512((__m512i*)(tmp + 64 * i), vb1);+ b1ptr += VLen * Unroll / 8;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }+ }++ __m512 v_b_scale[NReg];+ for (int i = 0; i < NReg; i++) {+ v_b_scale[i] = load_T_fp32(bsptr + i * VLen);+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NReg; in++) {+ acc[im * NReg + in] = _mm512_fmadd_ps(acc_loc[im * NReg + in], v_b_scale[in], acc[im * NReg + in]);+ }+ }+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_5bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = (utils::bit4x2*)B.b4ptr;+ auto b1ptr = (utils::bit1x8*)B.b1ptr;++ int constexpr VLen = 16;+ int blks = k / blocksize;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (5 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ int constexpr KTILE = 1;+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;++ __m512 acc_loc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc_loc[i] = _mm512_setzero_ps();+ }+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);++ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, bzptr, NTILE);+ }+ for (int i = 0; i < NReg; i++) {+ bzp[i] = _mm512_loadu_si512((const __m512i*)(tmp + i * 64));+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ _mm512_storeu_si512((__m512i*)(tmp + 64 * i), vb);+ b4ptr += VLen * Unroll / 2;+ b1ptr += VLen * Unroll / 8;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }++ } else {+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ _mm512_storeu_si512((__m512i*)(tmp + 64 * i), vb);+ b4ptr += VLen * Unroll / 2;+ b1ptr += VLen * Unroll / 8;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }+ }++ __m512 v_b_scale[NReg];+ for (int i = 0; i < NReg; i++) {+ v_b_scale[i] = load_T_fp32(bsptr + i * VLen);+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NReg; in++) {+ acc[im * NReg + in] = _mm512_fmadd_ps(acc_loc[im * NReg + in], v_b_scale[in], acc[im * NReg + in]);+ }+ }+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_6bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = (utils::bit4x2*)B.b4ptr;+ auto b2ptr = (utils::bit2x4*)B.b2ptr;++ int constexpr VLen = 16;+ int blks = k / blocksize;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (6 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ int constexpr KTILE = 1;+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;++ __m512 acc_loc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc_loc[i] = _mm512_setzero_ps();+ }+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);++ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, bzptr, NTILE);+ }+ for (int i = 0; i < NReg; i++) {+ bzp[i] = _mm512_loadu_si512((const __m512i*)(tmp + i * 64));+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ _mm512_storeu_si512((__m512i*)(tmp + 64 * i), vb);+ b4ptr += VLen * Unroll / 2;+ b2ptr += VLen * Unroll / 4;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }++ } else {+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ _mm512_storeu_si512((__m512i*)(tmp + 64 * i), vb);+ b4ptr += VLen * Unroll / 2;+ b2ptr += VLen * Unroll / 4;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }+ }++ __m512 v_b_scale[NReg];+ for (int i = 0; i < NReg; i++) {+ v_b_scale[i] = load_T_fp32(bsptr + i * VLen);+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NReg; in++) {+ acc[im * NReg + in] = _mm512_fmadd_ps(acc_loc[im * NReg + in], v_b_scale[in], acc[im * NReg + in]);+ }+ }+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_7bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = (utils::bit4x2*)B.b4ptr;+ auto b2ptr = (utils::bit2x4*)B.b2ptr;+ auto b1ptr = (utils::bit1x8*)B.b1ptr;++ int constexpr VLen = 16;+ int blks = k / blocksize;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (7 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);++ int constexpr KTILE = 1;+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;++ __m512 acc_loc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc_loc[i] = _mm512_setzero_ps();+ }+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);++ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < Unroll; i++) {+ memcpy(tmp + i * NTILE, bzptr, NTILE);+ }+ for (int i = 0; i < NReg; i++) {+ bzp[i] = _mm512_loadu_si512((const __m512i*)(tmp + i * 64));+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_or_si512(vb, vb2);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ _mm512_storeu_si512((__m512i*)(tmp + 64 * i), vb);+ b4ptr += VLen * Unroll / 2;+ b2ptr += VLen * Unroll / 4;+ b1ptr += VLen * Unroll / 8;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }++ } else {+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_or_si512(vb, vb2);+ vb = _mm512_sub_epi8(vb, vbias);+ _mm512_storeu_si512((__m512i*)(tmp + 64 * i), vb);+ b4ptr += VLen * Unroll / 2;+ b2ptr += VLen * Unroll / 4;+ b1ptr += VLen * Unroll / 8;+ }+ accumulate_fp32_s8_fp32<MTILE, NReg, Unroll>(A + ib * blocksize + ik, lda, tmp, acc_loc);+ }+ }++ __m512 v_b_scale[NReg];+ for (int i = 0; i < NReg; i++) {+ v_b_scale[i] = load_T_fp32(bsptr + i * VLen);+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NReg; in++) {+ acc[im * NReg + in] = _mm512_fmadd_ps(acc_loc[im * NReg + in], v_b_scale[in], acc[im * NReg + in]);+ }+ }+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++static inline __m512i _mm512_dpbusd_avx512bw_epi32(__m512i& c, const __m512i& a, const __m512i& b) {+ const __m512i dot2 = _mm512_maddubs_epi16(a, b);+ const __m512i ones = _mm512_set1_epi16(1);+ const __m512i sum4 = _mm512_madd_epi16(ones, dot2);+ return _mm512_add_epi32(c, sum4);+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_4bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto& a8ptr = A.aptr;+ auto& b4ptr = B.b4ptr;+ auto& asptr = A.sptr;+ auto& azptr = A.zpptr;+ int constexpr VLen = 16;+ int blks = k / blocksize;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ // Initialize accumulator with zeros+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (4 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ const __m512i onesu8 = _mm512_set1_epi8(1);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);++ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ __m512i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * VLen, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += 4) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits((void*)(b4ptr + i * 32 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_avx512bw_epi32(iacc[i], va, vb);+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits((void*)(b4ptr + i * 32 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_avx512bw_epi32(iacc[j * NReg + i], va[j], vb);+ }+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += 4) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits((void*)(b4ptr + i * 32 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_avx512bw_epi32(iacc[i], va, vb);+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits((void*)(b4ptr + i * 32 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_avx512bw_epi32(iacc[j * NReg + i], va[j], vb);+ }+ }+ }+ }+ }+ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_2bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ int constexpr VLen = 16;+ int blks = k / blocksize;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (2 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);+ const auto onesu8 = _mm512_set1_epi8(1);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ __m512i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_avx512bw_epi32(iacc[i], va, vb);+ b2ptr += VLen * KTILE / 4;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_avx512bw_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b2ptr += VLen * KTILE / 4;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_avx512bw_epi32(iacc[i], va, vb);+ b2ptr += VLen * KTILE / 4;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_avx512bw_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b2ptr += VLen * KTILE / 4;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_3bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (3 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ const auto onesu8 = _mm512_set1_epi8(1);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ __m512i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_avx512bw_epi32(iacc[i], va, vb);+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_avx512bw_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_avx512bw_epi32(iacc[i], va, vb);+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_avx512bw_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_1bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (1 - 1);+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ const auto onesu8 = _mm512_set1_epi8(1);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ __m512i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, bzp[i]);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb1);+ iacc[i] = _mm512_dpbusd_avx512bw_epi32(iacc[i], va, vb1);+ b1ptr += VLen * KTILE / 8;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, bzp[i]);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb1);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_avx512bw_epi32(iacc[j * NReg + i], va[j], vb1);+ }+ b1ptr += VLen * KTILE / 8;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, vbias);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb1);+ iacc[i] = _mm512_dpbusd_avx512bw_epi32(iacc[i], va, vb1);+ b1ptr += VLen * KTILE / 8;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_srli_epi32(vb1, 2);+ vb1 = _mm512_sub_epi8(vb1, vbias);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb1);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_avx512bw_epi32(iacc[j * NReg + i], va[j], vb1);+ }+ b1ptr += VLen * KTILE / 8;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_5bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (5 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ const auto onesu8 = _mm512_set1_epi8(1);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ __m512i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_avx512bw_epi32(iacc[i], va, vb);+ b4ptr += VLen * KTILE / 2;+ b1ptr += VLen * KTILE / 8;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_avx512bw_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += VLen * KTILE / 2;+ b1ptr += VLen * KTILE / 8;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_avx512bw_epi32(iacc[i], va, vb);+ b4ptr += VLen * KTILE / 2;+ b1ptr += VLen * KTILE / 8;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ vb1 = _mm512_slli_epi32(vb1, 2);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_avx512bw_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += VLen * KTILE / 2;+ b1ptr += VLen * KTILE / 8;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_6bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);++ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (6 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);+ const auto onesu8 = _mm512_set1_epi8(1);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ __m512i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_avx512bw_epi32(iacc[i], va, vb);+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_avx512bw_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_avx512bw_epi32(iacc[i], va, vb);+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_avx512bw_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_7bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr VLen = 16;+ int constexpr NReg = NTILE / VLen;+ int constexpr MReg = MTILE;+ __m512 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm512_setzero_ps();+ }+ int constexpr FullRange = 1 << (7 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm512_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm512_set1_epi8(FullRange);++ auto zmm_0x04 = _mm512_set1_epi8(0x04);+ auto zmm_0x00 = _mm512_set1_epi8(0x00);+ const auto vindex = _mm512_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12,+ 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0);++ uint64_t mask0 = 0x0303030303030303;+ auto vmask0 = _mm512_set1_epi64(*(int64_t*)&mask0);+ auto vshift_y = _mm512_set_epi32(6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm512_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0,+ 15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);+ const auto onesu8 = _mm512_set1_epi8(1);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m512i iacc[NReg * MReg];+ __m512i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm512_setzero_si512();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm512_setzero_si512();+ }+ if (B.zpptr) {+ __m512i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 16, vindex);+ bzp[i] = _mm512_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_or_si512(vb, vb2);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_avx512bw_epi32(iacc[i], va, vb);+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_or_si512(vb, vb2);+ vb = _mm512_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_avx512bw_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m512i va = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_or_si512(vb, vb2);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm512_dpbusd_avx512bw_epi32(iacc[i], va, vb);+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ } else {+ __m512i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm512_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, zmm_0x00, zmm_0x04);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm512_slli_epi32(vb1, 4);+ vb2 = _mm512_slli_epi32(vb2, 4);+ vb = _mm512_or_si512(vb, vb1);+ vb = _mm512_or_si512(vb, vb2);+ vb = _mm512_sub_epi8(vb, vbias);+ bacc[i] = _mm512_dpbusd_avx512bw_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm512_dpbusd_avx512bw_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += VLen * KTILE / 2;+ b2ptr += VLen * KTILE / 4;+ b1ptr += VLen * KTILE / 8;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm512_storeu_ps(C + i * VLen + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename T>+static inline BTLA_CODE mul(const T* src0ptr, const T* src1ptr, T* dstptr, size_t size) {+ int constexpr VLen = 16;+ size_t velt = utils::padto_le(size, VLen);+ size_t i = 0;+ auto vfunc = [&]() {+ auto v0 = load_T_fp32(src0ptr + i);+ auto v1 = load_T_fp32(src1ptr + i);+ auto out = _mm512_mul_ps(v0, v1);+ store_fp32_T(out, dstptr + i);+ };+ for (; i < velt; i += VLen) vfunc();+ if (i < size) {+ if (size >= VLen) {+ i = size - VLen;+ vfunc();+ } else {+ ref::mul(src0ptr + i, src1ptr + i, dstptr + i, size - i);+ }+ }+ return BTLA_CODE::Success;+}++template <typename T>+static inline BTLA_CODE inv(const T* src0ptr, T* dstptr, size_t size) {+ int constexpr VLen = 16;+ size_t velt = utils::padto_le(size, VLen);+ size_t i = 0;+ auto vfunc = [&]() {+ auto v0 = load_T_fp32(src0ptr + i);+ auto out = _mm512_rcp14_ps(v0);+ store_fp32_T(out, dstptr + i);+ };+ for (; i < velt; i += VLen) vfunc();+ if (i < size) {+ if (size >= VLen) {+ i = size - VLen;+ vfunc();+ } else {+ ref::inv(src0ptr + i, dstptr + i, size - i);+ }+ }+ return BTLA_CODE::Success;+}++template <typename T>+static inline BTLA_CODE add(const T* src0ptr, const T* src1ptr, T* dstptr, size_t size) {+ int constexpr VLen = 16;+ size_t velt = utils::padto_le(size, VLen);+ size_t i = 0;+ auto vfunc = [&]() {+ auto v0 = load_T_fp32(src0ptr + i);+ auto v1 = load_T_fp32(src1ptr + i);+ auto out = _mm512_add_ps(v0, v1);+ store_fp32_T(out, dstptr + i);+ };+ for (; i < velt; i += VLen) vfunc();+ if (i < size) {+ if (size >= VLen) {+ i = size - VLen;+ vfunc();+ } else {+ ref::add(src0ptr + i, src1ptr + i, dstptr + i, size - i);+ }+ }+ return BTLA_CODE::Success;+}++template <bool HAS_ALIBI, bool HAS_TANH>+static inline BTLA_CODE scale_track_max_fp32_fp32(const float* src, const int src_step, float* dst, float* dst_max,+ int ld_dst, const int M_offset, const int N_offset, const int M,+ const int N, float scale, int causal_offset, float _alibi_slope,+ float tanh_scale, int padding_type, void* tmpcache,+ size_t cachesize) {+ static constexpr float seq15[16]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};+ alignas(32) const uint32_t mask8[9][8]{+ {0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000},+ {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff},+ };+ const auto v_scale = _mm512_set1_ps(scale);+ const auto v_seq15 = _mm512_loadu_ps(seq15);+ const auto alibi_slope = _mm512_set1_ps(_alibi_slope);+ const auto alibi_base = _mm512_mul_ps(alibi_slope, _mm512_add_ps(v_seq15, _mm512_set1_ps(N_offset)));+ const auto alibi_step = _mm512_set1_ps(_alibi_slope * 16);++ for (int i = 0; i < M; ++i) {+ auto alibi_curr = alibi_base;+ const auto N_unmasked = std::min(N, padding_type == 0 ? INT32_MAX+ : padding_type == 1 ? i + M_offset - N_offset + causal_offset + 1+ : causal_offset - N_offset);++ const auto v_mask = _cvtu32_mask16((1U << (N_unmasked % 16)) - 1);+ int j = 0;+ auto v_max = _mm512_set1_ps(-INFINITY);+ for (; j < N_unmasked - 15; j += 16) {+ const auto xs = _mm512_fmadd_ps(v_scale, _mm512_loadu_ps(src + i * src_step + j), alibi_curr);+ v_max = _mm512_max_ps(v_max, xs);+ _mm512_storeu_ps(dst + i * ld_dst + j, xs);+ if constexpr (HAS_ALIBI)+ alibi_curr = _mm512_add_ps(alibi_curr, alibi_step);+ else if constexpr (HAS_TANH) {+ float* dst_ = dst + i * ld_dst + j;+ for (int jj = 0; jj < 16; jj++) dst_[jj] = tanh_scale * tanh(dst_[jj]);+ }+ }+ if (j < N_unmasked) {+ const auto xs = _mm512_fmadd_ps(v_scale, _mm512_maskz_loadu_ps(v_mask, src + i * src_step + j), alibi_curr);+ v_max = _mm512_mask_max_ps(v_max, v_mask, v_max, xs);+ _mm512_storeu_ps(dst + i * ld_dst + j, xs);+ if constexpr (HAS_ALIBI)+ alibi_curr = _mm512_add_ps(alibi_curr, alibi_step);+ else if constexpr (HAS_TANH) {+ float* dst_ = dst + i * ld_dst + j;+ for (int jj = 0; jj < N_unmasked - j; jj++) dst_[jj] = tanh_scale * tanh(dst_[jj]);+ }+ j += 16;+ }+ dst_max[i] = std::max(dst_max[i], _mm512_reduce_max_ps(v_max));+ if constexpr (HAS_TANH) dst_max[i] = tanh_scale * tanh(dst_max[i]);+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE scale_track_max_int32_fp32(const int32_t* src, const int src_step, float* dst, float* dst_max,+ int ld_dst, const int M_offset, const int N_offset, const int M,+ const int N, float scale, int causal_offset, float alibi_slope,+ float tanh_scale, void* tmpcache, size_t cachesize) {+ const auto v_scale = _mm512_set1_ps(scale);+ for (int i = 0; i < M; ++i) {+ const auto N_unmasked = std::min(N, causal_offset < 0 ? INT32_MAX : i + M_offset - N_offset + causal_offset + 1);+ const auto v_mask = _cvtu32_mask16((1U << (N_unmasked % 16)) - 1);+ int j = 0;+ auto v_max = _mm512_set1_ps(-INFINITY);+ for (; j < N_unmasked - 15; j += 16) {+ const auto xs = _mm512_mul_ps(v_scale, _mm512_cvtepi32_ps(_mm512_loadu_si512(src + i * src_step + j)));+ v_max = _mm512_max_ps(v_max, xs);+ _mm512_storeu_ps(dst + i * ld_dst + j, xs);+ }+ if (j < N_unmasked) {+ const auto xs =+ _mm512_mul_ps(v_scale, _mm512_cvtepi32_ps(_mm512_maskz_loadu_epi32(v_mask, src + i * src_step + j)));+ v_max = _mm512_mask_max_ps(v_max, v_mask, v_max, xs);+ _mm512_storeu_ps(dst + i * ld_dst + j, xs);+ j += 16;+ }+ dst_max[i] = std::max(dst_max[i], _mm512_reduce_max_ps(v_max));+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE weight_cvt_bf16_fp32_n48(const utils::bf16* B, int ldb, bool is_padded, float* dst_ptr,+ int dst_step, int k_size, int n_size, int k_offset, int n_offset,+ void* tmpcache, size_t cachesize) {+ assert(is_padded);+ const auto src = B + k_offset * 48 + n_offset * ldb;+ assert(n_size <= 48);+ assert(n_offset % 48 == 0 && k_offset % 2 == 0);+ static const auto mask_hi = _cvtu32_mask32(0xaaaaaaaaU);+ for (int i = 0; i < k_size; i += 2) {+ for (int j = 0; j < n_size; j += 16) {+ const auto cur_src = src + i * 48 + j * 2;+ const auto cur_dst = dst_ptr + i * 48 + j;+ const auto src_lo = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_loadu_si512(cur_src), 16U));+ const auto src_hi = _mm512_castsi512_ps(_mm512_maskz_loadu_epi16(mask_hi, cur_src));+ _mm512_store_ps(cur_dst + 0, src_lo);+ _mm512_store_ps(cur_dst + 48, src_hi);+ }+ }++ return BTLA_CODE::Success;+}++static inline BTLA_CODE inplace_precompute_max_softmax_fp32_fp32(int m_size, int n_size, int n_pad_size, bool is_causal,+ float* src, float* dst, const float* s_max,+ float* expsum, int ld_src, int ld_dst) {+ for (int ii = 0; ii < m_size; ++ii) {+ const auto i_src = src + ii * ld_src;+ const auto i_dst = dst + ii * ld_dst;+ const auto curr_n_size = n_size + (is_causal ? ii : 0);+ const auto v_mask = _cvtu32_mask16((1U << (curr_n_size % 16)) - 1);+ { // subtract max+ const auto row_max = _mm512_set1_ps(s_max[ii]);+ for (int jj = 0; jj < curr_n_size; jj += 16) { // should be fine to do extra work on idx >= curr_n_size+ _mm512_storeu_ps(i_src + jj, _mm512_sub_ps(_mm512_loadu_ps(i_src + jj), row_max));+ }+ }+ auto v_sum = _mm512_setzero_ps();+ { // exp & sum+ int jj = 0;+ for (; jj < curr_n_size / 16 * 16; jj += 16) {+ const auto v_exp = kernel::avx512f::exp_ps_0_1(_mm512_loadu_ps(i_src + jj));+ v_sum = _mm512_add_ps(v_sum, v_exp);+ _mm512_storeu_ps(i_src + jj, v_exp);+ }+ if (jj < curr_n_size) {+ const auto v_exp =+ kernel::avx512f::exp_ps_0_1(_mm512_loadu_ps(i_src + jj)); // should be fine to load some extra+ v_sum = _mm512_mask_add_ps(v_sum, v_mask, v_sum, v_exp);+ _mm512_storeu_ps(i_src + jj, v_exp); // should be fine to store some extra+ }+ expsum[ii] = _mm512_reduce_add_ps(v_sum);+ v_sum = _mm512_set1_ps(expsum[ii]);+ }+ { // scale & store+ int jj = 0;+ for (; jj < utils::padto_le(curr_n_size, 16); jj += 16) {+ _mm512_store_ps(i_dst + jj, _mm512_div_ps(_mm512_loadu_ps(i_src + jj), v_sum));+ }+ if (jj < curr_n_size) {+ _mm512_store_ps(i_dst + jj, _mm512_maskz_div_ps(v_mask, _mm512_loadu_ps(i_src + jj), v_sum));+ jj += 16;+ }+ if (jj < n_pad_size) memset(i_dst + jj, 0, sizeof(utils::bf16) * (n_pad_size - jj));+ }+ }++ return BTLA_CODE::Success;+}++static inline BTLA_CODE inplace_precompute_max_softmax_fp32_u8(int m_size, int n_size, int n_pad_size, bool is_causal,+ float* src, uint8_t* dst, const float* s_max,+ float* expsum, int ld_src, int ld_dst) {+ for (int ii = 0; ii < m_size; ++ii) {+ const auto i_src = src + ii * ld_src;+ const auto i_dst = dst + ii * ld_dst;+ const auto curr_n_size = n_size + (is_causal ? ii : 0);+ const uint16_t v_mask = _cvtu32_mask16((1U << (curr_n_size % 16)) - 1);+ { // subtract max+ const auto row_max = _mm512_set1_ps(s_max[ii]);+ for (int jj = 0; jj < curr_n_size; jj += 16) { // should be fine to do extra work on idx >= curr_n_size+ _mm512_storeu_ps(i_src + jj, _mm512_sub_ps(_mm512_loadu_ps(i_src + jj), row_max));+ }+ }+ { // exp & sum+ auto v_sum = _mm512_setzero_ps();+ int jj = 0;+ for (; jj < curr_n_size / 16 * 16; jj += 16) {+ const auto v_exp = kernel::avx512f::exp_ps_0_1(_mm512_loadu_ps(i_src + jj));+ v_sum = _mm512_add_ps(v_sum, v_exp);+ _mm512_storeu_ps(i_src + jj, v_exp);+ }+ if (jj < curr_n_size) {+ const auto v_exp =+ kernel::avx512f::exp_ps_0_1(_mm512_loadu_ps(i_src + jj)); // should be fine to load some extra+ v_sum = _mm512_mask_add_ps(v_sum, v_mask, v_sum, v_exp);+ _mm512_storeu_ps(i_src + jj, v_exp); // should be fine to store some extra+ }+ expsum[ii] = _mm512_reduce_add_ps(v_sum);+ }+ { // scale & int8+ const auto v_scale = _mm512_set1_ps(UINT8_MAX);+ int jj = 0;+ for (; jj < curr_n_size / 16 * 16; jj += 16) {+ const auto v_softmax = _mm512_mul_ps(_mm512_loadu_ps(i_src + jj), v_scale);+ _mm_storeu_si128(reinterpret_cast<__m128i*>(i_dst + jj), _mm512_cvtusepi32_epi8(_mm512_cvtps_epu32(v_softmax)));+ }+ if (jj < curr_n_size) {+ const auto v_softmax = _mm512_mul_ps(_mm512_loadu_ps(i_src + jj), v_scale);+ _mm_storeu_si128(reinterpret_cast<__m128i*>(i_dst + jj),+ _mm512_maskz_cvtusepi32_epi8(v_mask, _mm512_cvtps_epu32(v_softmax)));+ jj += 16;+ }+ if (jj < n_pad_size) memset(i_dst + jj, 0, sizeof(uint8_t) * (n_pad_size - jj));+ }+ }+ return BTLA_CODE::Success;+}+#if defined(__INTEL_LLVM_COMPILER)+#pragma clang attribute pop+#elif defined(__GNUC__)+#pragma GCC pop_options+#endif+#endif+} // namespace avx512f+} // namespace kernel+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx_vnni.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx_vnni.hnew file mode 100644index 000000000..0dff584c6--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_avx_vnni.h@@ -0,0 +1,1522 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include "kernel_avx2.h"++namespace bestla {+namespace kernel {+namespace avx2 {+#if CompileAVXVNNI()+#if defined(__GNUC__)+#pragma GCC push_options+#pragma GCC target("avx2", "fma", "f16c", "avxvnni")+#elif defined(ICX)+#pragma clang attribute push(__attribute__((target("avx,avx2,fma,avxvnni"))), apply_to = function)+#endif++namespace vnni {++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_1bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr FullRange = 1 << (1 - 1);+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ const __m256i onesu8 = _mm256_set1_epi8(1);+ const __m256i vbias = _mm256_set1_epi8(FullRange);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ __m256i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb1);+ iacc[i] = _mm256_dpbusd_avx_epi32(iacc[i], va, vb1);+ b1ptr += 8 * KTILE / 8;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb1);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], va[j], vb1);+ }+ b1ptr += 8 * KTILE / 8;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, vbias);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb1);+ iacc[i] = _mm256_dpbusd_avx_epi32(iacc[i], va, vb1);+ b1ptr += 8 * KTILE / 8;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, vbias);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb1);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], va[j], vb1);+ }+ b1ptr += 8 * KTILE / 8;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_1bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr FullRange = 1 << (1 - 1);+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }++ const __m256i onesu8 = _mm256_set1_epi8(1);+ const __m256i vbias = _mm256_set1_epi8(FullRange);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, bzp[i]);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm256_sign_epi8(vb1, va[j]);+ auto vabsa = _mm256_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b1ptr += 8 * KTILE / 8;+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_srli_epi32(vb1, 2);+ vb1 = _mm256_sub_epi8(vb1, vbias);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm256_sign_epi8(vb1, va[j]);+ auto vabsa = _mm256_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b1ptr += 8 * KTILE / 8;+ }+ }+ }++ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_4bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto& a8ptr = A.aptr;+ auto& b4ptr = B.b4ptr;+ auto& asptr = A.sptr;+ auto& azptr = A.zpptr;++ int blks = k / blocksize;+ int constexpr FullRange = 1 << (4 - 1);+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ // Initialize accumulator with zeros+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ const __m256i onesu8 = _mm256_set1_epi8(1);+ const __m256i vbias = _mm256_set1_epi8(FullRange);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ __m256i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += 4) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = kernel::avx2::unpack_4bits((void*)(b4ptr + i * 16 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx_epi32(iacc[i], va, vb);+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = kernel::avx2::unpack_4bits((void*)(b4ptr + i * 16 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], va[j], vb);+ }+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += 4) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = kernel::avx2::unpack_4bits((void*)(b4ptr + i * 16 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx_epi32(iacc[i], va, vb);+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = kernel::avx2::unpack_4bits((void*)(b4ptr + i * 16 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], va[j], vb);+ }+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_4bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto& a8ptr = A.aptr;+ auto& b4ptr = B.b4ptr;+ auto& asptr = A.sptr;++ int blks = k / blocksize;+ int constexpr FullRange = 1 << (4 - 1);+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ const __m256i vbias = _mm256_set1_epi8(FullRange);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += 4) {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = kernel::avx2::unpack_4bits((void*)(b4ptr + i * 16 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm256_sign_epi8(vb, va[j]);+ auto vabsa = _mm256_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += 4) {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(a8ptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = kernel::avx2::unpack_4bits((void*)(b4ptr + i * 16 + (ib * blocksize + ik) * NTILE / 2), vmask);+ vb = _mm256_sub_epi8(vb, vbias);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm256_sign_epi8(vb, va[j]);+ auto vabsa = _mm256_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ }+ }+ }++ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_2bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);++ int blks = k / blocksize;+ int constexpr FullRange = 1 << (2 - 1);+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0_y = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ const __m256i onesu8 = _mm256_set1_epi8(1);+ const __m256i vbias = _mm256_set1_epi8(FullRange);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ __m256i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx_epi32(iacc[i], va, vb);+ b2ptr += 8 * KTILE / 4;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b2ptr += 8 * KTILE / 4;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx_epi32(iacc[i], va, vb);+ b2ptr += 8 * KTILE / 4;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b2ptr += 8 * KTILE / 4;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_2bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);++ int blks = k / blocksize;+ int constexpr FullRange = 1 << (2 - 1);+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0_y = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ const __m256i onesu8 = _mm256_set1_epi8(1);+ const __m256i vbias = _mm256_set1_epi8(FullRange);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }++ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(vbias, bzp[i]);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm256_sign_epi8(vb, va[j]);+ auto vabsa = _mm256_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b2ptr += 8 * KTILE / 4;+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ vb = _mm256_sub_epi8(vb, vbias);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm256_sign_epi8(vb, va[j]);+ auto vabsa = _mm256_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b2ptr += 8 * KTILE / 4;+ }+ }+ }+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_3bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr FullRange = 1 << (3 - 1);+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0_y = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ const __m256i onesu8 = _mm256_set1_epi8(1);+ const __m256i vbias = _mm256_set1_epi8(FullRange);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ __m256i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx_epi32(iacc[i], va, vb);+ b2ptr += 8 * KTILE / 4;+ b1ptr += 8 * KTILE / 8;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b2ptr += 8 * KTILE / 4;+ b1ptr += 8 * KTILE / 8;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx_epi32(iacc[i], va, vb);++ b2ptr += 8 * KTILE / 4;+ b1ptr += 8 * KTILE / 8;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b2ptr += 8 * KTILE / 4;+ b1ptr += 8 * KTILE / 8;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_3bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr FullRange = 1 << (3 - 1);+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ uint64_t mask0 = 0x0303030303030303;+ auto vmask0_y = _mm256_set_epi64x(*(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0, *(int64_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ const __m256i onesu8 = _mm256_set1_epi8(1);+ const __m256i vbias = _mm256_set1_epi8(FullRange);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm256_sign_epi8(vb, va[j]);+ auto vabsa = _mm256_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b2ptr += 8 * KTILE / 4;+ b1ptr += 8 * KTILE / 8;+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_2bits(b2ptr, vshift_y, vmask0_y, vsfhl_mask_y, vorder_y);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm256_sign_epi8(vb, va[j]);+ auto vabsa = _mm256_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b2ptr += 8 * KTILE / 4;+ b1ptr += 8 * KTILE / 8;+ }+ }+ }++ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_5bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ int constexpr FullRange = 1 << (5 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);+ const __m256i onesu8 = _mm256_set1_epi8(1);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ __m256i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx_epi32(iacc[i], va, vb);+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx_epi32(iacc[i], va, vb);++ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_5bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ int constexpr FullRange = 1 << (5 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);+ const __m256i onesu8 = _mm256_set1_epi8(1);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm256_sign_epi8(vb, va[j]);+ auto vabsa = _mm256_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ vb1 = _mm256_slli_epi32(vb1, 2);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm256_sign_epi8(vb, va[j]);+ auto vabsa = _mm256_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ }+ }+ }++ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_6bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);++ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }++ int constexpr FullRange = 1 << (6 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ const __m256i onesu8 = _mm256_set1_epi8(1);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ uint32_t mask0 = 0x03030303;+ auto vmask0 = _mm256_set1_epi32(*(int32_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ __m256i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx_epi32(iacc[i], va, vb);+ b4ptr += 8 * KTILE / 2;+ b2ptr += 8 * KTILE / 4;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += 8 * KTILE / 2;+ b2ptr += 8 * KTILE / 4;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx_epi32(iacc[i], va, vb);+ b4ptr += 8 * KTILE / 2;+ b2ptr += 8 * KTILE / 4;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += 8 * KTILE / 2;+ b2ptr += 8 * KTILE / 4;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_6bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);++ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ int constexpr FullRange = 1 << (6 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);+ const __m256i onesu8 = _mm256_set1_epi8(1);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ uint32_t mask0 = 0x03030303;+ auto vmask0 = _mm256_set1_epi32(*(int32_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm256_sign_epi8(vb, va[j]);+ auto vabsa = _mm256_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b4ptr += 8 * KTILE / 2;+ b2ptr += 8 * KTILE / 4;+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_sub_epi8(vb, vbias);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm256_sign_epi8(vb, va[j]);+ auto vabsa = _mm256_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b4ptr += 8 * KTILE / 2;+ b2ptr += 8 * KTILE / 4;+ }+ }+ }++ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_7bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }++ int constexpr FullRange = 1 << (7 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);++ const __m256i onesu8 = _mm256_set1_epi8(1);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));++ uint32_t mask0 = 0x03030303;+ auto vmask0 = _mm256_set1_epi32(*(int32_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);+ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ __m256i bacc[NReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ for (int i = 0; i < NReg; i++) {+ bacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_or_si256(vb, vb2);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx_epi32(iacc[i], va, vb);+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ b2ptr += 8 * KTILE / 4;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_or_si256(vb, vb2);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ b2ptr += 8 * KTILE / 4;+ }+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ if constexpr (MTILE == 1) {+ __m256i va = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik));+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_or_si256(vb, vb2);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ iacc[i] = _mm256_dpbusd_avx_epi32(iacc[i], va, vb);+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ b2ptr += 8 * KTILE / 4;+ }+ } else {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_or_si256(vb, vb2);+ vb = _mm256_sub_epi8(vb, vbias);+ bacc[i] = _mm256_dpbusd_avx_epi32(bacc[i], onesu8, vb);+ for (int j = 0; j < MReg; j++) {+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], va[j], vb);+ }+ b4ptr += 8 * KTILE / 2;+ b1ptr += 8 * KTILE / 8;+ b2ptr += 8 * KTILE / 4;+ }+ }+ }+ }++ gemv_remove_zp<NReg, MReg>(A.zpptr + ib, A.ldzp, iacc, bacc);+ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_7bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);++ int blks = k / blocksize;+ int constexpr NReg = NTILE / 8;+ int constexpr MReg = MTILE;+ __m256 acc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ acc[i] = _mm256_setzero_ps();+ }+ int constexpr FullRange = 1 << (7 - 1);+ uint32_t mask = 0x0f0f0f0f;+ auto vmask = _mm256_set1_epi32(*reinterpret_cast<int*>(&mask));+ auto vbias = _mm256_set1_epi8(FullRange);+ const __m256i onesu8 = _mm256_set1_epi8(1);+ const auto vindex = _mm256_set_epi8(12, 12, 12, 12, 8, 8, 8, 8, 4, 4, 4, 4, 0, 0, 0, 0, 12, 12, 12, 12, 8, 8, 8, 8, 4,+ 4, 4, 4, 0, 0, 0, 0);++ uint32_t mask0 = 0x03030303;+ auto vmask0 = _mm256_set1_epi32(*(int32_t*)&mask0);+ auto vshift_y = _mm256_set_epi32(6, 4, 2, 0, 6, 4, 2, 0);+ auto vsfhl_mask_y = _mm256_set_epi8(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0, 15, 11, 7, 3, 14, 10, 6, 2,+ 13, 9, 5, 1, 12, 8, 4, 0);+ auto vorder_y = _mm256_set_epi32(1, 1, 1, 1, 0, 0, 0, 0);++ const __m256i highMask = _mm256_set1_epi8(0x04);+ const __m256i bit1Mask = _mm256_set1_epi32(0x0F);+ const __m256i bit1Shift_1 = _mm256_set_epi32(28, 24, 20, 16, 12, 8, 4, 0);+ const __m256i bit1Shift_2 = _mm256_set1_epi32((1 << 23) + (1 << 16) + (1 << 9) + (1 << 2));++ int constexpr KTILE = 4;+ for (int ib = 0; ib < blks; ib += 1) {+ __m256i iacc[NReg * MReg];+ for (int i = 0; i < NReg * MReg; i++) {+ iacc[i] = _mm256_setzero_si256();+ }+ if (B.zpptr) {+ __m256i bzp[NReg];+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int i = 0; i < NReg; i++) {+ bzp[i] = load_zp_epi8_broadcast_epi32(bzptr + i * 8, vindex);+ bzp[i] = _mm256_add_epi8(bzp[i], vbias);+ }+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_or_si256(vb, vb2);+ vb = _mm256_sub_epi8(vb, bzp[i]);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm256_sign_epi8(vb, va[j]);+ auto vabsa = _mm256_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b4ptr += 8 * KTILE / 2;+ b2ptr += 8 * KTILE / 4;+ b1ptr += 8 * KTILE / 8;+ }+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ __m256i va[MReg];+ for (int i = 0; i < MReg; i++) {+ va[i] = _mm256_set1_epi32(*(int*)(A.aptr + ib * blocksize + ik + i * A.lda));+ }+ for (int i = 0; i < NReg; i++) {+ auto vb = unpack_4bits(b4ptr, vmask);+ auto vb1 = unpack_1bits(b1ptr, bit1Shift_1, bit1Mask, bit1Shift_2, highMask);+ auto vb2 = unpack_2bits(b2ptr, vshift_y, vmask0, vsfhl_mask_y, vorder_y);+ vb1 = _mm256_slli_epi32(vb1, 4);+ vb2 = _mm256_slli_epi32(vb2, 4);+ vb = _mm256_or_si256(vb, vb1);+ vb = _mm256_or_si256(vb, vb2);+ vb = _mm256_sub_epi8(vb, vbias);+ for (int j = 0; j < MReg; j++) {+ auto vsb = _mm256_sign_epi8(vb, va[j]);+ auto vabsa = _mm256_sign_epi8(va[j], va[j]);+ iacc[j * NReg + i] = _mm256_dpbusd_avx_epi32(iacc[j * NReg + i], vabsa, vsb);+ }+ b4ptr += 8 * KTILE / 2;+ b2ptr += 8 * KTILE / 4;+ b1ptr += 8 * KTILE / 8;+ }+ }+ }++ gemv_dequant_s32fp32<ScaleT, NReg, MTILE>(A.sptr + ib, A.ldzp, B.sptr + ib * B.ldzp, iacc, acc);+ }++ for (int j = 0; j < MReg; j++) {+ for (int i = 0; i < NReg; i++) {+ _mm256_storeu_ps(C + i * 8 + j * ldc, acc[j * NReg + i]);+ }+ }+ return BTLA_CODE::Success;+}+} // namespace vnni++#ifdef __GNUC__+#pragma GCC pop_options+#else+#endif+#endif++} // namespace avx2+} // namespace kernel+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_jit.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_jit.hnew file mode 100644index 000000000..09ec4fcf8--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_jit.h@@ -0,0 +1,1730 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include <algorithm>+#include <array>+#include <cassert>+#include <functional>+#include <memory>+#include <unordered_map>+#include <vector>++#include "bestla.h"+#include "bestla_device.h"+#include "bestla_utils.h"+#include "bestla_jit.h"+#include "kernel_jit_injector.h"++namespace bestla {+namespace kernel {+namespace jit {++class DequanS8FP {+ public:+ class MicroKernelAVX512F : protected xbyak::JitAvx512f {+ public:+ struct params {+ void *srcptr, *dstptr;+ int row, col;+ int srcstride, dststride;+ float* scales;+ int8_t* zps;+ };+ typedef long long (*func_t)(params*);+ static int constexpr VBytes = 64;+ static int constexpr RegScale = 0;+ static int constexpr RegZP = 4;+ static int constexpr RegTmp = RegScale + 8;+ MicroKernelAVX512F(BTLA_DTYPE dst_dt, bool is_sym_, int pack_row) {+ assert(dst_dt == BTLA_DTYPE::F32 || dst_dt == BTLA_DTYPE::BF16);+ is_sym = is_sym_;+ generate(dst_dt, pack_row);+ this->ready();+ mKernel = this->getCode<func_t>();+ }++ void generate(BTLA_DTYPE dst_dt, int pack_row) {+ assert(pack_row == 1 || pack_row == 2 || pack_row == 4);+ int zmm_scale_step = 64 / pack_row;+ Xbyak::Label data_label;+ inLocalLabel(); // use local label for multiple instance+ {+ int SF_TmpSize = 64;+ int SF_TmpPos = 16 * 14;+ Xbyak::util::StackFrame st(this, 1, 13, SF_TmpPos + SF_TmpSize);+ parambase = st.p[0];+ reg_srcptr = st.t[0];+ reg_dstptr = st.t[1];+ reg_srcstride = st.t[2];+ reg_dststride = st.t[3];+ reg_rowsize = st.t[4];+ reg_colsize = st.t[5];+ reg_iterrow = st.t[6];+ reg_itercol = st.t[7];+ reg_tmp = st.t[8];+ reg_scaleptr = st.t[9];+ reg_tmpdst = st.t[10];+ reg_tmp1 = st.t[12];+ reg_ret = rax;++ vreg_push(rsp);++ mov(reg_srcptr, ptr[parambase + OFFSET(srcptr)]);+ mov(reg_dstptr, ptr[parambase + OFFSET(dstptr)]);+ mov(reg_scaleptr, ptr[parambase + OFFSET(scales)]);+ xor_(reg_srcstride, reg_srcstride);+ mov(reg_srcstride.cvt32(), ptr[parambase + OFFSET(srcstride)]);+ xor_(reg_dststride, reg_dststride);+ mov(reg_dststride.cvt32(), ptr[parambase + OFFSET(dststride)]);++ load32(reg_colsize, ptr[parambase + OFFSET(col)]);+ load32(reg_rowsize, ptr[parambase + OFFSET(row)]);+ xor_(reg_itercol, reg_itercol);++ // reuse parambase reg+ if (!is_sym) {+ mov(reg_tmp1, ptr[parambase + OFFSET(zps)]);+ mov(reg_zpptr, reg_tmp1);+ xor_(reg_tmp1, reg_tmp1);+ }++ auto get_dst_step = [&] {+ if (dst_dt == BTLA_DTYPE::BF16) return 2;+ return 4; // f32 case.+ };++ auto generateNTile = [&](int N, BTLA_DTYPE dst_dt, int zmm_scale_step, std::string row_label) {+ if (pack_row == 2) {+ vmovups(Xbyak::Zmm(RegTmp), ptr[rip + data_label + 8]);+ } else if (pack_row == 4) {+ vmovups(Xbyak::Zmm(RegTmp), ptr[rip + data_label + 72]);+ }+ for (int i = 0; i < N; i++) {+ vmovups(Xbyak::Zmm(RegScale + i), ptr[reg_scaleptr + i * zmm_scale_step]);+ if (pack_row == 2 || pack_row == 4) {+ vpermd(Xbyak::Zmm(RegScale + i), Xbyak::Zmm(RegTmp), Xbyak::Zmm(RegScale + i));+ }+ if (!is_sym) {+ vpmovsxbd(Xbyak::Zmm(RegZP + i),+ ptr[reg_zpptr + i * zmm_scale_step / sizeof(float)]); // revert to zp_step.+ if (pack_row == 2 || pack_row == 4) {+ vpermd(Xbyak::Zmm(RegZP + i), Xbyak::Zmm(RegTmp), Xbyak::Zmm(RegZP + i));+ }+ }+ }+ xor_(reg_iterrow, reg_iterrow);+ mov(reg_tmp, reg_srcptr);+ mov(reg_tmp1, reg_dstptr);+ L(row_label);+ for (int i = 0; i < N; i++) {+ vpmovsxbd(Xbyak::Zmm(RegTmp), ptr[reg_tmp + i * 16]);+ if (!is_sym) {+ vpsubd(Xbyak::Zmm(RegTmp), Xbyak::Zmm(RegTmp), Xbyak::Zmm(RegZP + i));+ }+ vcvtdq2ps(Xbyak::Zmm(RegTmp), Xbyak::Zmm(RegTmp));+ vmulps(Xbyak::Zmm(RegTmp), Xbyak::Zmm(RegScale + i));+ if (dst_dt == BTLA_DTYPE::F32) {+ vmovups(ptr[reg_tmp1 + i * 64], Xbyak::Zmm(RegTmp));+ }+ if (dst_dt == BTLA_DTYPE::BF16) {+ Xbyak::Ymm ymm_v = Xbyak::Ymm(RegTmp);+ Xbyak::Zmm zmm_v = Xbyak::Zmm(RegTmp);+ if (device::CpuDevice::getInstance()->AVX512_BF16()) {+ vcvtneps2bf16(ymm_v, zmm_v);+ } else {+ vmovups(Xbyak::Zmm(31), zmm_v);+ vpsrldq(zmm_v, zmm_v, 2);+ vpandd(zmm_v, zmm_v, zword_b[rip + data_label]);+ vpaddd(zmm_v, zmm_v, zword_b[rip + data_label + 4]);+ vpaddd(zmm_v, zmm_v, Xbyak::Zmm(31));+ vpsrld(zmm_v, zmm_v, 16);+ vpmovdw(ymm_v, zmm_v);+ }+ vmovups(ptr[reg_tmp1 + i * 32], ymm_v);+ }+ }+ add(reg_tmp, reg_srcstride);+ add(reg_tmp1, reg_dststride);+ add(reg_iterrow, 1);+ cmp(reg_iterrow, reg_rowsize);+ jb(row_label);+ };++ L(".colloop");+ mov(reg_tmp, reg_colsize);+ sub(reg_tmp, reg_itercol);+ cmp(reg_tmp, 64);+ jl(".proc48", T_NEAR);+ generateNTile(4, dst_dt, zmm_scale_step, ".rowloop1");+ add(reg_itercol, 64);+ add(reg_srcptr, 1 * 64);+ add(reg_dstptr, get_dst_step() * 64);+ add(reg_scaleptr, 4 * 64 / pack_row);+ if (!is_sym) add(reg_zpptr, 1 * 64 / pack_row);+ jmp(".colend", T_NEAR);++ L(".proc48");+ cmp(reg_tmp, 48);+ jl(".proc32", T_NEAR);+ generateNTile(3, dst_dt, zmm_scale_step, ".rowloop2");+ add(reg_itercol, 48);+ add(reg_srcptr, 1 * 48);+ add(reg_dstptr, get_dst_step() * 48);+ add(reg_scaleptr, 4 * 48 / pack_row);+ if (!is_sym) add(reg_zpptr, 1 * 48 / pack_row);+ jmp(".colend", T_NEAR);++ L(".proc32");+ generateNTile(2, dst_dt, zmm_scale_step, ".rowloop3");+ add(reg_itercol, 32);+ add(reg_srcptr, 1 * 32);+ add(reg_dstptr, get_dst_step() * 32);+ add(reg_scaleptr, 4 * 32 / pack_row);+ if (!is_sym) add(reg_zpptr, 1 * 32 / pack_row);++ L(".colend");+ cmp(reg_itercol, reg_colsize);+ jb(".colloop");++ mov(reg_ret, 0);+ vreg_pop(rsp);+ }+ outLocalLabel(); // end of local label+ L(data_label);+ uint32_t bf16_cvt_magic_num[2] = {0x00000001, 0X00007FFF};+ uint32_t packrow2_permute_idx[16] = {0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7};+ uint32_t packrow4_permute_idx[16] = {0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3};+ db(reinterpret_cast<uint8_t*>(bf16_cvt_magic_num), sizeof(bf16_cvt_magic_num));+ db(reinterpret_cast<uint8_t*>(packrow2_permute_idx), sizeof(packrow2_permute_idx));+ db(reinterpret_cast<uint8_t*>(packrow4_permute_idx), sizeof(packrow4_permute_idx));+ }++ func_t mKernel = nullptr;++ private:+ Xbyak::Reg64 parambase;+ Xbyak::Reg64 reg_srcptr;+ Xbyak::Reg64 reg_dstptr;+ Xbyak::Reg64 reg_srcstride;+ Xbyak::Reg64 reg_dststride;+ Xbyak::Reg64 reg_rowsize;+ Xbyak::Reg64 reg_colsize;+ Xbyak::Reg64 reg_iterrow;+ Xbyak::Reg64 reg_itercol;+ Xbyak::Reg64 reg_tmp;+ Xbyak::Reg64 reg_scaleptr;+ Xbyak::Reg64 reg_tmpdst;+ Xbyak::Reg64 reg_tmp1;+ Xbyak::Reg64 reg_ret;+ Xbyak::Reg64 reg_zpptr = reg_ret;+ bool is_sym;+ };+ template <int PACK_ROW, typename _DST_T>+ static void forward_avx512f(int8_t* srcptr, _DST_T* dstptr, int row, int col, int ld_src, int ld_dst, float* scales,+ int8_t* zero_points) {+ static MicroKernelAVX512F mAVX512FSym(utils::bestla_dtype<_DST_T>, true, PACK_ROW);+ static MicroKernelAVX512F mAVX512FASym(utils::bestla_dtype<_DST_T>, false, PACK_ROW);+ auto param = MicroKernelAVX512F::params{srcptr,+ dstptr,+ row,+ col,+ static_cast<int>(ld_src * sizeof(int8_t)),+ static_cast<int>(ld_dst * sizeof(_DST_T)),+ scales,+ zero_points};+ if (zero_points == nullptr) {+ mAVX512FSym.mKernel(¶m);+ } else {+ mAVX512FASym.mKernel(¶m);+ }+ }+};+class DecompressS3 {+ public:+ template <typename _DST_T>+ class MicroKernelAVX2 : protected xbyak::JitAvx2 {+ public:+ struct params {+ void *bit2ptr, *bit1ptr, *dstptr, *tmpbuf;+ int unpack_elt;+ const int8_t lowMask = 3, highMask = 4;+ const int32_t bit1Mask = 0x0F, bit1Shift2 = (1 << 23) + (1 << 16) + (1 << 9) + (1 << 2);+ const int32_t bit1Shift1[8] = {0, 4, 8, 12, 16, 20, 24, 28};+ };+ typedef long long (*func_t)(params*);+ static int constexpr VBytes = 32;+ MicroKernelAVX2() {+ generate();+ this->ready();+ mKernel = this->getCode<func_t>();+ }++ void generate() {+ inLocalLabel(); // use local label for multiple instance+ {+ Xbyak::util::StackFrame st(this, 1, 13, 10 * 16);+ parambase = st.p[0];+ reg_bit1ptr = st.t[0];+ reg_bit2ptr = st.t[1];+ reg_loop = st.t[2];+ reg_iter = st.t[3];+ reg_dst = st.t[4];+ reg_tmp = st.t[5];+ reg_cache = st.t[6];+ reg_ret = rax;+ xor_(reg_loop, reg_loop);+ mov(reg_loop.cvt32(), dword[parambase + OFFSET(unpack_elt)]);+ xor_(reg_iter, reg_iter);+ vreg_push(rsp);+ Xbyak::Ymm lowMask = ymm15;+ Xbyak::Ymm highMask = ymm14;+ Xbyak::Ymm bit1Mask = ymm13;+ Xbyak::Ymm bit1Shift1 = ymm12;+ Xbyak::Ymm bit1Shift2 = ymm11;+ vpbroadcastb(lowMask, byte[parambase + OFFSET(lowMask)]);+ vpbroadcastb(highMask, byte[parambase + OFFSET(highMask)]);+ vpbroadcastd(bit1Mask, dword[parambase + OFFSET(bit1Mask)]);+ vpbroadcastd(bit1Shift2, dword[parambase + OFFSET(bit1Shift2)]);+ vmovdqu(bit1Shift1, ptr[parambase + OFFSET(bit1Shift1)]);+ mov(reg_bit1ptr, ptr[parambase + OFFSET(bit1ptr)]);+ mov(reg_bit2ptr, ptr[parambase + OFFSET(bit2ptr)]);+ mov(reg_dst, ptr[parambase + OFFSET(dstptr)]);+ if constexpr (!std::is_same_v<_DST_T, int8_t>) mov(reg_cache, ptr[parambase + OFFSET(tmpbuf)]);+ L("loop_label");+ Xbyak::Ymm bit2_data = ymm8;+ vmovups(bit2_data, ptr[reg_bit2ptr]);+ for (int i = 0; i < 4; i++) {+ vpbroadcastd(Xbyak::Ymm(i), dword[reg_bit1ptr + 4 * i]);+ vpsrlvd(Xbyak::Ymm(i), Xbyak::Ymm(i), bit1Shift1); // todo : check m256+ vpand(Xbyak::Ymm(i), Xbyak::Ymm(i), bit1Mask);+ vpmulld(Xbyak::Ymm(i), Xbyak::Ymm(i), bit1Shift2);+ vpand(Xbyak::Ymm(i), Xbyak::Ymm(i), highMask);+ vpsrlw(Xbyak::Ymm(4 + i), bit2_data, 2 * i);+ vpand(Xbyak::Ymm(4 + i), Xbyak::Ymm(4 + i), lowMask);+ vpaddb(Xbyak::Ymm(i), Xbyak::Ymm(i), Xbyak::Ymm(4 + i));+ vpsubb(Xbyak::Ymm(i), Xbyak::Ymm(i), highMask);+ if constexpr (std::is_same_v<_DST_T, int8_t>) {+ vmovdqu(ptr[reg_dst + 32 * i], Xbyak::Ymm(i));+ } else if constexpr (std::is_same_v<_DST_T, float>) {+ vmovdqu(ptr[reg_cache + 32 * i], Xbyak::Ymm(i));++ vpmovsxbd(Xbyak::Ymm(i), qword[reg_cache + 32 * i + 0]);+ vpmovsxbd(Xbyak::Ymm(4 + i), qword[reg_cache + 32 * i + 8]);+ vcvtdq2ps(Xbyak::Ymm(i), Xbyak::Ymm(i));+ vcvtdq2ps(Xbyak::Ymm(4 + i), Xbyak::Ymm(4 + i));+ vmovups(ptr[reg_dst + 4 * (32 * i + 0)], Xbyak::Ymm(i));+ vmovups(ptr[reg_dst + 4 * (32 * i + 8)], Xbyak::Ymm(4 + i));++ vpmovsxbd(Xbyak::Ymm(i), qword[reg_cache + 32 * i + 16]);+ vpmovsxbd(Xbyak::Ymm(4 + i), qword[reg_cache + 32 * i + 24]);+ vcvtdq2ps(Xbyak::Ymm(i), Xbyak::Ymm(i));+ vcvtdq2ps(Xbyak::Ymm(4 + i), Xbyak::Ymm(4 + i));+ vmovups(ptr[reg_dst + 4 * (32 * i + 16)], Xbyak::Ymm(i));+ vmovups(ptr[reg_dst + 4 * (32 * i + 24)], Xbyak::Ymm(4 + i));+ } else {+ assert(0);+ }+ }++ add(reg_iter, 1);+ add(reg_bit1ptr, 16);+ add(reg_bit2ptr, 32);+ add(reg_dst, 128 * sizeof(_DST_T));+ cmp(reg_iter, reg_loop);+ jb("loop_label");+ mov(reg_ret, 0);+ vreg_pop(rsp);+ }+ outLocalLabel(); // end of local label+ }++ func_t mKernel = nullptr;++ private:+ Xbyak::Reg64 parambase;+ Xbyak::Reg64 reg_bit1ptr;+ Xbyak::Reg64 reg_bit2ptr;+ Xbyak::Reg64 reg_loop;+ Xbyak::Reg64 reg_iter;+ Xbyak::Reg64 reg_dst;+ Xbyak::Reg64 reg_tmp;+ Xbyak::Reg64 reg_cache;+ Xbyak::Reg64 reg_ret;+ Xbyak::Reg64 bit;+ };+ template <typename _DST_T>+ static void forward_avx2(void* bit2ptr, void* bit1ptr, _DST_T* dstptr, void* tmpbuf, int unpack_elt) {+ static MicroKernelAVX2<_DST_T> ker;+ typename MicroKernelAVX2<_DST_T>::params param{bit2ptr, bit1ptr, dstptr, tmpbuf, unpack_elt / 128};+ ker.mKernel(¶m);+ }+ template <typename _DST_T>+ class MicroKernelAVX512F : protected xbyak::JitAvx512f {+ public:+ struct params {+ void *bit2ptr, *bit1ptr, *dstptr, *tmpbuf;+ int unpack_elt;+ int8_t ox3, ox4;+ int ox5;+ };+ typedef long long (*func_t)(params*);+ static int constexpr VBytes = 64;+ MicroKernelAVX512F() {+ generate();+ this->ready();+ mKernel = this->getCode<func_t>();+ }++ void generate() {+ inLocalLabel(); // use local label for multiple instance+ {+ Xbyak::util::StackFrame st(this, 1, 13);+ parambase = st.p[0];+ reg_bit1ptr = st.t[0];+ reg_bit2ptr = st.t[1];+ reg_loop = st.t[2];+ reg_iter = st.t[3];+ reg_dst = st.t[4];+ reg_tmp = st.t[5];+ reg_cache = st.t[6];+ reg_ret = rax;+ xor_(reg_loop, reg_loop);+ mov(reg_loop.cvt32(), ptr[parambase + OFFSET(unpack_elt)]);+ xor_(reg_iter, reg_iter);+ Xbyak::Ymm LowMask = ymm1;+ Xbyak::Zmm zmm_0x04 = zmm31;+ Xbyak::Zmm zmm_shift = zmm30;+ vpbroadcastb(LowMask, ptr[parambase + OFFSET(ox3)]);+ vpbroadcastb(zmm_0x04, ptr[parambase + OFFSET(ox4)]);+ vpbroadcastd(zmm_shift, ptr[parambase + OFFSET(ox5)]);+ mov(reg_bit1ptr, ptr[parambase + OFFSET(bit1ptr)]);+ mov(reg_bit2ptr, ptr[parambase + OFFSET(bit2ptr)]);+ mov(reg_dst, ptr[parambase + OFFSET(dstptr)]);+ if constexpr (!std::is_same_v<_DST_T, int8_t>) mov(reg_cache, ptr[parambase + OFFSET(tmpbuf)]);+ L("loop_label");+ imul(reg_tmp, reg_iter, 16);+ kmovq(bit1_mask1, ptr[reg_bit1ptr + reg_tmp]);+ kmovq(bit1_mask2, ptr[reg_bit1ptr + reg_tmp + 8]);+ Xbyak::Zmm bit2_data_zmm = zmm0;+ imul(reg_tmp, reg_iter, 32);+ vmovups(ymm2, ptr[reg_bit2ptr + reg_tmp]);++ vpand(ymm4, LowMask, ymm2);+ vpsrlw(ymm2, ymm2, 2);+ vpand(ymm5, LowMask, ymm2);+ vpsrlw(ymm2, ymm2, 2);+ vpand(ymm6, LowMask, ymm2);+ vpsrlw(ymm2, ymm2, 2);+ vpand(ymm7, LowMask, ymm2);+ vinsertf32x8(zmm4, zmm4, ymm5, 1);+ vinsertf32x8(zmm6, zmm6, ymm7, 1);++ vxorps(zmm12, zmm12);+ vxorps(zmm13, zmm13);+ vmovdqu8(zmm12 | bit1_mask1, zmm_0x04);+ vmovdqu8(zmm13 | bit1_mask2, zmm_0x04);++ vpaddb(zmm4, zmm4, zmm12);+ vpaddb(zmm6, zmm6, zmm13);++ vpsllvd(zmm4, zmm4, zmm_shift);+ vpsllvd(zmm6, zmm6, zmm_shift);++ if constexpr (std::is_same_v<_DST_T, int8_t>) {+ imul(reg_tmp, reg_iter, 128);+ vmovups(ptr[reg_dst + reg_tmp], zmm4);+ vmovups(ptr[reg_dst + reg_tmp + 64], zmm6);+ } else if constexpr (std::is_same_v<_DST_T, float> || std::is_same_v<_DST_T, utils::bf16>) {+ vmovups(ptr[reg_cache], zmm4);+ vmovups(ptr[reg_cache + 64], zmm6);+ for (int i = 0; i < 8; i++) vpmovsxbd(Xbyak::Zmm(16 + i), ptr[reg_cache + 16 * i]);+ for (int i = 0; i < 8; i++) vcvtdq2ps(Xbyak::Zmm(16 + i), Xbyak::Zmm(16 + i));+ imul(reg_tmp, reg_iter, 128 * sizeof(_DST_T));+ if constexpr (std::is_same_v<_DST_T, float>) {+ for (int i = 0; i < 8; i++) vmovups(ptr[reg_dst + reg_tmp + i * 64], Xbyak::Zmm(16 + i));+ } else {+ for (int i = 0; i < 8; i++) vcvtneps2bf16(Xbyak::Ymm(16 + i), Xbyak::Zmm(16 + i));+ for (int i = 0; i < 8; i++) vmovups(ptr[reg_dst + reg_tmp + i * 32], Xbyak::Ymm(16 + i));+ }+ } else {+ assert(0);+ }++ add(reg_iter, 1);+ cmp(reg_iter, reg_loop);+ jb("loop_label");+ mov(reg_ret, 0);+ }+ outLocalLabel(); // end of local label+ }++ func_t mKernel = nullptr;++ private:+ Xbyak::Reg64 parambase;+ Xbyak::Reg64 reg_bit1ptr;+ Xbyak::Reg64 reg_bit2ptr;+ Xbyak::Reg64 reg_loop;+ Xbyak::Reg64 reg_iter;+ Xbyak::Reg64 reg_dst;+ Xbyak::Reg64 reg_tmp;+ Xbyak::Reg64 reg_cache;+ Xbyak::Reg64 reg_ret;+ Xbyak::Opmask bit1_mask1 = Xbyak::Opmask(1);+ Xbyak::Opmask bit1_mask2 = Xbyak::Opmask(2);+ Xbyak::Opmask bit1_mask3 = Xbyak::Opmask(3);+ Xbyak::Opmask bit1_mask4 = Xbyak::Opmask(4);+ };+ template <typename _DST_T>+ static void forward_avx512f(void* bit2ptr, void* bit1ptr, _DST_T* dstptr, void* tmpbuf, int unpack_elt) {+ static MicroKernelAVX512F<_DST_T> ker;+ typename MicroKernelAVX512F<_DST_T>::params param{bit2ptr, bit1ptr, dstptr, tmpbuf, unpack_elt / 128, 0x03, 0x4, 5};+ ker.mKernel(¶m);+ }+};++class DequanKBlockS8Fp {+ public:+ template <int _PACK_ROW, typename _ST, typename _DST_T>+ static inline BTLA_CODE forward_avx512f(int8_t* srcptr, _DST_T* dstptr, int row, int col, int ld_src, int ld_dst,+ _ST* scales, int8_t* zero_points, int k_offset, int kblock, int NPad) {+ int row0 = kblock - k_offset % kblock;+ row0 = row0 == kblock ? 0 : row0;+ row0 = row0 > row ? row : row0;+ int row1 = row - row0;+ int row1_blk = utils::padto_le(row1, kblock);+ int row2 = row - row1_blk - row0;+ auto sptr = scales + k_offset / kblock * NPad;+ int8_t* zptr = nullptr;+ if (zero_points != nullptr) zptr = zero_points + k_offset / kblock * NPad;+ if (row0 > 0) {+ DequanS8FP::forward_avx512f<_PACK_ROW>(srcptr, dstptr, row0, col, ld_src, ld_dst, sptr, zptr);+ srcptr += row0 * ld_src;+ dstptr += row0 * ld_dst;+ sptr += NPad;+ if (zero_points != nullptr) zptr += NPad;+ }+ for (int i = 0; i < row1_blk; i += kblock) {+ DequanS8FP::forward_avx512f<_PACK_ROW>(srcptr, dstptr, kblock, col, ld_src, ld_dst, sptr, zptr);+ srcptr += kblock * ld_src;+ dstptr += kblock * ld_dst;+ sptr += NPad;+ if (zero_points != nullptr) zptr += NPad;+ }+ if (row2 > 0) {+ DequanS8FP::forward_avx512f<_PACK_ROW>(srcptr, dstptr, row2, col, ld_src, ld_dst, sptr, zptr);+ }+ return BTLA_CODE::Success;+ }+};++struct DataConvertConfig {+ enum class cvt_direct {+ NO_CVT,+ BF16_TO_FP32,+ FP32_TO_BF16,+ F16_TO_FP32,+ FP32_TO_F16,+ };++ DataConvertConfig(BTLA_DTYPE src_t, BTLA_DTYPE dst_t, std::vector<kernel::jit_injector::eltwise_injector> injectors) {+ input_dt = src_t;+ output_dt = dst_t;+ if (injectors.size() != 0) {+ assert(src_t == BTLA_DTYPE::F32 || src_t == BTLA_DTYPE::BF16 || src_t == BTLA_DTYPE::F16);+ if (src_t == BTLA_DTYPE::BF16) before_postop = DataConvertConfig::cvt_direct::BF16_TO_FP32;+ if (src_t == BTLA_DTYPE::F16) before_postop = DataConvertConfig::cvt_direct::F16_TO_FP32;+ }+ // once contain postop, data_type before store will be fp32.+ if (injectors.size() != 0 || src_t == BTLA_DTYPE::F32) {+ if (dst_t == BTLA_DTYPE::BF16) before_store = DataConvertConfig::cvt_direct::FP32_TO_BF16;+ if (dst_t == BTLA_DTYPE::F16) {+ if (!device::CpuDevice::getInstance()->AVX512_FP16()) assert(0);+ before_store = DataConvertConfig::cvt_direct::FP32_TO_F16;+ }+ } else if (src_t == BTLA_DTYPE::BF16 && dst_t == BTLA_DTYPE::F32) {+ before_store = DataConvertConfig::cvt_direct::BF16_TO_FP32;+ } else if (src_t == BTLA_DTYPE::F16 && dst_t == BTLA_DTYPE::F32) {+ assert(device::CpuDevice::getInstance()->AVX512_FP16());+ before_store = DataConvertConfig::cvt_direct::F16_TO_FP32;+ }+ }++ int get_store_process_bytes(int VBytes) {+ if (before_store == DataConvertConfig::cvt_direct::BF16_TO_FP32 ||+ before_store == DataConvertConfig::cvt_direct::F16_TO_FP32)+ return 2 * VBytes;+ if (before_store == DataConvertConfig::cvt_direct::FP32_TO_BF16 ||+ before_store == DataConvertConfig::cvt_direct::FP32_TO_F16)+ return VBytes / 2;+ return VBytes;+ }++ cvt_direct before_postop = cvt_direct::NO_CVT;+ cvt_direct before_store = cvt_direct::NO_CVT;+ BTLA_DTYPE input_dt, output_dt;+};++template <typename SIMD_REG>+struct MemcpyStoreParam {+ SIMD_REG vmm_v;+ Xbyak::RegExp store_addr;+ bool tail;+ Xbyak::Opmask store_mask = Xbyak::util::k1;+};++class JitMemcpy2DAvx2 : protected xbyak::JitAvx2 {+ public:+ struct params {+ void *srcptr, *dstptr, *elt_const_v;+ int col;+ };+ typedef long long (*func_t)(params*);++ public:+ static int constexpr VBytes = 32;+ JitMemcpy2DAvx2(int unroll_row, BTLA_DTYPE src_t, BTLA_DTYPE dst_t,+ std::vector<kernel::jit_injector::eltwise_injector> injectors = {}) {+ DataConvertConfig dt_cvt_cfg(src_t, dst_t, injectors);+ generate(unroll_row, injectors, dt_cvt_cfg);+ }++ template <typename _SRC_T, typename _DST_T>+ static BTLA_CODE forward(const _SRC_T* srcptr, _DST_T* dstptr, int row, int col, int srcstep, int dststep,+ void* elt_const_v = nullptr) {+ static JitMemcpy2DAvx2 instance_withops(1, utils::bestla_dtype<_SRC_T>, utils::bestla_dtype<_DST_T>);+ for (int i = 0; i < row; i++) {+ auto param = params{reinterpret_cast<char*>(const_cast<_SRC_T*>(srcptr)) + i * srcstep * sizeof(_SRC_T),+ reinterpret_cast<char*>(dstptr) + i * dststep * sizeof(_DST_T), elt_const_v,+ static_cast<int>(col * sizeof(_SRC_T))};+ instance_withops.mKernel(¶m);+ }+ return BTLA_CODE::Success;+ }++ template <typename _SRC_T, typename _DST_T, BTLA_ELTWISEOP Op>+ static BTLA_CODE forward1(const _SRC_T* srcptr, _DST_T* dstptr, int row, int col, int srcstep, int dststep,+ void* elt_const_v = nullptr) {+ static JitMemcpy2DAvx2 instance_withops(1, utils::bestla_dtype<_SRC_T>, utils::bestla_dtype<_DST_T>,+ {kernel::jit_injector::eltwise_injector(Op)});+ for (int i = 0; i < row; i++) {+ auto param = params{reinterpret_cast<char*>(const_cast<_SRC_T*>(srcptr)) + i * srcstep * sizeof(_SRC_T),+ reinterpret_cast<char*>(dstptr) + i * dststep * sizeof(_DST_T), elt_const_v,+ static_cast<int>(col * sizeof(_SRC_T))};+ instance_withops.mKernel(¶m);+ }+ return BTLA_CODE::Success;+ }++ protected:+ void generate(int unrollk, std::vector<kernel::jit_injector::eltwise_injector>& injectors,+ DataConvertConfig dt_cvt_cfg) {+ // unrollK=[1,2]+ if (unrollk != 1 && unrollk != 2) {+ assert(false);+ return;+ }+ Xbyak::Label data_label;+ inLocalLabel(); // use local label for multiple instance+ {+ int SF_TmpSize = 64;+ int SF_TmpPos = 16 * 10;+ Xbyak::util::StackFrame st(this, 1, 13, 16 * 10 + SF_TmpSize);+ const Xbyak::Reg64& parambase = st.p[0];+ const Xbyak::Reg64& reg_srcptr = st.t[0];+ const Xbyak::Reg64& reg_dstptr = st.t[1];+ const Xbyak::Reg64& reg_colsize = st.t[2];+ const Xbyak::Reg64& reg_itercol = st.t[3];+ const Xbyak::Reg64& reg_tmp = st.t[4];+ const Xbyak::Reg64& reg_elt_constv = st.t[5]; // alias of reg_tmp.+ const Xbyak::Reg64& reg_tmp1 = st.t[6];+ const Xbyak::Reg64& reg_tmp2 = st.t[7];+ const Xbyak::Reg64& reg_ret = rax;++ vreg_push(rsp);++ mov(reg_srcptr, ptr[parambase + OFFSET(srcptr)]);+ mov(reg_dstptr, ptr[parambase + OFFSET(dstptr)]);+ mov(reg_elt_constv, ptr[parambase + OFFSET(elt_const_v)]);++ load32(reg_colsize, ptr[parambase + OFFSET(col)]);+ int const ColUnroll = 4;+ int const ymm_tmp_num = 2;+ std::array<Xbyak::Ymm, ymm_tmp_num> ymm_tmps = {Xbyak::Ymm(unrollk * ColUnroll),+ Xbyak::Ymm(unrollk * ColUnroll + 1)};+ for (int i = 0; i < unrollk * ColUnroll; i++) used_ymm_idx.insert(i);+ for (auto&& injector : injectors) {+ injector.assign_resources(this, used_ymm_idx, reg_ret);+ injector.assign_reg_elt_constp(reg_elt_constv);+ }++ auto store_ymm_v = [&](MemcpyStoreParam<Xbyak::Ymm> p) { vmovups(ptr[p.store_addr], p.vmm_v); };++ auto unpack_ymm_16bit_withfunc = [&](MemcpyStoreParam<Xbyak::Ymm> p,+ std::function<void(MemcpyStoreParam<Xbyak::Ymm>)> func,+ BTLA_DTYPE BIT16_DT) {+ vmovups(ymm_tmps[0], p.vmm_v);+ Xbyak::Ymm ymm_v = Xbyak::Ymm(p.vmm_v.getIdx());+ if (BIT16_DT == BTLA_DTYPE::BF16) {+ vpmovzxwd(p.vmm_v, ymm_v);+ vpslld(p.vmm_v, p.vmm_v, 16);+ }+ func(p);+ vextractf128(Xbyak::Xmm(ymm_tmps[0].getIdx()), ymm_tmps[0], 1);+ if (BIT16_DT == BTLA_DTYPE::BF16) {+ vpmovzxwd(ymm_tmps[0], Xbyak::Ymm(ymm_tmps[0].getIdx()));+ vpslld(ymm_tmps[0], ymm_tmps[0], 16);+ }+ p.vmm_v = ymm_tmps[0];+ p.store_addr = p.store_addr + VBytes;+ func(p);+ };++ auto apply_postop_and_store = [&](MemcpyStoreParam<Xbyak::Ymm> p) {+ for (int k = 0; k < injectors.size(); k++) injectors[k].vector_compute(p.vmm_v, k * 3 * sizeof(float));+ if (dt_cvt_cfg.before_store == DataConvertConfig::cvt_direct::NO_CVT) {+ store_ymm_v(p);+ } else if (dt_cvt_cfg.before_store == DataConvertConfig::cvt_direct::BF16_TO_FP32) {+ unpack_ymm_16bit_withfunc(p, store_ymm_v, BTLA_DTYPE::BF16);+ } else if (dt_cvt_cfg.before_store == DataConvertConfig::cvt_direct::FP32_TO_BF16) {+ Xbyak::Xmm xmm_v = Xbyak::Xmm(p.vmm_v.getIdx());+ Xbyak::Xmm xmm_tmp = Xbyak::Xmm(ymm_tmps[1].getIdx());+ vmovups(ymm_tmps[0], p.vmm_v);+ vpsrldq(p.vmm_v, p.vmm_v, 2);+ mov(reg_tmp.cvt32(), 0x00000001);+ vmovd(xmm_tmp, reg_tmp.cvt32());+ vpbroadcastd(ymm_tmps[1], xmm_tmp);+ vpand(p.vmm_v, p.vmm_v, ymm_tmps[1]);+ mov(reg_tmp.cvt32(), 0x00007FFF);+ vmovd(xmm_tmp, reg_tmp.cvt32());+ vpbroadcastd(ymm_tmps[1], xmm_tmp);+ vpaddd(p.vmm_v, p.vmm_v, ymm_tmps[1]);+ vpaddd(p.vmm_v, p.vmm_v, ymm_tmps[0]);+ vpshufb(p.vmm_v, p.vmm_v, ptr[rip + data_label + 32]);+ vpermq(p.vmm_v, p.vmm_v, 0x58);+ vmovups(ptr[p.store_addr], xmm_v);+ } else {+ assert(0);+ }+ };++ auto load_store_value = [&](Xbyak::Ymm ymm_v, Xbyak::RegExp load_addr, Xbyak::RegExp store_addr) {+ vmovups(ymm_v, ptr[load_addr]);+ if (dt_cvt_cfg.before_postop == DataConvertConfig::cvt_direct::NO_CVT) {+ apply_postop_and_store({ymm_v, store_addr});+ } else if (dt_cvt_cfg.before_postop == DataConvertConfig::cvt_direct::BF16_TO_FP32) {+ unpack_ymm_16bit_withfunc({ymm_v, store_addr}, apply_postop_and_store, BTLA_DTYPE::BF16);+ } else {+ assert(0);+ }+ };++ xor_(reg_itercol, reg_itercol);++ L(".colloop");+ mov(reg_tmp, reg_colsize);+ sub(reg_tmp, reg_itercol);+ cmp(reg_tmp, ColUnroll * VBytes);+ jl(".maskproc", T_NEAR);++ for (int i = 0; i < ColUnroll; i++)+ load_store_value(Xbyak::Ymm(i), reg_srcptr + i * VBytes,+ reg_dstptr + i * dt_cvt_cfg.get_store_process_bytes(VBytes));++ add(reg_srcptr, ColUnroll * VBytes);+ add(reg_dstptr, ColUnroll * dt_cvt_cfg.get_store_process_bytes(VBytes));+ add(reg_itercol, ColUnroll * VBytes);+ jmp(".colend", T_NEAR);+ L(".maskproc");+ mov(reg_tmp2, reg_colsize);+ sub(reg_tmp2, reg_itercol);+ cmp(reg_tmp2, VBytes);+ jb(".maskflag", T_NEAR);+ cmp(reg_tmp2, 0);+ jl(".maskend", T_NEAR);+ load_store_value(Xbyak::Ymm(0), reg_srcptr, reg_dstptr);+ jmp(".maskend", T_NEAR);+ L(".maskflag");+ // 0<tail<8+ mov(reg_tmp1.cvt32(), 1);+ shlx(reg_tmp1.cvt32(), reg_tmp1.cvt32(), reg_tmp2.cvt32());+ sub(reg_tmp1.cvt32(), 1);+ vmovd(Xbyak::Xmm(1), reg_tmp1.cvt32());+ vpbroadcastd(Xbyak::Ymm(1), Xbyak::Xmm(1));+ vpsllvd(Xbyak::Ymm(1), Xbyak::Ymm(1), ptr[rip + data_label]);+ mov(reg_elt_constv, ptr[parambase + OFFSET(elt_const_v)]);+ vpmaskmovd(Xbyak::Ymm(0), Xbyak::Ymm(1), ptr[reg_srcptr]);+ for (int k = 0; k < injectors.size(); k++) injectors[k].vector_compute(Xbyak::Ymm(0), k * 3 * sizeof(float));+ vpmaskmovd(ptr[reg_dstptr], Xbyak::Ymm(1), Xbyak::Ymm(0));+ L(".maskend");+ add(reg_srcptr, VBytes);+ add(reg_dstptr, dt_cvt_cfg.get_store_process_bytes(VBytes));+ add(reg_itercol, VBytes);+ L(".colend");+ cmp(reg_itercol, reg_colsize);+ jb(".colloop");+ mov(reg_ret, 0);+ vreg_pop(rsp);+ }+ outLocalLabel(); // end of local label+ L(data_label);+ uint32_t mask_bias[8] = {28, 24, 20, 16, 12, 8, 4, 0};+ const uint8_t avx2_bf16_convert_maigc_num[32] = {0x02, 0x03, 0x06, 0x07, 0x0a, 0x0b, 0x0e, 0x0f, 0x80, 0x80, 0x80,+ 0x80, 0x80, 0x80, 0x80, 0x80, 0x02, 0x03, 0x06, 0x07, 0x0a, 0x0b,+ 0x0e, 0x0f, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};+ db(reinterpret_cast<uint8_t*>(mask_bias), sizeof(mask_bias));+ db(avx2_bf16_convert_maigc_num, sizeof(avx2_bf16_convert_maigc_num));+ for (auto&& injector : injectors) injector.prepare_table();+ this->ready();+ mKernel = this->getCode<func_t>();+ }++ func_t mKernel = nullptr;+ std::set<int> used_ymm_idx;+};++class JitMemcpy2DAvx512f : protected xbyak::JitAvx512f {+ public:+ struct params {+ void *srcptr, *dstptr, *elt_const_v;+ int col;+ };+ typedef long long (*func_t)(params*);++ public:+ static int constexpr VBytes = 64;+ JitMemcpy2DAvx512f(int unroll_row, BTLA_DTYPE src_t, BTLA_DTYPE dst_t,+ std::vector<kernel::jit_injector::eltwise_injector> injectors = {}) {+ DataConvertConfig dt_cvt_cfg(src_t, dst_t, injectors);+ generate(unroll_row, injectors, dt_cvt_cfg);+ }++ template <typename _SRC_T, typename _DST_T>+ static BTLA_CODE forward(const _SRC_T* srcptr, _DST_T* dstptr, int row, int col, int srcstep, int dststep,+ void* elt_const_v = nullptr) {+ static JitMemcpy2DAvx512f instance_withops(1, utils::bestla_dtype<_SRC_T>, utils::bestla_dtype<_DST_T>);++ for (int i = 0; i < row; i++) {+ auto param = params{reinterpret_cast<char*>(const_cast<_SRC_T*>(srcptr)) + i * srcstep * sizeof(_SRC_T),+ reinterpret_cast<char*>(dstptr) + i * dststep * sizeof(_DST_T), elt_const_v,+ static_cast<int>(col * sizeof(_SRC_T))};+ instance_withops.mKernel(¶m);+ }+ return BTLA_CODE::Success;+ }++ template <typename _SRC_T, typename _DST_T, BTLA_ELTWISEOP Op>+ static BTLA_CODE forward1(const _SRC_T* srcptr, _DST_T* dstptr, int row, int col, int srcstep, int dststep,+ void* elt_const_v = nullptr) {+ static JitMemcpy2DAvx512f instance_withops(1, utils::bestla_dtype<_SRC_T>, utils::bestla_dtype<_DST_T>,+ {kernel::jit_injector::eltwise_injector(Op)});+ for (int i = 0; i < row; i++) {+ auto param = params{reinterpret_cast<char*>(const_cast<_SRC_T*>(srcptr)) + i * srcstep * sizeof(_SRC_T),+ reinterpret_cast<char*>(dstptr) + i * dststep * sizeof(_DST_T), elt_const_v,+ static_cast<int>(col * sizeof(_SRC_T))};+ instance_withops.mKernel(¶m);+ }+ return BTLA_CODE::Success;+ }++ protected:+ void generate(int unrollk, std::vector<kernel::jit_injector::eltwise_injector>& injectors,+ DataConvertConfig dt_cvt_cfg) {+ inLocalLabel(); // use local label for multiple instance+ Xbyak::Label data_label;+ {+ int SF_TmpSize = 64;+ Xbyak::util::StackFrame st(this, 1, 13, 10 * 16 + SF_TmpSize);+ const Xbyak::Reg64& parambase = st.p[0];+ const Xbyak::Reg64& reg_src = st.t[0];+ const Xbyak::Reg64& reg_dst = st.t[1];+ const Xbyak::Reg64& reg_size = st.t[2];+ const Xbyak::Reg64& reg_iter = st.t[3];+ const Xbyak::Reg64& reg_tmp = st.t[4];+ const Xbyak::Reg64& reg_tmp2 = st.t[5];+ const Xbyak::Reg64& reg_elt_constv = st.t[6];+ const Xbyak::Reg64& reg_ret = rax;++ vreg_push(rsp);++ int const ColUnroll = 4;+ int const zmm_tmp_num = 2;+ std::array<Xbyak::Zmm, zmm_tmp_num> zmm_tmps = {Xbyak::Zmm(unrollk * ColUnroll),+ Xbyak::Zmm(unrollk * ColUnroll + 1)};+ for (int i = 0; i < unrollk * ColUnroll; i++) used_zmm_idx.insert(i);+ for (int i = 0; i < zmm_tmp_num; i++) used_zmm_idx.insert(i + unrollk * ColUnroll);+ for (auto&& injector : injectors) {+ injector.assign_resources(this, used_zmm_idx, reg_ret, k2);+ injector.assign_reg_elt_constp(reg_elt_constv);+ }++ auto store_zmm_v = [&](MemcpyStoreParam<Xbyak::Zmm> p) {+ if (p.tail) {+ vmovdqu8(ptr[p.store_addr], p.vmm_v | p.store_mask);+ } else {+ vmovups(ptr[p.store_addr], p.vmm_v);+ }+ };++ auto unpack_zmm_16bit_withfunc = [&](MemcpyStoreParam<Xbyak::Zmm> p,+ std::function<void(MemcpyStoreParam<Xbyak::Zmm>)> func,+ BTLA_DTYPE BIT16_DT) {+ vmovups(zmm_tmps[0], p.vmm_v);+ Xbyak::Ymm ymm_v = Xbyak::Ymm(p.vmm_v.getIdx());+ if (BIT16_DT == BTLA_DTYPE::BF16) {+ vpmovzxwd(p.vmm_v, ymm_v);+ vpslld(p.vmm_v, p.vmm_v, 16);+ }+ if (BIT16_DT == BTLA_DTYPE::F16) vcvtph2psx(p.vmm_v, ymm_v);+ p.store_mask = k3;+ func(p);+ vextractf32x8(Xbyak::Ymm(zmm_tmps[0].getIdx()), zmm_tmps[0], 1);+ if (BIT16_DT == BTLA_DTYPE::BF16) {+ vpmovzxwd(zmm_tmps[0], Xbyak::Ymm(zmm_tmps[0].getIdx()));+ vpslld(zmm_tmps[0], zmm_tmps[0], 16);+ }+ if (BIT16_DT == BTLA_DTYPE::F16) vcvtph2psx(zmm_tmps[0], Xbyak::Ymm(zmm_tmps[0].getIdx()));+ p.vmm_v = zmm_tmps[0];+ p.store_addr = p.store_addr + VBytes;+ p.store_mask = k4;+ func(p);+ };++ auto apply_postop_and_store = [&](MemcpyStoreParam<Xbyak::Zmm> p) {+ for (int k = 0; k < injectors.size(); k++) injectors[k].vector_compute(p.vmm_v, k * 3 * sizeof(float));+ if (dt_cvt_cfg.before_store == DataConvertConfig::cvt_direct::NO_CVT) {+ store_zmm_v(p);+ } else if (dt_cvt_cfg.before_store == DataConvertConfig::cvt_direct::BF16_TO_FP32) {+ unpack_zmm_16bit_withfunc(p, store_zmm_v, BTLA_DTYPE::BF16);+ } else if (dt_cvt_cfg.before_store == DataConvertConfig::cvt_direct::F16_TO_FP32) {+ unpack_zmm_16bit_withfunc(p, store_zmm_v, BTLA_DTYPE::F16);+ } else if (dt_cvt_cfg.before_store == DataConvertConfig::cvt_direct::FP32_TO_BF16) {+ Xbyak::Ymm ymm_v = Xbyak::Ymm(p.vmm_v.getIdx());+ if (device::CpuDevice::getInstance()->AVX512_BF16()) {+ vcvtneps2bf16(ymm_v, p.vmm_v);+ } else {+ vmovups(zmm_tmps[1], p.vmm_v);+ vpsrldq(p.vmm_v, p.vmm_v, 2);+ vpandd(p.vmm_v, p.vmm_v, zword_b[rip + data_label]);+ vpaddd(p.vmm_v, p.vmm_v, zword_b[rip + data_label + 4]);+ vpaddd(p.vmm_v, p.vmm_v, zmm_tmps[1]);+ vpsrld(p.vmm_v, p.vmm_v, 16);+ vpmovdw(ymm_v, p.vmm_v);+ }+ if (p.tail) {+ vmovdqu8(ptr[p.store_addr], ymm_v | k3);+ } else {+ vmovups(ptr[p.store_addr], ymm_v);+ }+ } else if (dt_cvt_cfg.before_store == DataConvertConfig::cvt_direct::FP32_TO_F16) {+ Xbyak::Ymm ymm_v = Xbyak::Ymm(p.vmm_v.getIdx());+ vcvtps2phx(ymm_v, p.vmm_v);+ if (p.tail) {+ vmovdqu8(ptr[p.store_addr], ymm_v | k3);+ } else {+ vmovups(ptr[p.store_addr], ymm_v);+ }+ } else {+ assert(0);+ }+ };++ auto load_store_value = [&](Xbyak::Zmm zmm_v, Xbyak::RegExp load_addr, Xbyak::RegExp store_addr,+ bool tail = false) {+ if (tail) {+ vmovdqu8(zmm_v | k1, ptr[load_addr]);+ } else {+ vmovups(zmm_v, ptr[load_addr]);+ }+ if (dt_cvt_cfg.before_postop == DataConvertConfig::cvt_direct::NO_CVT) {+ apply_postop_and_store({zmm_v, store_addr, tail});+ } else if (dt_cvt_cfg.before_postop == DataConvertConfig::cvt_direct::BF16_TO_FP32) {+ unpack_zmm_16bit_withfunc({zmm_v, store_addr, tail}, apply_postop_and_store, BTLA_DTYPE::BF16);+ } else if (dt_cvt_cfg.before_postop == DataConvertConfig::cvt_direct::F16_TO_FP32) {+ unpack_zmm_16bit_withfunc({zmm_v, store_addr, tail}, apply_postop_and_store, BTLA_DTYPE::F16);+ } else {+ assert(0);+ }+ };++ mov(reg_elt_constv, ptr[parambase + OFFSET(elt_const_v)]);+ mov(reg_src, ptr[parambase + OFFSET(srcptr)]);+ mov(reg_dst, ptr[parambase + OFFSET(dstptr)]);+ load32(reg_size, ptr[parambase + OFFSET(col)]);+ xor_(reg_iter, reg_iter);+ L(".colloop");+ mov(reg_tmp, reg_size);+ sub(reg_tmp, reg_iter);+ cmp(reg_tmp, ColUnroll * VBytes);+ jl(".maskproc", T_NEAR);+ for (int i = 0; i < ColUnroll; i++)+ load_store_value(Xbyak::Zmm(i), reg_src + i * VBytes, reg_dst + i * dt_cvt_cfg.get_store_process_bytes(VBytes));+ add(reg_src, ColUnroll * VBytes);+ add(reg_dst, ColUnroll * dt_cvt_cfg.get_store_process_bytes(VBytes));+ add(reg_iter, ColUnroll * VBytes);+ jmp(".colend", T_NEAR);+ L(".maskproc");+ generate_Nbitsmask(k1, reg_iter, reg_size, reg_tmp, reg_tmp2, VBytes);++ if (dt_cvt_cfg.before_store == DataConvertConfig::cvt_direct::FP32_TO_BF16) {+ push(reg_iter);+ push(reg_size);+ int vbytes = VBytes;+ // consider a case that input==bf16 but apply postop, betore store will be fp32_to_bf16 but need to normal gen+ // mask.+ if (dt_cvt_cfg.input_dt == BTLA_DTYPE::F32) {+ shr(reg_iter, 1);+ shr(reg_size, 1);+ vbytes /= 2;+ }+ generate_Nbitsmask(k3, reg_iter, reg_size, reg_tmp, reg_tmp2, vbytes);+ pop(reg_size);+ pop(reg_iter);+ }+ // once enable postop the data-type before store will not be bf16.+ if (dt_cvt_cfg.before_store == DataConvertConfig::cvt_direct::BF16_TO_FP32 ||+ dt_cvt_cfg.before_store == DataConvertConfig::cvt_direct::F16_TO_FP32) {+ push(reg_iter);+ push(reg_size);+ shl(reg_iter, 1);+ shl(reg_size, 1);+ generate_Nbitsmask(k3, reg_iter, reg_size, reg_tmp, reg_tmp2, VBytes);+ add(reg_iter, VBytes);+ generate_Nbitsmask(k4, reg_iter, reg_size, reg_tmp, reg_tmp2, VBytes);+ pop(reg_size);+ pop(reg_iter);+ }+ load_store_value(Xbyak::Zmm(0), reg_src, reg_dst, true);+ add(reg_src, VBytes);+ add(reg_dst, dt_cvt_cfg.get_store_process_bytes(VBytes));+ add(reg_iter, VBytes);+ L(".colend");+ cmp(reg_iter, reg_size);+ jb(".colloop");+ mov(reg_ret, 0);+ vreg_pop(rsp);+ }+ outLocalLabel(); // end of local label+ L(data_label);+ uint32_t bf16_cvt_magic_num[2] = {0x00000001, 0X00007FFF};+ db(reinterpret_cast<uint8_t*>(bf16_cvt_magic_num), sizeof(bf16_cvt_magic_num));+ for (auto&& injector : injectors) injector.prepare_table();+ this->ready();+ mKernel = this->getCode<func_t>();+ }++ func_t mKernel = nullptr;+ std::set<int> used_zmm_idx;+};++static inline Xbyak::Zmm unpack_4bit(Xbyak::CodeGenerator* jit, Xbyak::Ymm v4bits, Xbyak::Zmm zmm, Xbyak::Zmm zmm1,+ Xbyak::Zmm vmask, Xbyak::Opmask unpack_mask) {+ Xbyak::Ymm ymm1(zmm1.getIdx());+ jit->vpmovsxbw(zmm, v4bits);+ jit->vpslld(ymm1, v4bits, 4);+ jit->vpmovsxbw(zmm1, ymm1);+ jit->vpsllw(zmm, zmm, 8);+ jit->vmovdqu8(zmm1 | unpack_mask, zmm);+ jit->vpandd(zmm1, vmask, zmm1);+ return zmm1;+}++static inline Xbyak::Zmm unpack_4bit_2regs(Xbyak::CodeGenerator* jit, Xbyak::Ymm v4bits, Xbyak::Zmm tmp,+ Xbyak::Zmm vmask, Xbyak::Opmask unpack_mask) {+ Xbyak::Zmm dst(v4bits.getIdx());+ jit->vpmovsxbw(tmp, v4bits);+ jit->vpslld(v4bits, v4bits, 4);+ jit->vpmovsxbw(dst, v4bits);+ jit->vpsllw(tmp, tmp, 8);+ jit->vmovdqu8(dst | unpack_mask, tmp);+ jit->vpandd(dst, vmask, dst);+ return dst;+}++class DecompressS4S8_AVX512F : protected xbyak::JitAvx512f {+ public:+ struct params {+ void *srcptr, *dstptr;+ size_t size;+ };+ typedef long long (*func_t)(params*);++ public:+ static int constexpr VBytes = 64;+ DecompressS4S8_AVX512F() {+ inLocalLabel(); // use local label for multiple instance+ int SF_TmpSize = 64;+ Xbyak::util::StackFrame st(this, 1, 13, 16 * 10 + SF_TmpSize);+ const Xbyak::Reg64& parambase = st.p[0];+ const Xbyak::Reg64& reg_srcptr = st.t[0];+ const Xbyak::Reg64& reg_dstptr = st.t[1];+ const Xbyak::Reg64& reg_size = st.t[5];+ const Xbyak::Reg64& reg_iterrow = st.t[6];+ const Xbyak::Reg64& reg_itercol = st.t[7];+ const Xbyak::Reg64& reg_tmp = st.t[8];+ const Xbyak::Reg64& reg_tmp1 = st.t[12];+ const Xbyak::Reg64& reg_ret = rax;++ vreg_push(rsp);++ mov(reg_srcptr, ptr[parambase + OFFSET(srcptr)]);+ mov(reg_dstptr, ptr[parambase + OFFSET(dstptr)]);+ mov(reg_size, ptr[parambase + OFFSET(size)]);+ Xbyak::Opmask unpack_mask(4);+ Xbyak::Zmm zmm_mask(31);+ mov(reg_tmp.cvt32(), uint32_t(0xf0f0f0f0));+ vpbroadcastd(zmm_mask, reg_tmp.cvt32());+ mov(reg_tmp, 0xaaaaaaaaaaaaaaaa);+ kmovq(unpack_mask, reg_tmp);+ int const ColUnroll = 4;+ xor_(reg_iterrow, reg_iterrow);+ xor_(reg_itercol, reg_itercol);+ L(".colloop");+ mov(reg_tmp, reg_size);+ sub(reg_tmp, reg_itercol);+ cmp(reg_tmp, ColUnroll * VBytes);+ jl(".maskproc", T_NEAR);+ mov(reg_tmp, reg_itercol);+ shr(reg_tmp, 1);+ for (int i = 0; i < ColUnroll; i++) {+ vmovups(Xbyak::Ymm(i), ptr[reg_srcptr + reg_tmp + i * VBytes / 2]);+ unpack_4bit_2regs(this, Xbyak::Ymm(i), Xbyak::Zmm(ColUnroll), zmm_mask, unpack_mask);+ vmovups(ptr[reg_dstptr + reg_itercol + i * VBytes], Xbyak::Zmm(i));+ }+ add(reg_itercol, ColUnroll * VBytes);+ jmp(".colend");+ L(".maskproc");+ generate_Nbitsmask(k1, reg_itercol, reg_size, reg_tmp, reg_tmp1, VBytes);+ mov(reg_tmp, reg_itercol);+ shr(reg_tmp, 1);+ vmovdqu8(Xbyak::Zmm(0) | k1, ptr[reg_srcptr + reg_tmp]);+ unpack_4bit_2regs(this, Xbyak::Ymm(0), Xbyak::Zmm(ColUnroll), zmm_mask, unpack_mask);+ vmovdqu8(ptr[reg_dstptr + reg_itercol], Xbyak::Zmm(0) | k1);+ add(reg_itercol, VBytes);+ L(".colend");+ cmp(reg_itercol, reg_size);+ jb(".colloop");++ mov(reg_ret, 0);+ vreg_pop(rsp);+ outLocalLabel(); // end of local label++ this->ready();+ mKernel = this->getCode<func_t>();+ }++ static BTLA_CODE forward(void* srcptr, void* dstptr, size_t size) {+ static DecompressS4S8_AVX512F instance;+ auto param = params{srcptr, dstptr, size};+ instance.mKernel(¶m);+ return BTLA_CODE::Success;+ }++ private:+ func_t mKernel = nullptr;+};++static inline BTLA_CODE decompress_s4_s8(utils::int4x2* srcptr, int8_t* dstptr, int row, int col, int ld_src,+ int ld_dst) {+ if (col != ld_src) { // memory is not continuous+ return BTLA_CODE::NotSupport;+ }+ DecompressS4S8_AVX512F::forward(srcptr, dstptr, (size_t)row * col);+ return BTLA_CODE::Success;+}++// src: row x col => dst: ⌈col/n_tile⌉ x ⌈row/row_pack⌉ x n_tile x row_pack (zeor-padded)+// Extra padding can be applied with memset calls in `static void forward(...)`+class PaddingInterleaveCvt : protected xbyak::JitAvx512f {+ public:+ struct params {+ const void* srcptr;+ void* dstptr;+ int row, col;+ int srcstride, dststride; // dst = dst_base + dststride * n_idx, where n_idx % n_tile == 0+ };+ typedef void (*func_t)(params* p);+ void operator()(params* p) const { mKernel(p); }++ private:+ static inline const uint16_t idx_interleave_self[32] = {+ 0, 16, 1, 17, 2, 18, 3, 19, //+ 4, 20, 5, 21, 6, 22, 7, 23, //+ 8, 24, 9, 25, 10, 26, 11, 27, //+ 12, 28, 13, 29, 14, 30, 15, 31, //+ };++ PaddingInterleaveCvt(int n_tile, BTLA_DTYPE dst_t) : PaddingInterleaveCvt(n_tile, dst_t, dst_t) {}+ PaddingInterleaveCvt(int n_tile, BTLA_DTYPE dst_t, BTLA_DTYPE src_t, int row_pack = 0) : xbyak::JitAvx512f() {+ inLocalLabel(); // use local label for multiple instance+ const auto src_bytes = static_cast<int>(utils::bestla_dtype_bytes(src_t));+ const auto dst_bytes = static_cast<int>(utils::bestla_dtype_bytes(dst_t));+ if (row_pack == 0) row_pack = 4 / dst_bytes; // default value+ const auto ne_zmm = 64 / std::max(src_bytes, dst_bytes);+ const auto src_bytes_vmm = ne_zmm * src_bytes;++ assert(n_tile % ne_zmm == 0);+ assert(row_pack > 0 && row_pack < 3); // TODO(yi): int8 interleave not implemented++ int SF_TmpSize = 64;+ Xbyak::Label l_idx_interleave_self;+ std::shared_ptr<void> epilogue{+ // generate code at the very end+ nullptr, [&](void*) {+ align(64);+ L(l_idx_interleave_self);+ db(reinterpret_cast<const uint8_t*>(idx_interleave_self), sizeof(idx_interleave_self));+ outLocalLabel(); // end of local label++ this->ready();+ this->mKernel = this->getCode<func_t>();+ }};+ Xbyak::util::StackFrame st(this, 1, 13, 16 * 10 + SF_TmpSize);+ const Xbyak::Reg64& parambase = st.p[0];+ const Xbyak::Reg64& reg_srcptr = st.t[0];+ const Xbyak::Reg64& reg_dstptr = st.t[1];+ const Xbyak::Reg64& reg_srcstride = st.t[2];+ const Xbyak::Reg64& reg_dststride = st.t[3];+ const Xbyak::Reg64& reg_colsize = st.t[5];+ const Xbyak::Reg64& reg_iterrow = st.t[6];+ const Xbyak::Reg64& reg_itercol = st.t[7];+ const Xbyak::Reg64& reg_tmp = st.t[8];+ const Xbyak::Reg64& reg_tmp1 = st.t[9];+ const Xbyak::Reg64& reg_tmp2 = st.t[12];+ const Xbyak::Reg64& reg_tmp3 = st.t[10];++ const Xbyak::Reg64& reg_ret = rax;+ auto& mask_rd = k1;+ const Xbyak::Zmm& vreg_idx0 = zmm31;++ vreg_push(rsp);+ vmovups(vreg_idx0, zword[rip + l_idx_interleave_self]);+ mov(reg_srcptr, ptr[parambase + OFFSET(srcptr)]);+ mov(reg_dstptr, ptr[parambase + OFFSET(dstptr)]);+ mov(reg_srcstride.cvt32(), ptr[parambase + OFFSET(srcstride)]);+ mov(reg_dststride.cvt32(), ptr[parambase + OFFSET(dststride)]);+ mov(reg_colsize.cvt32(), ptr[parambase + OFFSET(col)]);++ std::vector<Xbyak::Zmm> reg_srcs(row_pack), reg_tmps(row_pack);+ const int ZIDX_TranSrc = 0;+ const int ZIDX_TransTmp = row_pack;+ for (int i = 0; i < row_pack; i++) reg_srcs[i] = Xbyak::Zmm(ZIDX_TranSrc + i);+ for (int i = 0; i < row_pack; i++) reg_tmps[i] = Xbyak::Zmm(ZIDX_TransTmp + i);++ xor_(reg_iterrow, reg_iterrow);+ L(".rowloop");+ xor_(reg_itercol, reg_itercol);+ mov(reg_tmp2.cvt32(), ptr[parambase + OFFSET(row)]);+ sub(reg_tmp2, reg_iterrow);+ cmp(reg_tmp2, row_pack);+ jb(".tailrowloop", T_NEAR);++ L(".colloop");+ mov(reg_tmp1, reg_itercol);+ imul(reg_tmp1, reg_dststride);+ lea(reg_tmp, ptr[reg_dstptr + reg_tmp1]);+ lea(reg_tmp1, ptr[reg_srcptr + reg_itercol * src_bytes]);+ for (int jj = 0; jj < n_tile; jj += ne_zmm) {+ generate_Nbitsmask(mask_rd, reg_itercol, ptr[reg_colsize - jj], reg_tmp2, reg_tmp3, ne_zmm);+ for (int ii = 0; ii < row_pack; ii++) {+ const Xbyak::Xmm reg_srcs_ii = src_bytes_vmm == 64 ? Xbyak::Zmm(reg_srcs[ii].getIdx())+ : src_bytes_vmm == 32 ? Xbyak::Ymm(reg_srcs[ii].getIdx())+ : src_bytes_vmm == 16 ? Xbyak::Xmm(reg_srcs[ii].getIdx())+ : (assert(false), reg_srcs[ii]);+ if (src_bytes == 1) {+ vmovdqu8(reg_srcs_ii | mask_rd | T_z, ptr[reg_tmp1 + ii * reg_srcstride + jj * src_bytes]);+ } else if (src_bytes == 2) {+ vmovdqu16(reg_srcs_ii | mask_rd | T_z, ptr[reg_tmp1 + ii * reg_srcstride + jj * src_bytes]);+ } else if (src_bytes == 4) {+ vmovdqu32(reg_srcs_ii | mask_rd | T_z, ptr[reg_tmp1 + ii * reg_srcstride + jj * src_bytes]);+ }+ }+ if (src_t == BTLA_DTYPE::F32 && dst_t == BTLA_DTYPE::BF16) {+ vcvtne2ps2bf16(reg_tmps[0], reg_srcs[1], reg_srcs[0]);+ vpermt2w(reg_tmps[0], vreg_idx0, reg_tmps[0]);+ vmovups(ptr[reg_tmp + jj * row_pack * dst_bytes], reg_tmps[0]);+ } else {+ // interleave_2rows_4regs(reg_srcs.data(), reg_tmps.data());+ assert(false); // Not implemented+ }+ }+ add(reg_itercol, n_tile);+ cmp(reg_itercol.cvt32(), ptr[parambase + OFFSET(col)]);+ jb(".colloop");+ lea(reg_srcptr, ptr[reg_srcptr + row_pack * reg_srcstride]);+ lea(reg_dstptr, ptr[reg_dstptr + row_pack * n_tile * dst_bytes]);++ add(reg_iterrow, row_pack);+ cmp(reg_iterrow.cvt32(), ptr[parambase + OFFSET(row)]);+ jb(".rowloop");+ jmp(".aftercolloop", T_NEAR);++ L(".tailrowloop");+ L(".tailcolloop");+ mov(reg_tmp1, reg_itercol);+ imul(reg_tmp1, reg_dststride);+ lea(reg_tmp, ptr[reg_dstptr + reg_tmp1]);+ lea(reg_tmp1, ptr[reg_srcptr + reg_itercol * src_bytes]);+ for (int jj = 0; jj < n_tile; jj += ne_zmm) {+ generate_Nbitsmask(mask_rd, reg_itercol, ptr[reg_colsize - jj], reg_tmp2, reg_tmp3, ne_zmm);+ if (row_pack == 2) {+ const Xbyak::Xmm reg_srcs_0 = src_bytes_vmm == 64 ? Xbyak::Zmm(reg_srcs[0].getIdx())+ : src_bytes_vmm == 32 ? Xbyak::Ymm(reg_srcs[0].getIdx())+ : src_bytes_vmm == 16 ? Xbyak::Xmm(reg_srcs[0].getIdx())+ : (assert(false), reg_srcs[0]);+ if (src_bytes == 1) {+ vmovdqu8(reg_srcs_0 | mask_rd | T_z, ptr[reg_tmp1 + jj * src_bytes]);+ } else if (src_bytes == 2) {+ vmovdqu16(reg_srcs_0 | mask_rd | T_z, ptr[reg_tmp1 + jj * src_bytes]);+ } else if (src_bytes == 4) {+ vmovdqu32(reg_srcs_0 | mask_rd | T_z, ptr[reg_tmp1 + jj * src_bytes]);+ }+ vxorps(reg_srcs[1], reg_srcs[1]);+ } else {+ assert(false);+ }+ if (src_t == BTLA_DTYPE::F32 && dst_t == BTLA_DTYPE::BF16) {+ vcvtne2ps2bf16(reg_tmps[0], reg_srcs[1], reg_srcs[0]);+ vpermt2w(reg_tmps[0], vreg_idx0, reg_tmps[0]);+ vmovups(ptr[reg_tmp + jj * row_pack * dst_bytes], reg_tmps[0]);+ } else {+ assert(false);+ }+ }+ add(reg_itercol, n_tile);+ cmp(reg_itercol.cvt32(), ptr[parambase + OFFSET(col)]);+ jb(".tailcolloop");+ L(".aftercolloop");+ mov(reg_ret, 0);+ vreg_pop(rsp);+ }++ func_t mKernel = nullptr;++ public:+ template <int NTile, typename T_SRC, typename T_DST = T_SRC, int RowPack = 4 / sizeof(T_DST)>+ static void forward(const T_SRC* src, T_DST* dst, int row, int col, int row_pad, int col_pad, int src_step,+ int dst_step) {+ const auto kern_col_pad = utils::padto(col, NTile);+ const auto kern_row_pad = utils::padto(row, RowPack);+ assert(kern_col_pad <= col_pad && col_pad % NTile == 0);+ assert(kern_row_pad <= row_pad && row_pad % RowPack == 0);+ const auto src_stride = static_cast<int>(sizeof(T_SRC)) * src_step;+ const auto dst_stride = static_cast<int>(sizeof(T_DST)) * dst_step;+ params param = {src, dst, row, col, src_stride, dst_stride};+ static const PaddingInterleaveCvt kern(NTile, utils::bestla_dtype<T_DST>, utils::bestla_dtype<T_SRC>, RowPack);+ kern(¶m);++ // extra row and col pad+ const auto row_pad_size_memset = sizeof(T_DST) * (row_pad - kern_row_pad) * NTile;+ if (row_pad_size_memset) {+ for (int j = 0; j < kern_col_pad; j += NTile)+ memset(dst + j * dst_step + kern_row_pad * NTile, 0, row_pad_size_memset);+ }+ for (int j = kern_col_pad; j < col_pad; j += NTile) //+ memset(dst + j * dst_step, 0, sizeof(T_DST) * NTile * row_pad);+ }++ template <int NTile, typename T_SRC, typename T_DST = T_SRC, int RowPack = 4 / sizeof(T_DST)>+ static void reference(const T_SRC* src, T_DST* dst, int row, int col, int row_pad, int col_pad, int src_step,+ int dst_step) {+ assert(utils::padto(col, NTile) <= col_pad && col_pad % NTile == 0);+ assert(utils::padto(row, RowPack) <= row_pad && row_pad % RowPack == 0);+ for (int i = 0; i < row_pad; i += RowPack)+ for (int j = 0; j < col_pad; j += NTile)+ for (int ii = 0; ii < RowPack; ++ii)+ for (int jj = 0; jj < NTile; ++jj)+ dst[i * NTile + j * dst_step + ii + jj * RowPack] =+ static_cast<T_DST>((i + ii < row && j + jj < col) ? src[(i + ii) * src_step + j + jj] : 0);+ }+};++// src: row x col => dst: ⌈row/m_tile⌉ x ⌈col/(trans_cell*col_pack==64/sizeof(t_dst))⌉ x m_tile x col_pack (zeor-padded)+// Note1: the extra padding on the dimension of col due to the implementation limitation+// Note2: dst will only be zero-padded to a multiple of trans_cell in the dimension of m_tile+// Extra padding can be applied with memset calls in `static void forward(...)`+class PaddingTransInterleaveCvt : protected xbyak::JitAvx512f {+ public:+ struct params {+ const void* srcptr;+ void* dstptr;+ int row, col;+ int srcstride; // src = src_base + srcstride * m_idx+ int dststride; // dst = dst_base + dststride * m_idx, where m_idx % m_tile == 0+ };+ typedef void (*func_t)(params* p);+ void operator()(params* p) const { mKernel(p); }+ const int trans_cell; // transpose matrices of size trans_cellxtrans_cell (in terms of #elements or #packs)++ private:+ PaddingTransInterleaveCvt(int m_tile, BTLA_DTYPE dst_t) : PaddingTransInterleaveCvt(m_tile, dst_t, dst_t) {}+ PaddingTransInterleaveCvt(int m_tile, BTLA_DTYPE dst_t, BTLA_DTYPE src_t, int col_pack = 0)+ : xbyak::JitAvx512f(), trans_cell(64 / col_pack / int(utils::bestla_dtype_bytes(dst_t))) {+ const auto src_bytes = static_cast<int>(utils::bestla_dtype_bytes(src_t));+ const auto dst_bytes = static_cast<int>(utils::bestla_dtype_bytes(dst_t));+ if (col_pack == 0) col_pack = 4 / dst_bytes; // default value+ // const auto src_bytes_vmm = ne_zmm * src_bytes;+ // const auto dst_bytes_vmm = ne_zmm * dst_bytes;++ assert(m_tile % trans_cell == 0);+ assert(col_pack > 0 && col_pack < 3); // TODO(yi): int8 interleave not implemented++ inLocalLabel(); // use local label for multiple instance+ std::shared_ptr<void> epilogue{// generate code at the very end+ nullptr, [&](void*) {+ outLocalLabel(); // end of local label++ this->ready();+ this->mKernel = this->getCode<func_t>();+ }};+ Xbyak::util::StackFrame st(this, 1, 11 | Xbyak::util::UseRDX, 16 * 10);+ const Xbyak::Reg64& parambase = st.p[0];+ const Xbyak::Reg64& reg_srcptr = st.t[0];+ const Xbyak::Reg64& reg_dstptr = st.t[1];+ const Xbyak::Reg64& reg_srcstride = st.t[2];+ const Xbyak::Reg64& reg_dststride = st.t[3];+ const Xbyak::Reg64& reg_colsize = st.t[4];+ const Xbyak::Reg64& reg_iterrow = st.t[5];+ const Xbyak::Reg64& reg_itercol = st.t[6];+ const Xbyak::Reg64& reg_tmp = st.t[7];+ const Xbyak::Reg64& reg_tmp2 = st.t[9];+ const Xbyak::Reg64& reg_tmp3 = st.t[10];++ const Xbyak::Reg64& reg_ret = rax;+ const auto& mask_rd = k1;+ const auto& mask_rd2 = k2;++ vreg_push(rsp);+ mov(reg_srcptr, ptr[parambase + OFFSET(srcptr)]);+ mov(reg_srcstride.cvt32(), ptr[parambase + OFFSET(srcstride)]);+ mov(reg_dststride.cvt32(), ptr[parambase + OFFSET(dststride)]);+ mov(reg_colsize.cvt32(), ptr[parambase + OFFSET(col)]);++ std::vector<Xbyak::Zmm> reg_srcs(trans_cell), reg_tmps(trans_cell);+ const int ZIDX_TranSrc = 0;+ const int ZIDX_TransTmp = trans_cell;+ for (int i = 0; i < trans_cell; i++) reg_srcs[i] = Xbyak::Zmm(ZIDX_TranSrc + i);+ for (int i = 0; i < trans_cell; i++) reg_tmps[i] = Xbyak::Zmm(ZIDX_TransTmp + i);++ xor_(reg_iterrow, reg_iterrow);+ L(".rowloop");+ xor_(rdx, rdx);+ mov(rax, reg_iterrow);+ mov(reg_tmp, m_tile);+ div(reg_tmp); // reg_iterrow `div` m_tile+ imul(reg_dstptr, rdx, col_pack * dst_bytes); // ii * col_pack+ add(reg_dstptr, ptr[parambase + OFFSET(dstptr)]);+ imul(reg_tmp, rax, m_tile);+ imul(reg_tmp, reg_dststride);+ lea(reg_dstptr, ptr[reg_dstptr + reg_tmp]); // dst = dst_base + i * dst_step + ii * col_pack+ xor_(reg_itercol, reg_itercol);++ mov(reg_tmp2.cvt32(), ptr[parambase + OFFSET(row)]);+ sub(reg_tmp2, reg_iterrow);+ cmp(reg_tmp2, trans_cell);+ jb(".tailrowloop", T_NEAR);++ L(".colloop");+ generate_Nbitsmask(mask_rd, reg_itercol, ptr[reg_colsize], reg_tmp2, reg_tmp3, 64 / dst_bytes);+ if (src_t == BTLA_DTYPE::F32 && dst_t == BTLA_DTYPE::BF16) {+ kshiftrq(mask_rd2, mask_rd, 16);+ assert(trans_cell == 16);+ for (int ii = 0; ii < trans_cell; ++ii) {+ lea(reg_tmp, (ii == 0) ? ptr[reg_srcptr + reg_itercol * src_bytes] : ptr[reg_tmp + reg_srcstride]);+ vmovups(reg_srcs[ii] | mask_rd | T_z, zword[reg_tmp]);+ vmovups(reg_tmps[ii] | mask_rd2 | T_z, zword[reg_tmp + 64]);+ vcvtne2ps2bf16(reg_srcs[ii], reg_tmps[ii], reg_srcs[ii]);+ }+ transpose16x16_4B(reg_srcs.data(), reg_tmps.data());+ for (int jj = 0; jj < trans_cell; ++jj) {+ vmovups(ptr[reg_dstptr + jj * m_tile * col_pack * dst_bytes], reg_srcs[jj]);+ }+ } else {+ assert(false); // Not implemented+ }+ lea(reg_dstptr, ptr[reg_dstptr + col_pack * trans_cell * dst_bytes * m_tile]);+ lea(reg_itercol, ptr[reg_itercol + col_pack * trans_cell]);+ cmp(reg_itercol.cvt32(), ptr[parambase + OFFSET(col)]);+ jb(".colloop");++ imul(reg_tmp, reg_srcstride, trans_cell);+ lea(reg_srcptr, ptr[reg_srcptr + reg_tmp]); // srcptr += trans_cell * srcstride+ lea(reg_iterrow, ptr[reg_iterrow + trans_cell]);+ cmp(reg_iterrow.cvt32(), ptr[parambase + OFFSET(row)]);+ jb(".rowloop");+ jmp(".aftercolloop", T_NEAR);++ L(".tailrowloop");+ // reg_itercol, reg_dstptr should have been set in the non-tail section+ Xbyak::Label l_tail_tbl;+ std::vector<Xbyak::Label> l_tail_case(trans_cell);+ mov(reg_tmp, l_tail_tbl); // TODO(Yi): rip + l + offset?+ jmp(ptr[reg_tmp + reg_tmp2 * sizeof(void*)], T_NEAR); // switch(rows-iterrow) ...+ align(sizeof(intptr_t));+ L(l_tail_tbl);+ db(reinterpret_cast<uintptr_t>(nullptr), sizeof(intptr_t)); // case 0 should never occur+ for (int i = 1; i < trans_cell; ++i) putL(l_tail_case[i]);++ for (int m_tail = 1; m_tail < trans_cell; ++m_tail) { // case (m_tail):+ auto& tailcolloop = l_tail_case[m_tail];+ L(tailcolloop);+ generate_Nbitsmask(mask_rd, reg_itercol, ptr[reg_colsize], reg_tmp2, reg_tmp3, 64 / dst_bytes);+ if (src_t == BTLA_DTYPE::F32 && dst_t == BTLA_DTYPE::BF16) {+ kshiftrq(mask_rd2, mask_rd, 16);+ assert(trans_cell == 16);+ for (int ii = 0; ii < trans_cell; ++ii) {+ if (ii < m_tail) {+ lea(reg_tmp, (ii == 0) ? ptr[reg_srcptr + reg_itercol * src_bytes] : ptr[reg_tmp + reg_srcstride]);+ vmovups(reg_srcs[ii] | mask_rd | T_z, zword[reg_tmp]);+ vmovups(reg_tmps[ii] | mask_rd2 | T_z, zword[reg_tmp + 64]);+ vcvtne2ps2bf16(reg_srcs[ii], reg_tmps[ii], reg_srcs[ii]);+ } else if (ii == m_tail) {+ vxorps(reg_srcs[ii], reg_srcs[ii], reg_srcs[ii]);+ } else {+ vmovaps(reg_srcs[ii], reg_srcs[m_tail]);+ }+ }+ transpose16x16_4B(reg_srcs.data(), reg_tmps.data());+ for (int jj = 0; jj < trans_cell; ++jj) {+ vmovups(ptr[reg_dstptr + jj * m_tile * col_pack * dst_bytes], reg_srcs[jj]);+ }+ } else {+ assert(false); // Not implemented+ }+ lea(reg_dstptr, ptr[reg_dstptr + col_pack * trans_cell * dst_bytes * m_tile]);+ lea(reg_itercol, ptr[reg_itercol + col_pack * trans_cell]);+ cmp(reg_itercol.cvt32(), ptr[parambase + OFFSET(col)]);+ jb(tailcolloop);+ jmp(".aftercolloop", T_NEAR);+ }++ L(".aftercolloop");+ mov(reg_ret, 0);+ vreg_pop(rsp);+ }++ func_t mKernel = nullptr;++ public:+ template <int MTile, typename T_SRC, typename T_DST = T_SRC, int ColPack = 4 / sizeof(T_DST)>+ static void forward(const T_SRC* src, T_DST* dst, int row, int col, int row_pad, int col_pad, int src_step,+ int dst_step) {+ assert(utils::padto(row, MTile) <= row_pad && row_pad % MTile == 0);+ assert(utils::padto(col, ColPack) <= col_pad && col_pad % ColPack == 0);+ static const PaddingTransInterleaveCvt kern(MTile, utils::bestla_dtype<T_DST>, utils::bestla_dtype<T_SRC>, ColPack);+ // 0-padded guarantee by jit kern+ const auto kern_row_pad = utils::padto(row, kern.trans_cell),+ kern_col_pad = utils::padto(col, kern.trans_cell * ColPack);+ assert(kern_row_pad <= row_pad && row_pad % MTile == 0);+ assert(kern_col_pad <= col_pad && col_pad % ColPack == 0);+ const auto src_stride = static_cast<int>(sizeof(T_SRC)) * src_step;+ const auto dst_stride = static_cast<int>(sizeof(T_DST)) * dst_step;+ params param = {src, dst, row, col, src_stride, dst_stride};+ kern(¶m);++ // extra row and col pad+ const auto col_pad_size_memset = sizeof(T_DST) * (col_pad - kern_col_pad) * MTile;+ if (col_pad_size_memset) {+ for (int i = 0; i < kern_row_pad; i += MTile)+ memset(dst + i * dst_step + kern_col_pad * MTile, 0, col_pad_size_memset);+ }+ const auto row_tail_pad_size_memset = sizeof(T_DST) * (utils::padto(row, MTile) - kern_row_pad) * ColPack;+ if (row_tail_pad_size_memset) { // row tail due to kernel limitation: kern_row_pad < next_multiple_of_MTile+ const auto kern_row_pad_le_mtile = utils::padto_le(kern_row_pad, MTile);+ const auto tail_dst_base = dst + kern_row_pad_le_mtile * dst_step + kern_row_pad % MTile * ColPack;+ for (int j = 0; j < kern_col_pad; j += ColPack) memset(tail_dst_base + j * MTile, 0, row_tail_pad_size_memset);+ }+ for (int j = utils::padto(row, MTile); j < row_pad; j += MTile)+ memset(dst + kern_row_pad * dst_step, 0, sizeof(T_DST) * MTile * col_pad);+ }++ template <int MTile, typename T_SRC, typename T_DST = T_SRC, int ColPack = 4 / sizeof(T_DST)>+ static void reference(const T_SRC* src, T_DST* dst, int row, int col, int row_pad, int col_pad, int src_step,+ int dst_step) {+ assert(utils::padto(row, MTile) <= row_pad && row_pad % MTile == 0);+ assert(utils::padto(col, ColPack) <= col_pad && col_pad % ColPack == 0);+ for (int i = 0; i < row_pad; i += MTile)+ for (int j = 0; j < col_pad; j += ColPack)+ for (int ii = 0; ii < MTile; ++ii)+ for (int jj = 0; jj < ColPack; ++jj)+ dst[j * MTile + i * dst_step + jj + ii * ColPack] =+ static_cast<T_DST>((j + jj < col && i + ii < row) ? src[(i + ii) * src_step + j + jj] : 0);+ }+};++// Complex number matrix(interleaved) - vector(as diagonal matrix) multiplication; Typically used for+// shift-RoPE+//+// vector: fp16 values; view every adjacent 2 values on colunm as a complex num+// src: bf16 ⌈row/row_pack⌉ x n_tile x row_pack; view every adjacent 2 values on colunm as a complex num+// dst: same as src+class CScaleInterleavedBF16FP16 : protected xbyak::JitAvx512_fp16 {+ public:+ struct params {+ void* srcptr;+ const void* scaleptr;+ int row;+ };+ typedef void (*func_t)(params* p);+ void operator()(params* p) const { mKernel(p); }++ private:+ explicit CScaleInterleavedBF16FP16(int n_tile, int n_off, int row_pack = 2, int unroll = 2)+ : xbyak::JitAvx512_fp16() {+ inLocalLabel(); // use local label for multiple instance+ assert(("n_tile must be a multiple of 16", n_tile % 16 == 0));+ assert(row_pack > 0 && row_pack < 3); // TODO(yi): int8 interleave not implemented+ int SF_TmpSize = 64;+ std::shared_ptr<void> epilogue{// generate code at the very end+ nullptr, [&](void*) {+ outLocalLabel(); // end of local label+ this->ready();+ this->mKernel = this->getCode<func_t>();+ }};+ Xbyak::util::StackFrame st(this, 1, 4, 16 * 10 + SF_TmpSize);+ const Xbyak::Reg64& parambase = st.p[0];+ const Xbyak::Reg64& reg_src = st.t[0];+ const Xbyak::Reg64& reg_scale = st.t[1];+ const Xbyak::Reg64& reg_rowsize = st.t[2];+ const Xbyak::Reg64& reg_iterrow = st.t[3];+ const Xbyak::Zmm& vreg_scale = zmm31;+ const auto& mask = k1;+ const auto masked_off = n_off % 16;+ if (masked_off != 0) {+ mov(reg_src, ((1ULL << (16 - masked_off)) - 1) << masked_off);+ kmovw(mask, reg_src.cvt32());+ }++ vreg_push(rsp);+ mov(reg_rowsize.cvt32(), ptr[parambase + OFFSET(row)]);+ mov(reg_src, qword[parambase + OFFSET(srcptr)]);+ mov(reg_scale, qword[parambase + OFFSET(scaleptr)]);++ std::vector<Xbyak::Zmm> vreg_src(4 * n_tile / 16);+ const int ZIDX_TranSrc = 0;+ for (int i = 0; i < 4 * n_tile / 16; i++) vreg_src[i] = Xbyak::Zmm(ZIDX_TranSrc + i);++ xor_(reg_iterrow, reg_iterrow);+ Xbyak::Label rowloop;+ L(rowloop);+ {+ assert(("only implement for pack2 bf16", row_pack == 2));+ for (int i = 0; i < unroll * row_pack; i += row_pack) {+ vpbroadcastd(vreg_scale, dword[reg_scale + reg_iterrow * sizeof(utils::fp16) + i * sizeof(utils::fp16)]);++ if (masked_off != 0) {+ int j = utils::padto_le(n_off, 16);++ const auto& vreg0 = vreg_src[j / 16 * 4 + 0];+ const auto& vreg1 = vreg_src[j / 16 * 4 + 1];+ const auto& vreg2 = vreg_src[j / 16 * 4 + 2];+ const auto& vreg3 = vreg_src[j / 16 * 4 + 3];+ vpmovzxwd(vreg0, yword[reg_src + (i * n_tile + j * row_pack) * sizeof(utils::bf16) + 0]);+ vpmovzxwd(vreg1, yword[reg_src + (i * n_tile + j * row_pack) * sizeof(utils::bf16) + 32]);+ vpslldq(vreg0, vreg0, 2);+ vpslldq(vreg1, vreg1, 2);+ vcvtps2phx(Xbyak::Ymm(vreg0.getIdx()), vreg0);+ vcvtps2phx(Xbyak::Ymm(vreg1.getIdx()), vreg1);+ // #UD If (dest_reg == src1_reg) or (dest_reg == src2_reg)+ vfmulcph(Xbyak::Ymm(vreg2.getIdx()), Xbyak::Ymm(vreg0.getIdx()), Xbyak::Ymm(vreg_scale.getIdx()));+ vfmulcph(Xbyak::Ymm(vreg3.getIdx()), Xbyak::Ymm(vreg1.getIdx()), Xbyak::Ymm(vreg_scale.getIdx()));+ vcvtph2psx(vreg0, Xbyak::Ymm(vreg2.getIdx()));+ vcvtph2psx(vreg1, Xbyak::Ymm(vreg3.getIdx()));+ vcvtne2ps2bf16(vreg0, vreg1, vreg0);+ vmovups(zword[reg_src + (i * n_tile + j * row_pack) * sizeof(utils::bf16)] | mask, vreg0);+ }++ for (int j = utils::padto(n_off, 16); j < n_tile; j += 16) {+ const auto& vreg0 = vreg_src[j / 16 * 4 + 0];+ const auto& vreg1 = vreg_src[j / 16 * 4 + 1];+ const auto& vreg2 = vreg_src[j / 16 * 4 + 2];+ const auto& vreg3 = vreg_src[j / 16 * 4 + 3];+ vpmovzxwd(vreg0, yword[reg_src + (i * n_tile + j * row_pack) * sizeof(utils::bf16) + 0]);+ vpmovzxwd(vreg1, yword[reg_src + (i * n_tile + j * row_pack) * sizeof(utils::bf16) + 32]);+ vpslldq(vreg0, vreg0, 2);+ vpslldq(vreg1, vreg1, 2);+ vcvtps2phx(Xbyak::Ymm(vreg0.getIdx()), vreg0);+ vcvtps2phx(Xbyak::Ymm(vreg1.getIdx()), vreg1);+ // #UD If (dest_reg == src1_reg) or (dest_reg == src2_reg)+ vfmulcph(Xbyak::Ymm(vreg2.getIdx()), Xbyak::Ymm(vreg0.getIdx()), Xbyak::Ymm(vreg_scale.getIdx()));+ vfmulcph(Xbyak::Ymm(vreg3.getIdx()), Xbyak::Ymm(vreg1.getIdx()), Xbyak::Ymm(vreg_scale.getIdx()));+ vcvtph2psx(vreg0, Xbyak::Ymm(vreg2.getIdx()));+ vcvtph2psx(vreg1, Xbyak::Ymm(vreg3.getIdx()));+ vcvtne2ps2bf16(vreg0, vreg1, vreg0);+ vmovups(zword[reg_src + (i * n_tile + j * row_pack) * sizeof(utils::bf16)], vreg0);+ }+ }+ }+ lea(reg_iterrow, ptr[reg_iterrow + unroll * row_pack]);+ lea(reg_src, ptr[reg_src + unroll * row_pack * n_tile * sizeof(utils::bf16)]);+ cmp(reg_iterrow, reg_rowsize);+ jb(rowloop);++ vreg_pop(rsp);+ }++ func_t mKernel = nullptr;++ public:+ template <int NTile, int RowPack = 2>+ static void forward(utils::bf16* src, const utils::fp16* scale, int row, int col, int src_step, int n_offset) {+ static_assert(RowPack == 2, "Only implement rowpack2 bf16");+ static_assert(NTile % 16 == 0, "NTile must be a multiple of 16");+ constexpr auto unroll = 2;+ assert(("row should be paded", row % (RowPack * unroll) == 0));+ assert(("cow should be paded", col % NTile == 0));+ assert(("can not skip more than col", n_offset < col));+ int j = utils::padto_le(n_offset, NTile);+ if (n_offset % NTile != 0) {+ static const CScaleInterleavedBF16FP16 kern_off(NTile, n_offset % NTile, RowPack, unroll);+ params param = {src + j * src_step, scale, row};+ kern_off(¶m);+ j += NTile;+ }++ for (; j < col; j += NTile) {+ static const CScaleInterleavedBF16FP16 kern(NTile, 0, RowPack, unroll);+ params param = {src + j * src_step, scale, row};+ kern(¶m);+ }+ }++ template <int NTile, int RowPack = 2>+ static void reference(utils::bf16* src, const utils::fp16* scale, int row, int col, int src_step, int n_offset) {+ static_assert(RowPack == 2, "Only implement rowpack2 bf16");+ static_assert(NTile % 16 == 0, "NTile must be a multiple of 16");+ assert(("row should be paded", row % RowPack == 0));+ assert(("cow should be paded", col % NTile == 0));+ assert(("can not skip more than col", n_offset < col));+ for (int j = 0; j < col; j += NTile) {+ for (int i = 0; i < row; i += RowPack) {+ for (int jj = 0; jj < NTile; ++jj) {+ if (j + jj < n_offset) continue;+ auto& rel = (src + j * src_step)[i * NTile + jj * RowPack + 0];+ auto& img = (src + j * src_step)[i * NTile + jj * RowPack + 1];+ const auto rel_f32 = static_cast<float>(rel);+ const auto img_f32 = static_cast<float>(img);+ const auto rel_scale = static_cast<float>(scale[i + 0]);+ const auto img_scale = static_cast<float>(scale[i + 1]);+ rel = static_cast<utils::bf16>(rel_f32 * rel_scale - img_f32 * img_scale);+ img = static_cast<utils::bf16>(rel_f32 * img_scale + img_f32 * rel_scale);+ }+ }+ }+ }+};++} // namespace jit+} // namespace kernel+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_jit_injector.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_jit_injector.hnew file mode 100644index 000000000..08e24565a--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_jit_injector.h@@ -0,0 +1,934 @@+// Copyright (c) 2022 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once++#include <utility>+#include <string>+#include <vector>+#include <unordered_map>+#include <map>+#include <set>+#include <array>++#include "bestla.h"+#include "bestla_utils.h"+#include "xbyak/xbyak.h"++namespace bestla {+namespace kernel {+namespace jit_injector {+using Zmm = Xbyak::Zmm;+using Ymm = Xbyak::Ymm;+using Xmm = Xbyak::Xmm;+class eltwise_injector {+ public:+ eltwise_injector(BTLA_ELTWISEOP eltwiseop) : elt_op(eltwiseop) { reigster_table_entries(); }+ virtual ~eltwise_injector() {}++ void assign_resources(Xbyak::CodeGenerator* ptr, const std::set<int>& used_zmm_idx, const Xbyak::Reg64& table_reg,+ const Xbyak::Opmask& mask_reg) {+ h = ptr;+ k_mask = mask_reg;+ p_table = table_reg;+ assert(used_zmm_idx.size() <= 26);+ assign_zmm(used_zmm_idx, &zmm_mask);+ assign_zmm(used_zmm_idx, &zmm_aux0);+ assign_zmm(used_zmm_idx, &zmm_aux1);+ assign_zmm(used_zmm_idx, &zmm_aux2);+ assign_zmm(used_zmm_idx, &zmm_aux3);+ assign_zmm(used_zmm_idx, &zmm_aux4);+ }+ void assign_resources(Xbyak::CodeGenerator* ptr, const std::set<int>& used_ymm_idx, const Xbyak::Reg64& table_reg) {+ h = ptr;+ p_table = table_reg;+ assert(used_ymm_idx.size() <= 10);+ assign_ymm(used_ymm_idx, &ymm_mask);+ assign_ymm(used_ymm_idx, &ymm_aux0);+ assign_ymm(used_ymm_idx, &ymm_aux1);+ assign_ymm(used_ymm_idx, &ymm_aux2);+ assign_ymm(used_ymm_idx, &ymm_aux3);+ assign_ymm(used_ymm_idx, &ymm_aux4);+ }+ void assign_reg_elt_constp(const Xbyak::Reg64& reg) { reg_rt_const_p = reg; }+ void vector_compute(const Xbyak::Zmm& zmm_src, int const_p_offset = 0) {+ load_table_addr();+ switch (elt_op) {+ case BTLA_ELTWISEOP::EXP:+ exp_compute_vector_fwd(zmm_src);+ break;+ case BTLA_ELTWISEOP::TANH:+ tanh_compute_vector_fwd(zmm_src);+ break;+ case BTLA_ELTWISEOP::GELU:+ gelu_compute_vector_fwd(zmm_src);+ break;+ case BTLA_ELTWISEOP::RELU:+ relu_compute_vector_fwd(zmm_src, const_p_offset);+ break;+ case BTLA_ELTWISEOP::LINEAR:+ linear_compute_vector_fwd(zmm_src, const_p_offset);+ break;+ case BTLA_ELTWISEOP::LOW_PRECISION_EXP:+ low_precision_exp_compute_vector_fwd(zmm_src);+ break;+ case BTLA_ELTWISEOP::SWISH:+ swish_compute_vector_fwd(zmm_src, const_p_offset);+ break;+ default:+ assert(false);+ break;+ }+ }+ void vector_compute(const Xbyak::Ymm& ymm_src, int const_p_offset = 0) {+ load_table_addr();+ switch (elt_op) {+ case BTLA_ELTWISEOP::EXP:+ exp_compute_vector_fwd(ymm_src);+ break;+ case BTLA_ELTWISEOP::TANH:+ tanh_compute_vector_fwd(ymm_src);+ break;+ case BTLA_ELTWISEOP::GELU:+ gelu_compute_vector_fwd(ymm_src);+ break;+ case BTLA_ELTWISEOP::LOW_PRECISION_EXP:+ low_precision_exp_compute_vector_fwd(ymm_src);+ break;+ case BTLA_ELTWISEOP::SWISH:+ swish_compute_vector_fwd(ymm_src, const_p_offset);+ break;+ default:+ assert(false);+ break;+ }+ }+ void prepare_table() {+ h->align(64);+ h->L(l_table);+ assert(sizeof(table_entry_val_t) == 4); // sizeof(table_entry_val_t) should be 4+ for (auto it = entry_map.begin(); it != entry_map.end(); it++) {+ const auto& te = (*it).second;+ const auto len = te.bcast ? 64u : sizeof(table_entry_val_t);+ for (size_t d = 0; d < len; d += sizeof(table_entry_val_t)) h->dd(te.val);+ }+ }++ private:+ void reigster_table_entries() {+ static const table_t common_values{+ {zero, {0x00000000, true}}, {half, {0x3f000000, true}}, {one, {0x3f800000, true}},+ {two, {0x40000000, true}}, {minus_one, {0xbf800000, true}}, {minus_two, {0xc0000000, true}},+ {ln2f, {0x3f317218, true}}, {one_epi32, {0x00000001, true}}, {positive_mask, {0x7fffffff, true}},+ {sign_mask, {0x80000000, true}}, {exponent_bias, {0x0000007f, true}},+ };++ static constexpr std::array<float, 3> exp_approx_f32_coeff{0.35815147f, 0.96963238f, 1.f};+ static const table_t low_precision_exp_consts{+ {low_precision_exp_const_v0, {utils::bit_cast<uint32_t>(exp_approx_f32_coeff[0]), true}},+ {low_precision_exp_const_v1, {utils::bit_cast<uint32_t>(exp_approx_f32_coeff[1]), true}},+ {low_precision_exp_const_v2, {utils::bit_cast<uint32_t>(exp_approx_f32_coeff[2]), true}},+ };++ static const table_t exp_consts{{exp_log2ef, {0x3fb8aa3b, true}},+ {exp_ln_flt_max_f, {0x42b17218, true}},+ {exp_ln_flt_min_f, {0xc2aeac50, true}}};++ static const table_t exp_polynomial{+ // p0 = 1.0f+ {exp_pol, {0x3f7ffffb, true}}, // p1 = 0.999999701f+ {exp_pol, {0x3efffee3, true}}, // p2 = 0.499991506f+ {exp_pol, {0x3e2aad40, true}}, // p3 = 0.166676521f+ {exp_pol, {0x3d2b9d0d, true}}, // p4 = 0.0418978221f+ {exp_pol, {0x3c07cfce, true}} // p5 = 0.00828929059f+ };++ static const table_t gelu_tanh_const{{gelu_tanh_fitting_const, {0x3d372713, true}},+ {gelu_tanh_fitting_const_times_three, {0x3e095d4f, true}},+ {gelu_tanh_sqrt_two_over_pi, {0x3f4c422a, true}},+ {gelu_tanh_flt_max_x, {0x4154C480, true}},+ {gelu_tanh_flt_min_x, {0xC154C480, true}}};++ // tanh(x) constants for four interval approximation+ static const table_t tanh_consts{{tanh_idx_bias, {0x39800000, true}},+ {tanh_idx_mask, {0xffc00000, true}},+ {tanh_linear_ubound, {0x39ddb3d7, true}},+ {tanh_saturation_lbound, {0x41102cb3, true}}};++ // tanh(x) polynomial approximation+ // For each coefficient, there is 32 entries+ static const table_t tanh_polynomial_table{+ // coefficients of degree 0+ {tanh_pol_table, {0x00000000, false}},+ {tanh_pol_table, {0x39bfffff, false}},+ {tanh_pol_table, {0x39ffffff, false}},+ {tanh_pol_table, {0x3a3ffffe, false}},+ {tanh_pol_table, {0x3a7ffffb, false}},+ {tanh_pol_table, {0x3abffff7, false}},+ {tanh_pol_table, {0x3affffeb, false}},+ {tanh_pol_table, {0x3b3fffdc, false}},+ {tanh_pol_table, {0x3b7fffab, false}},+ {tanh_pol_table, {0x3bbfff70, false}},+ {tanh_pol_table, {0x3bfffeab, false}},+ {tanh_pol_table, {0x3c3ffdc0, false}},+ {tanh_pol_table, {0x3c7ffaab, false}},+ {tanh_pol_table, {0x3cbff701, false}},+ {tanh_pol_table, {0x3cffeaad, false}},+ {tanh_pol_table, {0x3d3fdc08, false}},+ {tanh_pol_table, {0x3d7faacd, false}},+ {tanh_pol_table, {0x3dbf7081, false}},+ {tanh_pol_table, {0x3dfeacc9, false}},+ {tanh_pol_table, {0x3e3dc7fd, false}},+ {tanh_pol_table, {0x3e7acbf5, false}},+ {tanh_pol_table, {0x3eb77a9f, false}},+ {tanh_pol_table, {0x3eec9a9f, false}},+ {tanh_pol_table, {0x3f22991f, false}},+ {tanh_pol_table, {0x3f42f7d6, false}},+ {tanh_pol_table, {0x3f67b7cc, false}},+ {tanh_pol_table, {0x3f76ca83, false}},+ {tanh_pol_table, {0x3f7ebbe9, false}},+ {tanh_pol_table, {0x3f7fd40c, false}},+ {tanh_pol_table, {0x3f7fff32, false}},+ {tanh_pol_table, {0x3f7ffffc, false}},+ {tanh_pol_table, {0x3f800000, false}},+ // coefficients of degree 1+ {tanh_pol_table, {0x3f800000, false}},+ {tanh_pol_table, {0x3f800018, false}},+ {tanh_pol_table, {0x3f7fffe8, false}},+ {tanh_pol_table, {0x3f7fffda, false}},+ {tanh_pol_table, {0x3f7fffdc, false}},+ {tanh_pol_table, {0x3f7fffdc, false}},+ {tanh_pol_table, {0x3f7fffac, false}},+ {tanh_pol_table, {0x3f7fff70, false}},+ {tanh_pol_table, {0x3f7ffeec, false}},+ {tanh_pol_table, {0x3f7ffdc0, false}},+ {tanh_pol_table, {0x3f7ffbed, false}},+ {tanh_pol_table, {0x3f7ff704, false}},+ {tanh_pol_table, {0x3f7feff5, false}},+ {tanh_pol_table, {0x3f7fdbca, false}},+ {tanh_pol_table, {0x3f7fbfff, false}},+ {tanh_pol_table, {0x3f7f7041, false}},+ {tanh_pol_table, {0x3f7f009b, false}},+ {tanh_pol_table, {0x3f7dc36c, false}},+ {tanh_pol_table, {0x3f7c0aa8, false}},+ {tanh_pol_table, {0x3f7734b8, false}},+ {tanh_pol_table, {0x3f70a4de, false}},+ {tanh_pol_table, {0x3f5f1fd8, false}},+ {tanh_pol_table, {0x3f495493, false}},+ {tanh_pol_table, {0x3f18b9ec, false}},+ {tanh_pol_table, {0x3ed706cb, false}},+ {tanh_pol_table, {0x3e390b06, false}},+ {tanh_pol_table, {0x3d90b11f, false}},+ {tanh_pol_table, {0x3c21a053, false}},+ {tanh_pol_table, {0x3aaf7fdb, false}},+ {tanh_pol_table, {0x37ccc1a3, false}},+ {tanh_pol_table, {0x355c6733, false}},+ {tanh_pol_table, {0x00000000, false}},+ // coefficients of degree 2+ {tanh_pol_table, {0x00000000, false}},+ {tanh_pol_table, {0xbe4e0ff1, false}},+ {tanh_pol_table, {0x3d25b1b1, false}},+ {tanh_pol_table, {0x3d6b6dab, false}},+ {tanh_pol_table, {0x3c9fb1d5, false}},+ {tanh_pol_table, {0xbabff06f, false}},+ {tanh_pol_table, {0x3c07b3f6, false}},+ {tanh_pol_table, {0xbb3fc1bc, false}},+ {tanh_pol_table, {0x3a9f5921, false}},+ {tanh_pol_table, {0xbbbf06f2, false}},+ {tanh_pol_table, {0xbbb0f402, false}},+ {tanh_pol_table, {0xbc47db9e, false}},+ {tanh_pol_table, {0xbc73d5e7, false}},+ {tanh_pol_table, {0xbca25bda, false}},+ {tanh_pol_table, {0xbcfca780, false}},+ {tanh_pol_table, {0xbd40e07c, false}},+ {tanh_pol_table, {0xbd7dab03, false}},+ {tanh_pol_table, {0xbdbe4a0f, false}},+ {tanh_pol_table, {0xbdfb14a5, false}},+ {tanh_pol_table, {0xbe36cc8d, false}},+ {tanh_pol_table, {0xbe6bd102, false}},+ {tanh_pol_table, {0xbe9fe7c5, false}},+ {tanh_pol_table, {0xbeba0f10, false}},+ {tanh_pol_table, {0xbec206a8, false}},+ {tanh_pol_table, {0xbea3c388, false}},+ {tanh_pol_table, {0xbe277d62, false}},+ {tanh_pol_table, {0xbd8b7960, false}},+ {tanh_pol_table, {0xbc209f49, false}},+ {tanh_pol_table, {0xbaad44ca, false}},+ {tanh_pol_table, {0xb7c6eeac, false}},+ {tanh_pol_table, {0xb663aa41, false}},+ {tanh_pol_table, {0x00000000, false}},+ // coefficients of degree 3+ {tanh_pol_table, {0x00000000, false}},+ {tanh_pol_table, {0x45b3ae96, false}},+ {tanh_pol_table, {0xc414eb20, false}},+ {tanh_pol_table, {0xc450e02e, false}},+ {tanh_pol_table, {0xc3152b4e, false}},+ {tanh_pol_table, {0xbead2f56, false}},+ {tanh_pol_table, {0xc2162e02, false}},+ {tanh_pol_table, {0xbeb4bd5a, false}},+ {tanh_pol_table, {0xc11a59a4, false}},+ {tanh_pol_table, {0xbed2f507, false}},+ {tanh_pol_table, {0xc020d32c, false}},+ {tanh_pol_table, {0x3dd0f506, false}},+ {tanh_pol_table, {0xbf2a75e2, false}},+ {tanh_pol_table, {0xbff950e3, false}},+ {tanh_pol_table, {0xbed47334, false}},+ {tanh_pol_table, {0xbe809b8c, false}},+ {tanh_pol_table, {0xbeb64532, false}},+ {tanh_pol_table, {0xbe961a5b, false}},+ {tanh_pol_table, {0xbe9b63ac, false}},+ {tanh_pol_table, {0xbea0d4b2, false}},+ {tanh_pol_table, {0xbe828a77, false}},+ {tanh_pol_table, {0xbe378612, false}},+ {tanh_pol_table, {0xbdc20908, false}},+ {tanh_pol_table, {0x3d2d3957, false}},+ {tanh_pol_table, {0x3dd46e89, false}},+ {tanh_pol_table, {0x3db3f629, false}},+ {tanh_pol_table, {0x3d2c5e7b, false}},+ {tanh_pol_table, {0x3bd20403, false}},+ {tanh_pol_table, {0x3a59dfae, false}},+ {tanh_pol_table, {0x3770af45, false}},+ {tanh_pol_table, {0x372cc014, false}},+ {tanh_pol_table, {0x00000000, false}},+ // coefficients of degree 4+ {tanh_pol_table, {0x00000000, false}},+ {tanh_pol_table, {0xcc981a1b, false}},+ {tanh_pol_table, {0x4a7edd3d, false}},+ {tanh_pol_table, {0x4ab1007c, false}},+ {tanh_pol_table, {0x48fedd9c, false}},+ {tanh_pol_table, {0x41a557b5, false}},+ {tanh_pol_table, {0x477ee32a, false}},+ {tanh_pol_table, {0x422557f5, false}},+ {tanh_pol_table, {0x45ff3ce4, false}},+ {tanh_pol_table, {0x42a55641, false}},+ {tanh_pol_table, {0x446e0867, false}},+ {tanh_pol_table, {0xc33dc19a, false}},+ {tanh_pol_table, {0x42915214, false}},+ {tanh_pol_table, {0x43af4fad, false}},+ {tanh_pol_table, {0x4110fe88, false}},+ {tanh_pol_table, {0xc1099b75, false}},+ {tanh_pol_table, {0x3fc8a8dc, false}},+ {tanh_pol_table, {0xbfbeaef5, false}},+ {tanh_pol_table, {0xbe365aad, false}},+ {tanh_pol_table, {0x3f4d9652, false}},+ {tanh_pol_table, {0x3ddfa08f, false}},+ {tanh_pol_table, {0x3e34e9b8, false}},+ {tanh_pol_table, {0x3e2d07a6, false}},+ {tanh_pol_table, {0x3dc63567, false}},+ {tanh_pol_table, {0x3cdaeb78, false}},+ {tanh_pol_table, {0xbcd17537, false}},+ {tanh_pol_table, {0xbc92829c, false}},+ {tanh_pol_table, {0xbb43ab99, false}},+ {tanh_pol_table, {0xb9b471dd, false}},+ {tanh_pol_table, {0xb6baad5a, false}},+ {tanh_pol_table, {0xb78bafc7, false}},+ {tanh_pol_table, {0x00000000, false}},+ // coefficients of degree 5+ {tanh_pol_table, {0x00000000, false}},+ {tanh_pol_table, {0x52f688d5, false}},+ {tanh_pol_table, {0xd0505c72, false}},+ {tanh_pol_table, {0xd08f98e3, false}},+ {tanh_pol_table, {0xce505cc9, false}},+ {tanh_pol_table, {0xc7162b8a, false}},+ {tanh_pol_table, {0xcc5061d6, false}},+ {tanh_pol_table, {0xc7162bdf, false}},+ {tanh_pol_table, {0xca50b37f, false}},+ {tanh_pol_table, {0xc7162a3a, false}},+ {tanh_pol_table, {0xc8422086, false}},+ {tanh_pol_table, {0x471a714e, false}},+ {tanh_pol_table, {0xc5ece1f1, false}},+ {tanh_pol_table, {0xc70e3d90, false}},+ {tanh_pol_table, {0xc3eba94a, false}},+ {tanh_pol_table, {0x43e0c424, false}},+ {tanh_pol_table, {0xc21f4552, false}},+ {tanh_pol_table, {0x42217cc8, false}},+ {tanh_pol_table, {0x405e7dc4, false}},+ {tanh_pol_table, {0xc10dd401, false}},+ {tanh_pol_table, {0x3e96b602, false}},+ {tanh_pol_table, {0xbd1a6d2f, false}},+ {tanh_pol_table, {0xbd393883, false}},+ {tanh_pol_table, {0xbd674682, false}},+ {tanh_pol_table, {0xbd310016, false}},+ {tanh_pol_table, {0xb961e269, false}},+ {tanh_pol_table, {0x3ba32495, false}},+ {tanh_pol_table, {0x3a7680d5, false}},+ {tanh_pol_table, {0x38b3173c, false}},+ {tanh_pol_table, {0x35a9deea, false}},+ {tanh_pol_table, {0x375c3f2a, false}},+ {tanh_pol_table, {0x00000000, false}},+ // coefficients of degree 6+ {tanh_pol_table, {0x00000000, false}},+ {tanh_pol_table, {0xd8995ed1, false}},+ {tanh_pol_table, {0x558285ea, false}},+ {tanh_pol_table, {0x55b2cd69, false}},+ {tanh_pol_table, {0x53028625, false}},+ {tanh_pol_table, {0x4bc9991f, false}},+ {tanh_pol_table, {0x5082898a, false}},+ {tanh_pol_table, {0x4b4999b3, false}},+ {tanh_pol_table, {0x4e02c07c, false}},+ {tanh_pol_table, {0x4ac99764, false}},+ {tanh_pol_table, {0x4b72c822, false}},+ {tanh_pol_table, {0xca40c0e1, false}},+ {tanh_pol_table, {0x489413e4, false}},+ {tanh_pol_table, {0x49b12224, false}},+ {tanh_pol_table, {0x46134c4e, false}},+ {tanh_pol_table, {0xc60c2d57, false}},+ {tanh_pol_table, {0x43c83910, false}},+ {tanh_pol_table, {0xc3c872d1, false}},+ {tanh_pol_table, {0xc186bc9e, false}},+ {tanh_pol_table, {0x42325bc3, false}},+ {tanh_pol_table, {0xbf2ffa4a, false}},+ {tanh_pol_table, {0x3d9a203c, false}},+ {tanh_pol_table, {0xbc545a43, false}},+ {tanh_pol_table, {0xbae08fee, false}},+ {tanh_pol_table, {0x3c80225d, false}},+ {tanh_pol_table, {0x3b1fd1df, false}},+ {tanh_pol_table, {0xba36b9d1, false}},+ {tanh_pol_table, {0xb91de544, false}},+ {tanh_pol_table, {0xb71f100f, false}},+ {tanh_pol_table, {0xb408e2ed, false}},+ {tanh_pol_table, {0xb685fec8, false}},+ {tanh_pol_table, {0x00000000, false}},+ };++ auto push_arg_entry_of = [&](const key_t key, const table_entry_val_t val, const bool broadcast) {+ mapped_table_entry_t te{0, val, broadcast};+ entry_map.insert(std::make_pair(key, te));+ };++ auto push_entries_of = [&](const table_t& t) {+ for (auto it = t.begin(); it != t.end(); it++) {+ auto key = it->first;+ auto te = it->second;+ push_arg_entry_of(key, te.val, te.bcast);+ }+ };++ auto set_table_term_offset = [&]() {+ size_t off = 0;+ for (auto it = entry_map.begin(); it != entry_map.end(); it++) {+ auto& te = (*it).second;+ te.off = off;+ off += te.bcast ? 64u : sizeof(table_entry_val_t);+ }+ };++ struct need_t {+ explicit need_t(BTLA_ELTWISEOP& op) {+ if (op == BTLA_ELTWISEOP::EXP) exp_ = true;+ if (op == BTLA_ELTWISEOP::TANH) tanh_ = true;+ if (op == BTLA_ELTWISEOP::GELU) gelu_ = true;+ if (op == BTLA_ELTWISEOP::SWISH) swish_ = true;+ if (op == BTLA_ELTWISEOP::LOW_PRECISION_EXP) low_precision_exp_ = true;+ }+ bool bf16_ = false;+ bool exp_ = false;+ bool tanh_ = false;+ bool gelu_ = false;+ bool low_precision_exp_ = false;+ bool swish_ = false;++ bool bf16() const { return bf16_; }+ bool exp() const { return exp_; }+ bool tanh() const { return tanh_; }+ bool gelu() const { return gelu_; }+ bool low_precision_exp() { return low_precision_exp_; }+ bool swish() const { return swish_; }+ };++ need_t need(elt_op);+ push_entries_of(common_values);+ if (need.exp()) {+ push_entries_of(exp_consts);+ push_entries_of(exp_polynomial);+ }+ if (need.low_precision_exp() || need.swish()) {+ push_entries_of(exp_polynomial);+ push_entries_of(exp_consts);+ push_entries_of(low_precision_exp_consts);+ }+ if (need.tanh() || need.gelu()) {+ push_entries_of(tanh_consts);+ push_entries_of(tanh_polynomial_table);+ }+ if (need.gelu()) push_entries_of(gelu_tanh_const);++ set_table_term_offset();+ }+ void exp_compute_vector_fwd(const Xbyak::Ymm& ymm_src) {+ /* exp code */+ h->vcmpps(ymm_mask, ymm_src, table_val(exp_ln_flt_min_f), _cmp_lt_os);+ h->vminps(ymm_src, ymm_src, table_val(exp_ln_flt_max_f));+ h->vmaxps(ymm_src, ymm_src, table_val(exp_ln_flt_min_f));+ h->vmovups(ymm_aux1, ymm_src);+ h->vmulps(ymm_src, ymm_src, table_val(exp_log2ef));+ h->vaddps(ymm_src, ymm_src, table_val(half));+ h->vroundps(ymm_aux2, ymm_src, _op_floor);++ // keep ymm_src = fx for further computations+ h->vmovups(ymm_src, ymm_aux2);++ // x = x - fx * ln2+ h->vfnmadd231ps(ymm_aux1, ymm_aux2, table_val(ln2f));++ // We do not count 2^n here, because n can reach 128 and 2^128 is not+ // representable by fp32, so to get around this problem, instead of+ // computing 2^n * exp(r) will be counted 2*2^(n-1)*exp(r), because 2^127+ // and 2 are numbers representable in fp32.++ // compute 2^(n-1)+ h->vsubps(ymm_src, ymm_src, table_val(one));+ h->vcvtps2dq(ymm_aux2, ymm_src);+ h->vpaddd(ymm_aux2, ymm_aux2, table_val(exponent_bias));+ h->vpslld(ymm_aux2, ymm_aux2, n_mantissa_bits);++ // use ymm_src as tmp ymm_zero when applying mask+ h->vxorps(ymm_src, ymm_src, ymm_src);++ // set zeroes at those points which were < log(FLT_MIN)+ h->vblendvps(ymm_aux2, ymm_aux2, ymm_src, ymm_mask);++ // compute polynomial+ h->vmovups(ymm_src, table_val(exp_pol, 4));+ h->vfmadd213ps(ymm_src, ymm_aux1, table_val(exp_pol, 3));+ h->vfmadd213ps(ymm_src, ymm_aux1, table_val(exp_pol, 2));+ h->vfmadd213ps(ymm_src, ymm_aux1, table_val(exp_pol, 1));+ h->vfmadd213ps(ymm_src, ymm_aux1, table_val(exp_pol, 0));+ h->vfmadd213ps(ymm_src, ymm_aux1, table_val(one));++ // y = y * 2^n++ h->vmulps(ymm_src, ymm_src, ymm_aux2);+ h->vmulps(ymm_src, ymm_src, table_val(two));+ }+ void exp_compute_vector_fwd(const Xbyak::Zmm& zmm_src) {+ /* exp code */+ h->vcmpps(k_mask, zmm_src, table_val(exp_ln_flt_min_f), _cmp_lt_os);+ h->vminps(zmm_src, zmm_src, table_val(exp_ln_flt_max_f));+ h->vmaxps(zmm_src, zmm_src, table_val(exp_ln_flt_min_f));+ h->vmovups(zmm_aux1, zmm_src);+ h->vmulps(zmm_src, zmm_src, table_val(exp_log2ef));+ h->vaddps(zmm_src, zmm_src, table_val(half));+ h->vrndscaleps(zmm_aux2, zmm_src, _op_floor & 0x3);++ // keep zmm_src = fx for further computations+ h->vmovups(zmm_src, zmm_aux2);++ // x = x - fx * ln2+ h->vfnmadd231ps(zmm_aux1, zmm_aux2, table_val(ln2f));++ // We do not count 2^n here, because n can reach 128 and 2^128 is not+ // representable by fp32, so to get around this problem, instead of computing+ // 2^n * exp(r) will be counted 2*2^(n-1)*exp(r), because 2^127+ // and 2 are numbers representable in fp32.++ // compute 2^(n-1)+ h->vsubps(zmm_src, zmm_src, table_val(one));+ h->vcvtps2dq(zmm_aux2, zmm_src);+ h->vpaddd(zmm_aux2, zmm_aux2, table_val(exponent_bias));+ h->vpslld(zmm_aux2, zmm_aux2, n_mantissa_bits);++ // use zmm_src as tmp zmm_zero when applying mask+ h->vxorps(zmm_src, zmm_src, zmm_src);++ // set zeroes at those points which were < log(FLT_MIN)+ h->vblendmps(zmm_aux2 | k_mask, zmm_aux2, zmm_src);++ // compute polynomial+ h->vmovups(zmm_src, table_val(exp_pol, 4));+ h->vfmadd213ps(zmm_src, zmm_aux1, table_val(exp_pol, 3));+ h->vfmadd213ps(zmm_src, zmm_aux1, table_val(exp_pol, 2));+ h->vfmadd213ps(zmm_src, zmm_aux1, table_val(exp_pol, 1));+ h->vfmadd213ps(zmm_src, zmm_aux1, table_val(exp_pol, 0));+ h->vfmadd213ps(zmm_src, zmm_aux1, table_val(one));++ // y = y * 2^n++ h->vmulps(zmm_src, zmm_src, zmm_aux2);+ h->vmulps(zmm_src, zmm_src, table_val(two));+ }+ void low_precision_exp_compute_vector_fwd(const Xbyak::Ymm& ymm_src) {+ // support abs(x)<23+ auto code = [&](Xbyak::CodeGenerator* h, const Ymm& dst, const Ymm& src, const Xbyak::Operand& log2e,+ const Xbyak::Operand& ln2, const Xbyak::Operand& coeff0, const Xbyak::Operand& coeff1,+ const Xbyak::Operand& coeff2, const std::array<Ymm, 4>& tmp) {+ h->vmulps(tmp[0], src, log2e); // x / ln2+ h->vroundps(tmp[0], tmp[0], 0x0A); // round up+ const auto& z = tmp[0];+ h->vmulps(tmp[1], tmp[0], ln2);+ h->vsubps(tmp[1], src, tmp[1]); // x mod ln2 (can we use fmsub?)+ h->vmovaps(dst, coeff1);+ h->vfmadd231ps(dst, tmp[1], coeff0); // dst = f * c0 + c1+ h->vfmadd213ps(dst, tmp[1], coeff2); // dst = (f * c0 + c1) * f + c2++ const auto& z_sign = tmp[2];+ const auto& z_abs = tmp[3];+ h->vcmpps(z_sign, z, table_val(zero), _cmp_lt_os);+ h->vcvtps2dq(z, z);+ h->vpabsd(z_abs, z);+ h->vmovdqu(tmp[1], table_val(one_epi32));+ h->vpsllvd(z_abs, tmp[1], z_abs); // 2^z+ h->vcvtdq2ps(z_abs, z_abs);+ h->vrcpps(z, z_abs);+ h->vblendvps(z, z_abs, z, z_sign);+ h->vmulps(dst, dst, z); // dst = exp(f) * 2^z+ };+ code(h, ymm_src, ymm_src, table_val(exp_log2ef), table_val(ln2f), //+ table_val(low_precision_exp_const_v0), table_val(low_precision_exp_const_v1),+ table_val(low_precision_exp_const_v2), {ymm_aux1, ymm_aux2, ymm_aux3, ymm_aux4});+ }+ void low_precision_exp_compute_vector_fwd(const Xbyak::Zmm& zmm_src) {+ auto code = [&](Xbyak::CodeGenerator* h, const Zmm& dst, const Zmm& src, const Xbyak::Operand& log2e,+ const Xbyak::Operand& ln2, const Xbyak::Operand& coeff0, const Xbyak::Operand& coeff1,+ const Xbyak::Operand& coeff2, const std::array<Zmm, 2>& tmp) {+ h->vmovups(tmp[0], log2e);+ h->vmulps(tmp[0] | h->T_ru_sae, src, tmp[0]); // round up(x / ln2)+ const auto& z = tmp[0];+ h->vmulps(tmp[1], tmp[0], ln2);+ h->vsubps(tmp[1], src, tmp[1]); // x mod ln2 (can we use fmsub?)+ h->vmovaps(dst, coeff1);+ h->vfmadd231ps(dst, tmp[1], coeff0); // dst = f * c0 + c1+ h->vfmadd213ps(dst, tmp[1], coeff2); // dst = (f * c0 + c1) * f + c2+ h->vscalefps(dst, dst, z); // dst = exp(f) * 2^z+ };+ code(h, zmm_src, zmm_src, table_val(exp_log2ef), table_val(ln2f), //+ table_val(low_precision_exp_const_v0), table_val(low_precision_exp_const_v1),+ table_val(low_precision_exp_const_v2), {zmm_aux1, zmm_aux2});+ }+ void swish_compute_vector_fwd(const Xbyak::Ymm& ymm_src, int const_p_offset) {+ h->vbroadcastss(ymm_aux0, h->ptr[reg_rt_const_p + const_p_offset]);+ h->vmulps(ymm_aux0, ymm_aux0, ymm_src);+ exp_compute_vector_fwd(ymm_aux0);+ h->vaddps(ymm_aux0, ymm_aux0, table_val(one));+ h->vrcpps(ymm_aux0, ymm_aux0);+ h->vmulps(ymm_src, ymm_src, ymm_aux0);+ }+ void swish_compute_vector_fwd(const Xbyak::Zmm& zmm_src, int const_p_offset) {+ h->vmovups(zmm_aux0, zmm_src);+ h->vmulps(zmm_aux0, zmm_aux0, h->zword_b[reg_rt_const_p + const_p_offset]);+ if (bestla::utils::isFastExp()) {+ low_precision_exp_compute_vector_fwd(zmm_aux0);+ } else {+ exp_compute_vector_fwd(zmm_aux0);+ }+ h->vaddps(zmm_aux0, zmm_aux0, table_val(one));+ h->vrcp14ps(zmm_aux0, zmm_aux0);+ h->vmulps(zmm_src, zmm_src, zmm_aux0);+ }+ void tanh_compute_vector_fwd(const Xbyak::Ymm& ymm_src) {+ // register mapping+ Ymm ymm_dst = ymm_aux1, ymm_src_shift = ymm_aux1, ymm_coeff = ymm_aux1, ymm_pol = ymm_aux2, ymm_indices = ymm_aux3,+ ymm_src_original = ymm_aux4, ymm_sign = ymm_aux4;++ const int tanh_n_polynomials = 32;++ // We split the positive domain in 33 intervals:+ // a) [0; linear_ubound]: in this interval tanh(x) = x+ // b) [linear_ubound; 0x1.8p-12]: This interval spans part of a+ // half binade+ // c) [0x1.8p-12; 0x1.0p-11], ..., [0x1.8p2; 0x1.0p3]:+ // one interval for each half binade, there are 29 of those+ // d) [0x1.0p3; saturation_ubound]:+ // This interval spans part of a half binade+ // e) [0x1.205966p3; saturation_ubound]: in this interval, tanh(x) = 1+ // For b-d, we need 31 polynomials and will do a table lookup for those.+ // To simplify the logic, we will also put a) in the table.+ auto coeffs_address = [&](int coeff_off, int off = 0) {+ return table_val(tanh_pol_table, coeff_off * tanh_n_polynomials + off);+ };+ auto gather_coefficient = [&](Ymm vmm_coeff, int coeff_idx, Ymm vmm_pol_idx) {+ Ymm ymm_coeff(vmm_coeff.getIdx());+ Ymm ymm_pol_idx(vmm_pol_idx.getIdx());+ Xbyak::Address idx_addr =+ h->ptr[p_table + table_off(tanh_pol_table, coeff_idx * tanh_n_polynomials) + ymm_pol_idx * sizeof(float)];+ h->vpcmpeqd(ymm_mask, ymm_mask, ymm_mask);+ h->vgatherdps(vmm_coeff, idx_addr, ymm_mask);+ };++ // because tanh(x) = -tanh(-x), we extract sign to make x positive+ // and reapply sign at the end+ h->vmovups(ymm_src_original, ymm_src);+ h->vandps(ymm_src, ymm_src, table_val(positive_mask));++ // We compute the indices for the table lookup+ h->vmovups(ymm_indices, ymm_src);+ h->vpsubd(ymm_indices, ymm_indices, table_val(tanh_idx_bias));+ h->vandps(ymm_indices, ymm_indices, table_val(tanh_idx_mask));+ h->vpsrld(ymm_indices, ymm_indices, 22);++ // we do the argument reduction+ h->vmovups(ymm_src_shift, ymm_src);+ h->vandps(ymm_src_shift, ymm_src_shift, table_val(tanh_idx_mask));+ h->vsubps(ymm_src, ymm_src, ymm_src_shift);++ // we gather and evaluate the polynonials+ gather_coefficient(ymm_pol, 6, ymm_indices);+ for (int deg = 5; deg >= 0; --deg) {+ gather_coefficient(ymm_coeff, deg, ymm_indices);+ h->vfmadd213ps(ymm_pol, ymm_src, ymm_coeff);+ }++ // we restore src with cleared sign, and keep sign+ h->vmovups(ymm_src, ymm_src_original);+ h->vandps(ymm_sign, ymm_sign, table_val(sign_mask));+ h->vandps(ymm_src, ymm_src, table_val(positive_mask));++ // Now we blend the results+ // [saturation_ubound; +inf[ : we return +/- 1+ h->vmovups(ymm_dst, table_val(one));+ // [linear_ubound; saturation_lbound] : we return +/- P(x)+ h->vmovups(ymm_mask, table_val(tanh_saturation_lbound));+ h->vcmpps(ymm_mask, ymm_mask, ymm_src, _cmp_nle_us);+ h->vblendvps(ymm_dst, ymm_dst, ymm_pol, ymm_mask);+ // [0; linear_ubound] : we return x+ h->vmovups(ymm_mask, table_val(tanh_linear_ubound));+ h->vcmpps(ymm_mask, ymm_mask, ymm_src, _cmp_nle_us);+ h->vblendvps(ymm_dst, ymm_dst, ymm_src, ymm_mask);++ // We reapply the sign and return+ h->vxorps(ymm_dst, ymm_dst, ymm_sign);+ h->vmovups(ymm_src, ymm_dst);+ }+ void tanh_compute_vector_fwd(const Xbyak::Zmm& zmm_src) {+ // register mapping+ Zmm zmm_dst = zmm_aux1, zmm_src_shift = zmm_aux1, zmm_coeff = zmm_aux1, zmm_pol = zmm_aux2, zmm_indices = zmm_aux3,+ zmm_src_original = zmm_aux4, zmm_sign = zmm_aux4;++ const int tanh_n_polynomials = 32;++ // We split the positive domain in 33 intervals:+ // a) [0; linear_ubound]: in this interval tanh(x) = x+ // b) [linear_ubound; 0x1.8p-12]: This interval spans part of a+ // half binade+ // c) [0x1.8p-12; 0x1.0p-11], ..., [0x1.8p2; 0x1.0p3]:+ // one interval for each half binade, there are 29 of those+ // d) [0x1.0p3; saturation_ubound]:+ // This interval spans part of a half binade+ // e) [0x1.205966p3; saturation_ubound]: in this interval, tanh(x) = 1+ // For b-d, we need 31 polynomials and will do a table lookup for those.+ // To simplify the logic, we will also put a) in the table.+ auto coeffs_address = [&](int coeff_off, int off = 0) {+ return table_val(tanh_pol_table, (size_t)coeff_off * tanh_n_polynomials + off);+ };+ auto gather_coefficient = [&](Zmm vmm_coeff, int coeff_idx, Zmm vmm_pol_idx) {+ Zmm zmm_coeff(vmm_coeff.getIdx());+ Zmm zmm_pol_idx(vmm_pol_idx.getIdx());+ h->vmovups(zmm_coeff, coeffs_address(coeff_idx, 0));+ h->vpermt2ps(zmm_coeff, zmm_pol_idx, coeffs_address(coeff_idx, 16));+ };++ // because tanh(x) = -tanh(-x), we extract sign to make x positive+ // and reapply sign at the end+ h->vmovups(zmm_src_original, zmm_src);+ h->vpandd(zmm_src, zmm_src, table_val(positive_mask));++ // We compute the indices for the table lookup+ h->vmovups(zmm_indices, zmm_src);+ h->vpsubd(zmm_indices, zmm_indices, table_val(tanh_idx_bias));+ h->vpandd(zmm_indices, zmm_indices, table_val(tanh_idx_mask));+ h->vpsrld(zmm_indices, zmm_indices, 22);++ // we do the argument reduction+ h->vmovups(zmm_src_shift, zmm_src);+ h->vpandd(zmm_src_shift, zmm_src_shift, table_val(tanh_idx_mask));+ h->vsubps(zmm_src, zmm_src, zmm_src_shift);++ // we gather and evaluate the polynonials+ gather_coefficient(zmm_pol, 6, zmm_indices);+ for (int deg = 5; deg >= 0; --deg) {+ gather_coefficient(zmm_coeff, deg, zmm_indices);+ h->vfmadd213ps(zmm_pol, zmm_src, zmm_coeff);+ }++ // we restore src with cleared sign, and keep sign+ h->vmovups(zmm_src, zmm_src_original);+ h->vpandd(zmm_sign, zmm_sign, table_val(sign_mask));+ h->vpandd(zmm_src, zmm_src, table_val(positive_mask));++ // Now we blend the results+ // [saturation_ubound; +inf[ : we return +/- 1+ h->vmovups(zmm_dst, table_val(one));+ // [linear_ubound; saturation_lbound] : we return +/- P(x)+ h->vmovups(zmm_mask, table_val(tanh_saturation_lbound));+ h->vcmpps(k_mask, zmm_mask, zmm_src, _cmp_nle_us);+ h->vblendmps(zmm_dst | k_mask, zmm_dst, zmm_pol);+ // [0; linear_ubound] : we return x+ h->vmovups(zmm_mask, table_val(tanh_linear_ubound));+ h->vcmpps(k_mask, zmm_mask, zmm_src, _cmp_nle_us);+ h->vblendmps(zmm_dst | k_mask, zmm_dst, zmm_src);++ // We reapply the sign and return+ h->vpxord(zmm_dst, zmm_dst, zmm_sign);+ h->vmovups(zmm_src, zmm_dst);+ }+ void gelu_compute_vector_fwd(const Xbyak::Ymm& ymm_src) {+ h->vmovups(ymm_aux0, ymm_src);+ // compute G(x) = sqrt_root_two_over_pi * x * (1 + fitting_const * x * x)+ h->vmulps(ymm_src, ymm_src, ymm_src);+ h->vmovups(ymm_aux1, table_val(gelu_tanh_fitting_const));+ h->vfmadd213ps(ymm_src, ymm_aux1, table_val(one));+ h->vmulps(ymm_src, ymm_src, ymm_aux0);+ h->vmulps(ymm_src, ymm_src, table_val(gelu_tanh_sqrt_two_over_pi));++ // compute tanh(G(x))+ tanh_compute_vector_fwd(ymm_src);++ // compute 0.5 * x * (1 + tanh(G(x)))+ h->vaddps(ymm_src, ymm_src, table_val(one));+ h->vmulps(ymm_src, ymm_src, table_val(half));+ h->vmulps(ymm_src, ymm_src, ymm_aux0);+ }+ void gelu_compute_vector_fwd(const Xbyak::Zmm& zmm_src) {+ h->vmovups(zmm_aux0, zmm_src);+ // compute G(x) = sqrt_root_two_over_pi * x * (1 + fitting_const * x * x)+ h->vmulps(zmm_src, zmm_src, zmm_src);+ h->vmovups(zmm_aux1, table_val(gelu_tanh_fitting_const));+ h->vfmadd213ps(zmm_src, zmm_aux1, table_val(one));+ h->vmulps(zmm_src, zmm_src, zmm_aux0);+ h->vmulps(zmm_src, zmm_src, table_val(gelu_tanh_sqrt_two_over_pi));++ // compute tanh(G(x))+ tanh_compute_vector_fwd(zmm_src);++ // compute 0.5 * x * (1 + tanh(G(x)))+ h->vaddps(zmm_src, zmm_src, table_val(one));+ h->vmulps(zmm_src, zmm_src, table_val(half));+ h->vmulps(zmm_src, zmm_src, zmm_aux0);+ }+ void relu_compute_vector_fwd(const Xbyak::Zmm& zmm_src, int const_p_offset) {+ h->vmovups(zmm_aux1, zmm_src);+ h->vcmpps(k_mask, zmm_src, table_val(zero), _cmp_nle_us);+ h->vmulps(zmm_src, zmm_src, h->zword_b[reg_rt_const_p + const_p_offset]);+ h->vblendmps(zmm_src | k_mask, zmm_src, zmm_aux1);+ }+ void linear_compute_vector_fwd(const Xbyak::Zmm& zmm_src, int const_p_offset) {+ h->vbroadcastss(zmm_aux0, h->dword[reg_rt_const_p + const_p_offset]);+ h->vfmadd213ps(zmm_src, zmm_aux0, h->zword_b[reg_rt_const_p + const_p_offset + 1 * sizeof(float)]);+ }+ void load_table_addr() { h->mov(p_table, l_table); }+ void assign_zmm(const std::set<int>& used_zmm_idx, Zmm* zmm) {+ constexpr int max_zmm_idx = 32;+ for (int idx = 0; idx < max_zmm_idx; idx++) {+ if (used_zmm_idx.count(idx) == 0 && assign_vmm_idx.count(idx) == 0) {+ *zmm = Zmm(idx);+ assign_vmm_idx.insert(idx);+ break;+ }+ }+ }+ void assign_ymm(const std::set<int>& used_ymm_idx, Ymm* ymm) {+ constexpr int max_ymm_idx = 16;+ for (int idx = 0; idx < max_ymm_idx; idx++) {+ if (used_ymm_idx.count(idx) == 0 && assign_vmm_idx.count(idx) == 0) {+ *ymm = Ymm(idx);+ assign_vmm_idx.insert(idx);+ break;+ }+ }+ }++ private:+ BTLA_ELTWISEOP elt_op;+ Xbyak::CodeGenerator* h = nullptr;++ /*labels*/+ Xbyak::Label l_table;++ /*register for fwd*/+ Xbyak::Reg64 p_table;+ Xbyak::Reg64 reg_rt_const_p;+ std::set<int> assign_vmm_idx; // use for zmm (in avx512) or ymm (in avx2)+ Zmm zmm_mask, zmm_aux0, zmm_aux1, zmm_aux2, zmm_aux3, zmm_aux4;+ Ymm ymm_mask, ymm_aux0, ymm_aux1, ymm_aux2, ymm_aux3, ymm_aux4;+ Xbyak::Opmask k_mask;+ static constexpr int n_mantissa_bits = 23;++ enum {+ _cmp_eq_oq = 0u,+ _cmp_lt_os = 1u,+ _cmp_le_os = 2u,+ _cmp_neq_uq = 4u,+ _cmp_nlt_us = 5u,+ _cmp_nle_us = 6u,++ _op_floor = 1u,+ _op_mxcsr = 4u,+ };++ enum key_t {+ zero = 0, // 0.f+ half, // 0.5f+ one, // 1.f or mask for exponent bits+ two, // 2.f+ three, // 3.f+ six, // 6.f+ minus_one, // -1.f or changes sign to opposite+ minus_two, // -2.f+ minus_three, // -3.f+ ln2f, // 0.69314718f+ one_epi32, // 1 in int32+ positive_mask, // changes sign to positive+ sign_mask, // gets sign value+ exponent_bias, // (127 = 2^7 - 1), gets exponent bits+ exp_log2ef, // 1.44269502f - formula-based for approx+ exp_ln_flt_max_f, // logf(FLT_MAX) - max normal value+ exp_ln_flt_min_f, // logf(FLT_MIN) - min normal value+ exp_pol, // see correspondent table for float values+ gelu_tanh_fitting_const, // 0.044715f+ gelu_tanh_fitting_const_times_three, // 0.134145f+ gelu_tanh_sqrt_two_over_pi, // sqrtf(2.f/pi) = 0.797884f+ gelu_tanh_flt_max_x,+ gelu_tanh_flt_min_x,+ tanh_idx_bias,+ tanh_idx_mask,+ tanh_linear_ubound,+ tanh_saturation_lbound,+ tanh_pol_table,+ low_precision_exp_const_v0,+ low_precision_exp_const_v1,+ low_precision_exp_const_v2,+ undef_key,+ };++ size_t table_off(key_t key, size_t key_off_val_shift = 0) {+ const auto it = entry_map.find(key);+ assert(it != entry_map.end()); // "key is not in entry_map"+ const auto& te = (*it).second;+ const auto scale = te.bcast ? 64u : sizeof(table_entry_val_t);+ return te.off + key_off_val_shift * scale;+ }+ Xbyak::Address table_val(key_t key, size_t key_off_val_shift = 0) {+ auto off = table_off(key, key_off_val_shift);+ return h->ptr[p_table + off];+ }+ using table_entry_val_t = uint32_t;+ using table_entry_offset_t = size_t; // offsets are in bytes wrt p_table+ using table_entry_bcast_t = bool;++ struct table_entry_t {+ table_entry_val_t val;+ table_entry_bcast_t bcast;+ };+ struct mapped_table_entry_t {+ table_entry_offset_t off;+ table_entry_val_t val;+ table_entry_bcast_t bcast;+ };+ using table_t = std::multimap<key_t, table_entry_t>;+ using mapped_table_t = std::multimap<key_t, mapped_table_entry_t>;+ mapped_table_t entry_map = {};+};+} // namespace jit_injector+} // namespace kernel+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_ref.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_ref.hnew file mode 100644index 000000000..561c8f3b7--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_ref.h@@ -0,0 +1,3439 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include "bestla.h"+#include "bestla_utils.h"+#include <algorithm>+#include <cassert>+#include <limits>+#include <type_traits>+#include <vector>++namespace bestla {+namespace kernel {+namespace ref {++template <typename T>+static inline BTLA_CODE shuffle_activation(T* src, T* dst, int shuffle_m, int shuffle_k, int m_offset, int k_offset,+ int* indices, int src_stride, int dst_stride) {+ T* cur_src = src + m_offset * src_stride;+ for (int i = 0; i < shuffle_m; i++) {+ for (int j = 0; j < shuffle_k; j++) {+ dst[i * dst_stride + j] = cur_src[i * src_stride + indices[k_offset + j]];+ }+ }+ return BTLA_CODE::Success;+}++template <typename T_SRC, typename T_DST = T_SRC>+static inline BTLA_CODE padding_interleave(const T_SRC* src_ptr, T_DST* dst_ptr, int row, int col, int rowpad,+ int colpad, int src_step, int dst_step, int NTile, int RowPack) {+ const T_DST dst_0(0);+ static_assert(sizeof(T_SRC) == sizeof(T_DST), "SRC & DST size should be the same");+ for (int i = 0; i < rowpad; i += RowPack) {+ for (int j = 0; j < colpad; j += NTile) {+ for (int jj = 0; jj < NTile; jj++) {+ for (int ii = 0; ii < RowPack; ii++) {+ dst_ptr[i * NTile + j * dst_step + jj * RowPack + ii] =+ (i + ii) < row && (j + jj) < col //+ ? static_cast<T_DST>(src_ptr[(i + ii) * src_step + (j + jj)])+ : dst_0;+ }+ }+ }+ }+ return BTLA_CODE::Success;+}++// revert padding and interleave+// row*col <= colpad/NTile*rowpad*NTile+template <typename T_SRC, typename T_DST = T_SRC>+static inline BTLA_CODE revert_padding_interleave(const T_SRC* src_ptr, T_DST* dst_ptr, int row, int col, int rowpad,+ int colpad, int src_step, int dst_step, int NTile, int RowPack) {+ static_assert(sizeof(T_SRC) == sizeof(T_DST), "SRC & DST size should be the same");+ for (int i = 0; i < rowpad; i += RowPack) {+ for (int j = 0; j < colpad; j += NTile) {+ for (int jj = 0; jj < NTile; jj++) {+ if ((j + jj) < col) {+ for (int ii = 0; ii < RowPack; ii++) {+ if ((i + ii) < row) {+ dst_ptr[(i + ii) * dst_step + (j + jj)] =+ static_cast<T_DST>(src_ptr[i * NTile + j * src_step + jj * RowPack + ii]);+ }+ }+ }+ }+ }+ }+ return BTLA_CODE::Success;+}++// M x N ===> M/MTile x N/colPack x MTile x colPack (leading dim stride = MTile+// * dst_stride)+template <typename T_SRC, typename T_DST = T_SRC>+static inline BTLA_CODE padding_trans_interleave(const T_SRC* src, T_DST* dst, int row, int col, int rowpad, int colpad,+ int src_step, int dst_step, int MTile, int ColPack) {+ // Note: rows/cols and i/j are in terms of src+ static_assert(sizeof(T_SRC) == sizeof(T_DST), "SRC & DST size should be the same");+ const T_DST dst_0(0);+ for (int i = 0; i < rowpad; i += MTile) {+ for (int j = 0; j < colpad; j += ColPack) {+ for (int ii = 0; ii < MTile; ii++) {+ for (int jj = 0; jj < ColPack; jj++) {+ dst[i * dst_step + j * MTile + ii * ColPack + jj] =+ (i + ii) < row && (j + jj) < col //+ ? static_cast<T_DST>(src[(i + ii) * src_step + (j + jj)])+ : dst_0;+ }+ }+ }+ }+ return BTLA_CODE::Success;+}++template <typename SRC_DT, typename DST_DT>+static inline BTLA_CODE dt_cvt_2D_write_back(const void* raw_srcptr, void* raw_dstptr, int row, int col, int srcstride,+ int dststride, bool zeropadding) {+ for (int i = 0; i < row; i++) {+ int j = 0;+ for (; j < col; j++) {+ const auto src = reinterpret_cast<const SRC_DT*>(reinterpret_cast<const char*>(raw_srcptr) + i * srcstride);+ const auto dst = reinterpret_cast<DST_DT*>(reinterpret_cast<char*>(raw_dstptr) + i * dststride);+ dst[j] = static_cast<DST_DT>(src[j]);+ }+ if (zeropadding) {+ for (int bj = j * sizeof(DST_DT); bj < dststride; bj++) {+ (reinterpret_cast<char*>(raw_dstptr) + i * dststride)[bj] = 0;+ }+ }+ }+ return BTLA_CODE::Success;+}++template <typename _DST_T>+static inline BTLA_CODE dequan_s8_fp(int8_t* srcptr, _DST_T* dstptr, int row, int col, int ld_src, int ld_dst,+ float* scales) {+ for (int i = 0; i < row; i++) {+ for (int j = 0; j < col; j++) {+ dstptr[i * ld_dst + j] = static_cast<float>(srcptr[i * ld_src + j]) * scales[j];+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE dequan_s8_bf16(int8_t* srcptr, uint16_t* dstptr, int row, int col, int ld_src, int ld_dst,+ float* scales) {+ for (int i = 0; i < row; i++) {+ for (int j = 0; j < col; j++) {+ dstptr[i * ld_dst + j] =+ utils::cast<float, utils::bf16>(static_cast<float>(srcptr[i * ld_src + j]) * scales[j]).x;+ }+ }+ return BTLA_CODE::Success;+}++template <typename _T>+static inline BTLA_CODE transpose2d(const _T* srcptr, _T* dstptr, int row, int col, int ld_src, int ld_dst) {+ for (int i = 0; i < col; i++) {+ for (size_t j = 0; j < row; j++) {+ dstptr[j + i * ld_dst] = srcptr[j * ld_src + i];+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE compress_s8_s4(const int8_t* srcptr, utils::int4x2* dstptr, size_t size) {+ int8_t constexpr FullRange = 1 << (4 - 1);+ assert(size % 2 == 0);+ for (int ii = 0; ii < size; ii += 2) {+ utils::int4x2 tmp;+ tmp.x = srcptr[ii + 0] + FullRange;+ tmp.y = srcptr[ii + 1] + FullRange;+ dstptr[ii / 2] = tmp;+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE compress_f4(const int8_t* srcptr, utils::f4x2* dstptr, size_t size) {+ for (int ii = 0; ii < size; ii += 2) {+ assert(size % 2 == 0);+ utils::f4x2 tmp;+ tmp.x = srcptr[ii + 0];+ tmp.y = srcptr[ii + 1];+ dstptr[ii / 2] = tmp;+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE compress_7bit(const int8_t* srcptr, bestla::utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr,+ utils::bit1x8* bit1ptr, size_t size) {+ assert(size % 8 == 0);+ int8_t constexpr FullRange = 1 << (7 - 1);+ for (int j = 0; j < size; j += 8) {+ auto tmp = srcptr[j + 0] + FullRange;+ bit4ptr[j / 2 + 0].x = tmp & 0xf;+ tmp = tmp >> 4;+ bit2ptr[j / 4].a = tmp & 0x3;+ tmp = tmp >> 2;+ bit1ptr[j / 8].a = tmp;+ tmp = srcptr[j + 1] + FullRange;+ bit4ptr[j / 2 + 0].y = tmp & 0xf;+ tmp = tmp >> 4;+ bit2ptr[j / 4].b = tmp & 0x3;+ tmp = tmp >> 2;+ bit1ptr[j / 8].b = tmp;+ tmp = srcptr[j + 2] + FullRange;+ bit4ptr[j / 2 + 1].x = tmp & 0xf;+ tmp = tmp >> 4;+ bit2ptr[j / 4].c = tmp & 0x3;+ tmp = tmp >> 2;+ bit1ptr[j / 8].c = tmp;+ tmp = srcptr[j + 3] + FullRange;+ bit4ptr[j / 2 + 1].y = tmp & 0xf;+ tmp = tmp >> 4;+ bit2ptr[j / 4].d = tmp & 0x3;+ tmp = tmp >> 2;+ bit1ptr[j / 8].d = tmp;++ tmp = srcptr[j + 4] + FullRange;+ bit4ptr[j / 2 + 2].x = tmp & 0xf;+ tmp = tmp >> 4;+ bit2ptr[j / 4 + 1].a = tmp & 0x3;+ tmp = tmp >> 2;+ bit1ptr[j / 8].e = tmp;+ tmp = srcptr[j + 5] + FullRange;+ bit4ptr[j / 2 + 2].y = tmp & 0xf;+ tmp = tmp >> 4;+ bit2ptr[j / 4 + 1].b = tmp & 0x3;+ tmp = tmp >> 2;+ bit1ptr[j / 8].f = tmp;+ tmp = srcptr[j + 6] + FullRange;+ bit4ptr[j / 2 + 3].x = tmp & 0xf;+ tmp = tmp >> 4;+ bit2ptr[j / 4 + 1].c = tmp & 0x3;+ tmp = tmp >> 2;+ bit1ptr[j / 8].g = tmp;+ tmp = srcptr[j + 7] + FullRange;+ bit4ptr[j / 2 + 3].y = tmp & 0xf;+ tmp = tmp >> 4;+ bit2ptr[j / 4 + 1].d = tmp & 0x3;+ tmp = tmp >> 2;+ bit1ptr[j / 8].h = tmp;+ }++ return BTLA_CODE::Success;+}++static inline BTLA_CODE compress_6bit(const int8_t* srcptr, bestla::utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr,+ size_t size) {+ assert(size % 4 == 0);+ int8_t constexpr FullRange = 1 << (6 - 1);+ for (int j = 0; j < size; j += 4) {+ auto tmp = srcptr[j + 0] + FullRange;+ bit4ptr[j / 2 + 0].x = tmp & 0xf;+ bit2ptr[j / 4].a = tmp >> 4;+ tmp = srcptr[j + 1] + FullRange;+ bit4ptr[j / 2 + 0].y = tmp & 0xf;+ bit2ptr[j / 4].b = tmp >> 4;+ tmp = srcptr[j + 2] + FullRange;+ bit4ptr[j / 2 + 1].x = tmp & 0xf;+ bit2ptr[j / 4].c = tmp >> 4;+ tmp = srcptr[j + 3] + FullRange;+ bit4ptr[j / 2 + 1].y = tmp & 0xf;+ bit2ptr[j / 4].d = tmp >> 4;+ }++ return BTLA_CODE::Success;+}++static inline BTLA_CODE compress_5bit(const int8_t* srcptr, bestla::utils::bit4x2* bit4ptr, utils::bit1x8* bit1ptr,+ size_t size) {+ assert(size % 8 == 0);+ int8_t constexpr FullRange = 1 << (5 - 1);+ for (int j = 0; j < size; j += 8) {+ auto tmp = srcptr[j + 0] + FullRange;+ bit4ptr[j / 2 + 0].x = tmp & 0xf;+ bit1ptr[j / 8].a = tmp >> 4;+ tmp = srcptr[j + 1] + FullRange;+ bit4ptr[j / 2 + 0].y = tmp & 0xf;+ bit1ptr[j / 8].b = tmp >> 4;+ tmp = srcptr[j + 2] + FullRange;+ bit4ptr[j / 2 + 1].x = tmp & 0xf;+ bit1ptr[j / 8].c = tmp >> 4;+ tmp = srcptr[j + 3] + FullRange;+ bit4ptr[j / 2 + 1].y = tmp & 0xf;+ bit1ptr[j / 8].d = tmp >> 4;++ tmp = srcptr[j + 4] + FullRange;+ bit4ptr[j / 2 + 2].x = tmp & 0xf;+ bit1ptr[j / 8].e = tmp >> 4;+ tmp = srcptr[j + 5] + FullRange;+ bit4ptr[j / 2 + 2].y = tmp & 0xf;+ bit1ptr[j / 8].f = tmp >> 4;+ tmp = srcptr[j + 6] + FullRange;+ bit4ptr[j / 2 + 3].x = tmp & 0xf;+ bit1ptr[j / 8].g = tmp >> 4;+ tmp = srcptr[j + 7] + FullRange;+ bit4ptr[j / 2 + 3].y = tmp & 0xf;+ bit1ptr[j / 8].h = tmp >> 4;+ }++ return BTLA_CODE::Success;+}++static inline BTLA_CODE compress_3bit(const int8_t* srcptr, bestla::utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr,+ size_t size) {+ int8_t constexpr FullRange = 1 << (3 - 1);+ assert(size % 8 == 0);++ for (int j = 0; j < size; j += 8) {+ auto tmp = srcptr[j + 0] + FullRange;+ bit2ptr[j / 4 + 0].a = tmp & 0x3;+ bit1ptr[j / 8].a = tmp >> 2;+ tmp = srcptr[j + 1] + FullRange;+ bit2ptr[j / 4 + 0].b = tmp & 0x3;+ bit1ptr[j / 8].b = tmp >> 2;+ tmp = srcptr[j + 2] + FullRange;+ bit2ptr[j / 4 + 0].c = tmp & 0x3;+ bit1ptr[j / 8].c = tmp >> 2;+ tmp = srcptr[j + 3] + FullRange;+ bit2ptr[j / 4 + 0].d = tmp & 0x3;+ bit1ptr[j / 8].d = tmp >> 2;++ tmp = srcptr[j + FullRange] + FullRange;+ bit2ptr[j / 4 + 1].a = tmp & 0x3;+ bit1ptr[j / 8].e = tmp >> 2;+ tmp = srcptr[j + 5] + FullRange;+ bit2ptr[j / 4 + 1].b = tmp & 0x3;+ bit1ptr[j / 8].f = tmp >> 2;+ tmp = srcptr[j + 6] + FullRange;+ bit2ptr[j / 4 + 1].c = tmp & 0x3;+ bit1ptr[j / 8].g = tmp >> 2;+ tmp = srcptr[j + 7] + FullRange;+ bit2ptr[j / 4 + 1].d = tmp & 0x3;+ bit1ptr[j / 8].h = tmp >> 2;+ }++ return BTLA_CODE::Success;+}++static inline BTLA_CODE compress_2bit(const int8_t* srcptr, bestla::utils::bit2x4* bit2ptr, size_t size) {+ int8_t constexpr FullRange = 1 << (2 - 1);+ assert(size % 4 == 0);+ for (size_t i = 0; i < size; i += 4) {+ bit2ptr[i / 4].a = *(srcptr + i) + FullRange;+ bit2ptr[i / 4].b = *(srcptr + i + 1) + FullRange;+ bit2ptr[i / 4].c = *(srcptr + i + 2) + FullRange;+ bit2ptr[i / 4].d = *(srcptr + i + 3) + FullRange;+ }++ return BTLA_CODE::Success;+}++static inline BTLA_CODE compress_1bit(const int8_t* srcptr, bestla::utils::bit1x8* bit1ptr, size_t size) {+ int8_t constexpr FullRange = 1 << (1 - 1);+ assert(size % 8 == 0);+ for (int j = 0; j < size; j += 8) {+ auto tmp = srcptr[j + 0] + FullRange;+ bit1ptr[j / 8].a = tmp;+ tmp = srcptr[j + 1] + FullRange;+ bit1ptr[j / 8].b = tmp;+ tmp = srcptr[j + 2] + FullRange;+ bit1ptr[j / 8].c = tmp;+ tmp = srcptr[j + 3] + FullRange;+ bit1ptr[j / 8].d = tmp;+ tmp = srcptr[j + FullRange] + FullRange;+ bit1ptr[j / 8].e = tmp;+ tmp = srcptr[j + 5] + FullRange;+ bit1ptr[j / 8].f = tmp;+ tmp = srcptr[j + 6] + FullRange;+ bit1ptr[j / 8].g = tmp;+ tmp = srcptr[j + 7] + FullRange;+ bit1ptr[j / 8].h = tmp;+ }+ return BTLA_CODE::Success;+}++template <BTLA_DTYPE Q4T>+static inline void convert_s4_s8_8(int8_t* dstptr, int8_t* srcptr) {+ auto src32 = *reinterpret_cast<uint32_t*>(srcptr);+ auto tmp = static_cast<int>(src32 & 0xf);+ if constexpr (Q4T == BTLA_DTYPE::S4_CLIP) {+ tmp -= 8;+ }+ dstptr[0] = static_cast<int8_t>(tmp);+ tmp = static_cast<int>(src32 & 0xf0) >> 4;+ if constexpr (Q4T == BTLA_DTYPE::S4_CLIP) {+ tmp -= 8;+ }+ dstptr[1] = static_cast<int8_t>(tmp);+ tmp = static_cast<int>((src32 & 0xf00) >> 8);+ if constexpr (Q4T == BTLA_DTYPE::S4_CLIP) {+ tmp -= 8;+ }+ dstptr[2] = static_cast<int8_t>(tmp);+ tmp = static_cast<int>((src32 & 0xf000) >> 12);+ if constexpr (Q4T == BTLA_DTYPE::S4_CLIP) {+ tmp -= 8;+ }+ dstptr[3] = static_cast<int8_t>(tmp);+ tmp = static_cast<int>((src32 & 0xf0000) >> 16);+ if constexpr (Q4T == BTLA_DTYPE::S4_CLIP) {+ tmp -= 8;+ }+ dstptr[4] = static_cast<int8_t>(tmp);+ tmp = static_cast<int>((src32 & 0xf00000) >> 20);+ if constexpr (Q4T == BTLA_DTYPE::S4_CLIP) {+ tmp -= 8;+ }+ dstptr[5] = static_cast<int8_t>(tmp);+ tmp = static_cast<int>((src32 & 0xf000000) >> 24);+ if constexpr (Q4T == BTLA_DTYPE::S4_CLIP) {+ tmp -= 8;+ }+ dstptr[6] = static_cast<int8_t>(tmp);+ tmp = static_cast<int>((src32 & 0xf0000000) >> 28);+ if constexpr (Q4T == BTLA_DTYPE::S4_CLIP) {+ tmp -= 8;+ }+ dstptr[7] = static_cast<int8_t>(tmp);+}++static inline BTLA_CODE decompress_s6_s8(utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, int8_t* dstptr,+ size_t unpack_elt, int8_t* tmp, size_t tmpsize) {+ int constexpr FullRange = 1 << (6 - 1);+ for (size_t i = 0; i < unpack_elt; i += 4) {+ auto bit2 = bit2ptr[i / 4];+ auto tmp = bit4ptr[i / 2];+ dstptr[i + 0] = (tmp.x | (bit2.a << 4)) - FullRange;+ dstptr[i + 1] = (tmp.y | (bit2.b << 4)) - FullRange;+ tmp = bit4ptr[i / 2 + 1];+ dstptr[i + 2] = (tmp.x | (bit2.c << 4)) - FullRange;+ dstptr[i + 3] = (tmp.y | (bit2.d << 4)) - FullRange;+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s5_s8(utils::bit4x2* bit4ptr, utils::bit1x8* bit1ptr, int8_t* dstptr,+ size_t unpack_elt, int8_t* tmp, size_t tmpsize) {+ int constexpr FullRange = 1 << (5 - 1);+ for (size_t i = 0; i < unpack_elt; i += 8) {+ auto bit1 = bit1ptr[i / 8];+ auto tmp = bit4ptr[i / 2];+ dstptr[i + 0] = (tmp.x | (bit1.a << 4)) - FullRange;+ dstptr[i + 1] = (tmp.y | (bit1.b << 4)) - FullRange;+ tmp = bit4ptr[i / 2 + 1];+ dstptr[i + 2] = (tmp.x | (bit1.c << 4)) - FullRange;+ dstptr[i + 3] = (tmp.y | (bit1.d << 4)) - FullRange;+ tmp = bit4ptr[i / 2 + 2];+ dstptr[i + 4] = (tmp.x | (bit1.e << 4)) - FullRange;+ dstptr[i + 5] = (tmp.y | (bit1.f << 4)) - FullRange;+ tmp = bit4ptr[i / 2 + 3];+ dstptr[i + 6] = (tmp.x | (bit1.g << 4)) - FullRange;+ dstptr[i + 7] = (tmp.y | (bit1.h << 4)) - FullRange;+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s7_s8(utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr,+ int8_t* dstptr, size_t unpack_elt, int8_t* tmp, size_t tmpsize) {+ int constexpr FullRange = 1 << (7 - 1);+ for (size_t i = 0; i < unpack_elt; i += 8) {+ auto bit2 = bit2ptr[i / 4];+ auto bit1 = bit1ptr[i / 8];+ auto tmp = bit4ptr[i / 2];+ dstptr[i + 0] = (tmp.x | (bit2.a << 4) | (bit1.a << 6)) - FullRange;+ dstptr[i + 1] = (tmp.y | (bit2.b << 4) | (bit1.b << 6)) - FullRange;+ tmp = bit4ptr[i / 2 + 1];+ dstptr[i + 2] = (tmp.x | (bit2.c << 4) | (bit1.c << 6)) - FullRange;+ dstptr[i + 3] = (tmp.y | (bit2.d << 4) | (bit1.d << 6)) - FullRange;+ bit2 = bit2ptr[i / 4 + 1];+ tmp = bit4ptr[i / 2 + 2];+ dstptr[i + 4] = (tmp.x | (bit2.a << 4) | (bit1.e << 6)) - FullRange;+ dstptr[i + 5] = (tmp.y | (bit2.b << 4) | (bit1.f << 6)) - FullRange;+ tmp = bit4ptr[i / 2 + 3];+ dstptr[i + 6] = (tmp.x | (bit2.c << 4) | (bit1.g << 6)) - FullRange;+ dstptr[i + 7] = (tmp.y | (bit2.d << 4) | (bit1.h << 6)) - FullRange;+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s4_s8(utils::int4x2* srcptr, int8_t* dstptr, size_t unpackelt, int8_t* tmp,+ size_t tmpsize) {+ for (int j = 0; j < unpackelt; j += 2) {+ auto tmp = srcptr[j / 2];+ dstptr[j + 0] = tmp.x - 8;+ dstptr[j + 1] = tmp.y - 8;+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s3_s8(utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr, int8_t* dstptr,+ size_t unpack_elt, int8_t* tmp, size_t tmpsize) {+ for (size_t i = 0; i < unpack_elt; i += 8) {+ auto bit1 = bit1ptr[i / 8];+ auto tmp = bit2ptr[i / 4];+ dstptr[i + 0] = (tmp.a | (bit1.a << 2)) - 4;+ dstptr[i + 1] = (tmp.b | (bit1.b << 2)) - 4;+ dstptr[i + 2] = (tmp.c | (bit1.c << 2)) - 4;+ dstptr[i + 3] = (tmp.d | (bit1.d << 2)) - 4;+ tmp = bit2ptr[i / 4 + 1];+ dstptr[i + 4] = (tmp.a | (bit1.e << 2)) - 4;+ dstptr[i + 5] = (tmp.b | (bit1.f << 2)) - 4;+ dstptr[i + 6] = (tmp.c | (bit1.g << 2)) - 4;+ dstptr[i + 7] = (tmp.d | (bit1.h << 2)) - 4;+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s2_s8(utils::bit2x4* srcptr, int8_t* dstptr, size_t unpackelt, int8_t* tmp,+ size_t tmpsize) {+ for (int j = 0; j < unpackelt; j += 4) {+ auto tmp = srcptr[j / 4];+ dstptr[j + 0] = tmp.a - 2;+ dstptr[j + 1] = tmp.b - 2;+ dstptr[j + 2] = tmp.c - 2;+ dstptr[j + 3] = tmp.d - 2;+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE decompress_s1_s8(utils::bit1x8* bit1ptr, int8_t* dstptr, size_t unpack_elt, int8_t* tmp,+ size_t tmpsize) {+ int8_t constexpr FullRange = 1 << (1 - 1);+ for (size_t i = 0; i < unpack_elt; i += 8) {+ auto bit1 = bit1ptr[i / 8];+ dstptr[i + 0] = bit1.a - FullRange;+ dstptr[i + 1] = bit1.b - FullRange;+ dstptr[i + 2] = bit1.c - FullRange;+ dstptr[i + 3] = bit1.d - FullRange;+ dstptr[i + 4] = bit1.e - FullRange;+ dstptr[i + 5] = bit1.f - FullRange;+ dstptr[i + 6] = bit1.g - FullRange;+ dstptr[i + 7] = bit1.h - FullRange;+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s7_s8(utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr,+ int8_t* zpptr, int8_t* dstptr, int blocksize, int ldzp, int n_offset,+ int k_offset, int row, int col, int8_t* tmp, size_t tmpsize) {+ int constexpr FullRange = 1 << (7 - 1);+ static_assert(NTILE % 8 == 0);+ assert(((col * PackRow) % 8) == 0);+ if (zpptr) {+ if constexpr (PackRow == 4) {+ for (int i = 0; i < row; i += PackRow) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 2) {+ auto zp = zptr[j] + FullRange;+ auto bit1 = bit1ptr[(i * col + j * PackRow) / 8];+ auto bit2 = bit2ptr[(i * col + j * PackRow) / 4];+ auto tmp = bit4ptr[(i * col + j * PackRow) / 2];+ dstptr[i * col + j * PackRow + 0] = (tmp.x | (bit2.a << 4) | (bit1.a << 6)) - zp;+ dstptr[i * col + j * PackRow + 1] = (tmp.y | (bit2.b << 4) | (bit1.b << 6)) - zp;+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 1];+ dstptr[i * col + j * PackRow + 2] = (tmp.x | (bit2.c << 4) | (bit1.c << 6)) - zp;+ dstptr[i * col + j * PackRow + 3] = (tmp.y | (bit2.d << 4) | (bit1.d << 6)) - zp;+ zp = zptr[j + 1] + FullRange;+ bit2 = bit2ptr[(i * col + j * PackRow) / 4 + 1];+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 2];+ dstptr[i * col + j * PackRow + 4] = (tmp.x | (bit2.a << 4) | (bit1.e << 6)) - zp;+ dstptr[i * col + j * PackRow + 5] = (tmp.y | (bit2.b << 4) | (bit1.f << 6)) - zp;+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 3];+ dstptr[i * col + j * PackRow + 6] = (tmp.x | (bit2.c << 4) | (bit1.g << 6)) - zp;+ dstptr[i * col + j * PackRow + 7] = (tmp.y | (bit2.d << 4) | (bit1.h << 6)) - zp;+ }+ }+ } else if constexpr (PackRow == 1) {+ for (int i = 0; i < row; i += 1) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 8) {+ auto bit2 = bit2ptr[(i * col + j * PackRow) / 4];+ auto bit1 = bit1ptr[(i * col + j * PackRow) / 8];+ auto tmp = bit4ptr[(i * col + j * PackRow) / 2];+ dstptr[i * col + j * PackRow + 0] = (tmp.x | (bit2.a << 4) | (bit1.a << 6)) - FullRange - zptr[j + 0];+ dstptr[i * col + j * PackRow + 1] = (tmp.y | (bit2.b << 4) | (bit1.b << 6)) - FullRange - zptr[j + 1];+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 1];+ dstptr[i * col + j * PackRow + 2] = (tmp.x | (bit2.c << 4) | (bit1.c << 6)) - FullRange - zptr[j + 2];+ dstptr[i * col + j * PackRow + 3] = (tmp.y | (bit2.d << 4) | (bit1.d << 6)) - FullRange - zptr[j + 3];+ bit2 = bit2ptr[(i * col + j * PackRow) / 4 + 1];+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 2];+ dstptr[i * col + j * PackRow + 4] = (tmp.x | (bit2.a << 4) | (bit1.e << 6)) - FullRange - zptr[j + 4];+ dstptr[i * col + j * PackRow + 5] = (tmp.y | (bit2.b << 4) | (bit1.f << 6)) - FullRange - zptr[j + 5];+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 3];+ dstptr[i * col + j * PackRow + 6] = (tmp.x | (bit2.c << 4) | (bit1.g << 6)) - FullRange - zptr[j + 6];+ dstptr[i * col + j * PackRow + 7] = (tmp.y | (bit2.d << 4) | (bit1.h << 6)) - FullRange - zptr[j + 7];+ }+ }+ } else if constexpr (PackRow == 2) {+ for (int i = 0; i < row; i += PackRow) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 4) {+ auto bit2 = bit2ptr[(i * col + j * PackRow) / 4];+ auto bit1 = bit1ptr[(i * col + j * PackRow) / 8];+ auto tmp = bit4ptr[(i * col + j * PackRow) / 2];+ auto zp = zptr[j] + FullRange;+ dstptr[i * col + j * PackRow + 0] = (tmp.x | (bit2.a << 4) | (bit1.a << 6)) - zp;+ dstptr[i * col + j * PackRow + 1] = (tmp.y | (bit2.b << 4) | (bit1.b << 6)) - zp;+ zp = zptr[j + 1] + FullRange;+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 1];+ dstptr[i * col + j * PackRow + 2] = (tmp.x | (bit2.c << 4) | (bit1.c << 6)) - zp;+ dstptr[i * col + j * PackRow + 3] = (tmp.y | (bit2.d << 4) | (bit1.d << 6)) - zp;+ bit2 = bit2ptr[(i * col + j * PackRow) / 4 + 1];+ zp = zptr[j + 2] + FullRange;+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 2];+ dstptr[i * col + j * PackRow + 4] = (tmp.x | (bit2.a << 4) | (bit1.e << 6)) - zp;+ dstptr[i * col + j * PackRow + 5] = (tmp.y | (bit2.b << 4) | (bit1.f << 6)) - zp;+ zp = zptr[j + 3] + FullRange;+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 3];+ dstptr[i * col + j * PackRow + 6] = (tmp.x | (bit2.c << 4) | (bit1.g << 6)) - zp;+ dstptr[i * col + j * PackRow + 7] = (tmp.y | (bit2.d << 4) | (bit1.h << 6)) - zp;+ }+ }+ } else {+ static_assert(PackRow == 1 || PackRow == 2 || PackRow == 4);+ }+ } else {+ return decompress_s7_s8(bit4ptr, bit2ptr, bit1ptr, dstptr, size_t(row) * col, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s6_s8(utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset, int k_offset,+ int row, int col, int8_t* tmp, size_t tmpsize) {+ int constexpr FullRange = 1 << (6 - 1);+ static_assert(NTILE % 4 == 0);+ assert(((col * PackRow) % 4) == 0);+ if (zpptr) {+ if constexpr (PackRow == 4) {+ for (int i = 0; i < row; i += PackRow) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 1) {+ auto zp = zptr[j] + FullRange;+ auto bit2 = bit2ptr[(i * col + j * PackRow) / 4];+ auto tmp = bit4ptr[(i * col + j * PackRow) / 2];+ dstptr[i * col + j * PackRow + 0] = (tmp.x | (bit2.a << 4)) - zp;+ dstptr[i * col + j * PackRow + 1] = (tmp.y | (bit2.b << 4)) - zp;+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 1];+ dstptr[i * col + j * PackRow + 2] = (tmp.x | (bit2.c << 4)) - zp;+ dstptr[i * col + j * PackRow + 3] = (tmp.y | (bit2.d << 4)) - zp;+ }+ }+ } else if constexpr (PackRow == 1) {+ for (int i = 0; i < row; i += 1) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 4) {+ auto bit2 = bit2ptr[(i * col + j * PackRow) / 4];+ auto tmp = bit4ptr[(i * col + j * PackRow) / 2];+ dstptr[i * col + j * PackRow + 0] = (tmp.x | (bit2.a << 4)) - FullRange - zptr[j + 0];+ dstptr[i * col + j * PackRow + 1] = (tmp.y | (bit2.b << 4)) - FullRange - zptr[j + 1];+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 1];+ dstptr[i * col + j * PackRow + 2] = (tmp.x | (bit2.c << 4)) - FullRange - zptr[j + 2];+ dstptr[i * col + j * PackRow + 3] = (tmp.y | (bit2.d << 4)) - FullRange - zptr[j + 3];+ }+ }+ } else if constexpr (PackRow == 2) {+ for (int i = 0; i < row; i += PackRow) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 2) {+ auto bit2 = bit2ptr[(i * col + j * PackRow) / 4];+ auto tmp = bit4ptr[(i * col + j * PackRow) / 2];+ auto zp = zptr[j] + FullRange;+ dstptr[i * col + j * PackRow + 0] = (tmp.x | (bit2.a << 4)) - zp;+ dstptr[i * col + j * PackRow + 1] = (tmp.y | (bit2.b << 4)) - zp;+ zp = zptr[j + 1] + FullRange;+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 1];+ dstptr[i * col + j * PackRow + 2] = (tmp.x | (bit2.c << 4)) - zp;+ dstptr[i * col + j * PackRow + 3] = (tmp.y | (bit2.d << 4)) - zp;+ }+ }+ } else {+ static_assert(PackRow == 1 || PackRow == 2 || PackRow == 4);+ }+ } else {+ return decompress_s6_s8(bit4ptr, bit2ptr, dstptr, size_t(row) * col, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s5_s8(utils::bit4x2* bit4ptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset, int k_offset,+ int row, int col, int8_t* tmp, size_t tmpsize) {+ int constexpr FullRange = 1 << (5 - 1);+ static_assert(NTILE % 8 == 0);+ assert(((col * PackRow) % 8) == 0);+ if (zpptr) {+ if constexpr (PackRow == 4) {+ for (int i = 0; i < row; i += PackRow) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 2) {+ auto zp = zptr[j] + FullRange;+ auto bit1 = bit1ptr[(i * col + j * PackRow) / 8];+ auto tmp = bit4ptr[(i * col + j * PackRow) / 2];+ dstptr[i * col + j * PackRow + 0] = (tmp.x | (bit1.a << 4)) - zp;+ dstptr[i * col + j * PackRow + 1] = (tmp.y | (bit1.b << 4)) - zp;+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 1];+ dstptr[i * col + j * PackRow + 2] = (tmp.x | (bit1.c << 4)) - zp;+ dstptr[i * col + j * PackRow + 3] = (tmp.y | (bit1.d << 4)) - zp;+ zp = zptr[j + 1] + FullRange;+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 2];+ dstptr[i * col + j * PackRow + 4] = (tmp.x | (bit1.e << 4)) - zp;+ dstptr[i * col + j * PackRow + 5] = (tmp.y | (bit1.f << 4)) - zp;+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 3];+ dstptr[i * col + j * PackRow + 6] = (tmp.x | (bit1.g << 4)) - zp;+ dstptr[i * col + j * PackRow + 7] = (tmp.y | (bit1.h << 4)) - zp;+ }+ }+ } else if constexpr (PackRow == 1) {+ for (int i = 0; i < row; i += 1) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 8) {+ auto bit1 = bit1ptr[(i * col + j * PackRow) / 8];+ auto tmp = bit4ptr[(i * col + j * PackRow) / 2];+ dstptr[i * col + j * PackRow + 0] = (tmp.x | (bit1.a << 4)) - FullRange - zptr[j + 0];+ dstptr[i * col + j * PackRow + 1] = (tmp.y | (bit1.b << 4)) - FullRange - zptr[j + 1];+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 1];+ dstptr[i * col + j * PackRow + 2] = (tmp.x | (bit1.c << 4)) - FullRange - zptr[j + 2];+ dstptr[i * col + j * PackRow + 3] = (tmp.y | (bit1.d << 4)) - FullRange - zptr[j + 3];+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 2];+ dstptr[i * col + j * PackRow + 4] = (tmp.x | (bit1.e << 4)) - FullRange - zptr[j + 4];+ dstptr[i * col + j * PackRow + 5] = (tmp.y | (bit1.f << 4)) - FullRange - zptr[j + 5];+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 3];+ dstptr[i * col + j * PackRow + 6] = (tmp.x | (bit1.g << 4)) - FullRange - zptr[j + 6];+ dstptr[i * col + j * PackRow + 7] = (tmp.y | (bit1.h << 4)) - FullRange - zptr[j + 7];+ }+ }+ } else if constexpr (PackRow == 2) {+ for (int i = 0; i < row; i += PackRow) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 4) {+ auto bit1 = bit1ptr[(i * col + j * PackRow) / 8];+ auto tmp = bit4ptr[(i * col + j * PackRow) / 2];+ auto zp = zptr[j] + FullRange;+ dstptr[i * col + j * PackRow + 0] = (tmp.x | (bit1.a << 4)) - zp;+ dstptr[i * col + j * PackRow + 1] = (tmp.y | (bit1.b << 4)) - zp;+ zp = zptr[j + 1] + FullRange;+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 1];+ dstptr[i * col + j * PackRow + 2] = (tmp.x | (bit1.c << 4)) - zp;+ dstptr[i * col + j * PackRow + 3] = (tmp.y | (bit1.d << 4)) - zp;+ zp = zptr[j + 2] + FullRange;+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 2];+ dstptr[i * col + j * PackRow + 4] = (tmp.x | (bit1.e << 4)) - zp;+ dstptr[i * col + j * PackRow + 5] = (tmp.y | (bit1.f << 4)) - zp;+ zp = zptr[j + 3] + FullRange;+ tmp = bit4ptr[(i * col + j * PackRow) / 2 + 3];+ dstptr[i * col + j * PackRow + 6] = (tmp.x | (bit1.g << 4)) - zp;+ dstptr[i * col + j * PackRow + 7] = (tmp.y | (bit1.h << 4)) - zp;+ }+ }+ } else {+ static_assert(PackRow == 1 || PackRow == 2 || PackRow == 4);+ }+ } else {+ return decompress_s5_s8(bit4ptr, bit1ptr, dstptr, size_t(row) * col, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s4_s8(utils::int4x2* srcptr, int8_t* zpptr, int8_t* dstptr, int blocksize,+ int ldzp, int n_offset, int k_offset, int row, int col, int8_t* tmp,+ size_t tmpsize) {+ int8_t constexpr FullRange = 1 << (4 - 1);+ if (zpptr) {+ if constexpr (PackRow == 4 || PackRow == 2) {+ for (int i = 0; i < row; i += PackRow) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 1) {+ auto zp = zptr[j] + FullRange;+ for (int ir = 0; ir < PackRow; ir += 2) {+ auto tmp = srcptr[i * col / 2 + j * PackRow / 2 + ir / 2];+ dstptr[i * col + j * PackRow + ir + 0] = tmp.x - zp;+ dstptr[i * col + j * PackRow + ir + 1] = tmp.y - zp;+ }+ }+ }+ } else if constexpr (PackRow == 1) {+ for (int i = 0; i < row; i += 1) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 2) {+ auto tmp = srcptr[i * col / 2 + j / 2];+ dstptr[i * col + j + 0] = tmp.x - FullRange - zptr[j + 0];+ dstptr[i * col + j + 1] = tmp.y - FullRange - zptr[j + 1];+ }+ }+ } else {+ static_assert(PackRow == 1 || PackRow == 2 || PackRow == 4);+ }+ } else {+ return decompress_s4_s8(srcptr, dstptr, size_t(row) * col, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s3_s8(utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset, int k_offset,+ int row, int col, int8_t* tmp, size_t tmpsize) {+ static_assert(NTILE % 8 == 0);+ int8_t constexpr FullRange = 1 << (3 - 1);+ assert(((col * PackRow) % 8) == 0);+ if (zpptr) {+ if constexpr (PackRow == 4) {+ for (int i = 0; i < row; i += PackRow) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 2) {+ auto zp = zptr[j] + FullRange;+ auto bit1 = bit1ptr[(i * col + j * PackRow) / 8];+ auto tmp = bit2ptr[(i * col + j * PackRow) / 4];+ dstptr[i * col + j * PackRow + 0] = (tmp.a | (bit1.a << 2)) - zp;+ dstptr[i * col + j * PackRow + 1] = (tmp.b | (bit1.b << 2)) - zp;+ dstptr[i * col + j * PackRow + 2] = (tmp.c | (bit1.c << 2)) - zp;+ dstptr[i * col + j * PackRow + 3] = (tmp.d | (bit1.d << 2)) - zp;+ zp = zptr[j + 1] + FullRange;+ tmp = bit2ptr[(i * col + j * PackRow) / 4 + 1];+ dstptr[i * col + j * PackRow + 4] = (tmp.a | (bit1.e << 2)) - zp;+ dstptr[i * col + j * PackRow + 5] = (tmp.b | (bit1.f << 2)) - zp;+ dstptr[i * col + j * PackRow + 6] = (tmp.c | (bit1.g << 2)) - zp;+ dstptr[i * col + j * PackRow + 7] = (tmp.d | (bit1.h << 2)) - zp;+ }+ }+ } else if constexpr (PackRow == 1) {+ for (int i = 0; i < row; i += 1) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 8) {+ auto bit1 = bit1ptr[(i * col + j * PackRow) / 8];+ auto tmp = bit2ptr[(i * col + j * PackRow) / 4];+ dstptr[i * col + j * PackRow + 0] = (tmp.a | (bit1.a << 2)) - FullRange - zptr[j + 0];+ dstptr[i * col + j * PackRow + 1] = (tmp.b | (bit1.b << 2)) - FullRange - zptr[j + 1];+ dstptr[i * col + j * PackRow + 2] = (tmp.c | (bit1.c << 2)) - FullRange - zptr[j + 2];+ dstptr[i * col + j * PackRow + 3] = (tmp.d | (bit1.d << 2)) - FullRange - zptr[j + 3];+ tmp = bit2ptr[(i * col + j * PackRow) / 4 + 1];+ dstptr[i * col + j * PackRow + 4] = (tmp.a | (bit1.e << 2)) - FullRange - zptr[j + 4];+ dstptr[i * col + j * PackRow + 5] = (tmp.b | (bit1.f << 2)) - FullRange - zptr[j + 5];+ dstptr[i * col + j * PackRow + 6] = (tmp.c | (bit1.g << 2)) - FullRange - zptr[j + 6];+ dstptr[i * col + j * PackRow + 7] = (tmp.d | (bit1.h << 2)) - FullRange - zptr[j + 7];+ }+ }+ } else if constexpr (PackRow == 2) {+ for (int i = 0; i < row; i += PackRow) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 4) {+ auto bit1 = bit1ptr[(i * col + j * PackRow) / 8];+ auto tmp = bit2ptr[(i * col + j * PackRow) / 4];+ auto zp = zptr[j] + FullRange;+ dstptr[i * col + j * PackRow + 0] = (tmp.a | (bit1.a << 2)) - zp;+ dstptr[i * col + j * PackRow + 1] = (tmp.b | (bit1.b << 2)) - zp;+ zp = zptr[j + 1] + FullRange;+ dstptr[i * col + j * PackRow + 2] = (tmp.c | (bit1.c << 2)) - zp;+ dstptr[i * col + j * PackRow + 3] = (tmp.d | (bit1.d << 2)) - zp;+ tmp = bit2ptr[(i * col + j * PackRow) / 4 + 1];+ zp = zptr[j + 2] + FullRange;+ dstptr[i * col + j * PackRow + 4] = (tmp.a | (bit1.e << 2)) - zp;+ dstptr[i * col + j * PackRow + 5] = (tmp.b | (bit1.f << 2)) - zp;+ zp = zptr[j + 3] + FullRange;+ dstptr[i * col + j * PackRow + 6] = (tmp.c | (bit1.g << 2)) - zp;+ dstptr[i * col + j * PackRow + 7] = (tmp.d | (bit1.h << 2)) - zp;+ }+ }+ } else {+ static_assert(PackRow == 1 || PackRow == 2 || PackRow == 4);+ }+ } else {+ return decompress_s3_s8(bit2ptr, bit1ptr, dstptr, size_t(row) * col, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s2_s8(utils::bit2x4* bit2ptr, int8_t* zpptr, int8_t* dstptr, int blocksize,+ int ldzp, int n_offset, int k_offset, int row, int col, int8_t* tmp,+ size_t tmpsize) {+ static_assert(NTILE % 4 == 0);+ int8_t constexpr FullRange = 1 << (2 - 1);+ assert(((col * PackRow) % 4) == 0);+ if (zpptr) {+ if constexpr (PackRow == 4) {+ for (int i = 0; i < row; i += PackRow) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 1) {+ auto zp = zptr[j] + FullRange;+ auto tmp = bit2ptr[(i * col + j * PackRow) / 4];+ dstptr[i * col + j * PackRow + 0] = (tmp.a) - zp;+ dstptr[i * col + j * PackRow + 1] = (tmp.b) - zp;+ dstptr[i * col + j * PackRow + 2] = (tmp.c) - zp;+ dstptr[i * col + j * PackRow + 3] = (tmp.d) - zp;+ }+ }+ } else if constexpr (PackRow == 1) {+ for (int i = 0; i < row; i += 1) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 4) {+ auto tmp = bit2ptr[(i * col + j * PackRow) / 4];+ dstptr[i * col + j * PackRow + 0] = (tmp.a) - FullRange - zptr[j + 0];+ dstptr[i * col + j * PackRow + 1] = (tmp.b) - FullRange - zptr[j + 1];+ dstptr[i * col + j * PackRow + 2] = (tmp.c) - FullRange - zptr[j + 2];+ dstptr[i * col + j * PackRow + 3] = (tmp.d) - FullRange - zptr[j + 3];+ }+ }+ } else if constexpr (PackRow == 2) {+ for (int i = 0; i < row; i += PackRow) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 2) {+ auto tmp = bit2ptr[(i * col + j * PackRow) / 4];+ auto zp = zptr[j] + FullRange;+ dstptr[i * col + j * PackRow + 0] = (tmp.a) - zp;+ dstptr[i * col + j * PackRow + 1] = (tmp.b) - zp;+ zp = zptr[j + 1] + FullRange;+ dstptr[i * col + j * PackRow + 2] = (tmp.c) - zp;+ dstptr[i * col + j * PackRow + 3] = (tmp.d) - zp;+ }+ }+ } else {+ static_assert(PackRow == 1 || PackRow == 2 || PackRow == 4);+ }+ } else {+ return decompress_s2_s8(bit2ptr, dstptr, size_t(row) * col, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE>+static inline BTLA_CODE decompress_kblock_s1_s8(utils::bit1x8* bit1ptr, int8_t* zpptr, int8_t* dstptr, int blocksize,+ int ldzp, int n_offset, int k_offset, int row, int col, int8_t* tmp,+ size_t tmpsize) {+ static_assert(NTILE % 8 == 0);+ int8_t constexpr FullRange = 1 << (1 - 1);+ assert(((col * PackRow) % 8) == 0);+ if (zpptr) {+ if constexpr (PackRow == 4) {+ for (int i = 0; i < row; i += PackRow) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 2) {+ auto zp = zptr[j] + FullRange;+ auto bit1 = bit1ptr[(i * col + j * PackRow) / 8];+ dstptr[i * col + j * PackRow + 0] = bit1.a - zp;+ dstptr[i * col + j * PackRow + 1] = bit1.b - zp;+ dstptr[i * col + j * PackRow + 2] = bit1.c - zp;+ dstptr[i * col + j * PackRow + 3] = bit1.d - zp;+ zp = zptr[j + 1] + FullRange;+ dstptr[i * col + j * PackRow + 4] = bit1.e - zp;+ dstptr[i * col + j * PackRow + 5] = bit1.f - zp;+ dstptr[i * col + j * PackRow + 6] = bit1.g - zp;+ dstptr[i * col + j * PackRow + 7] = bit1.h - zp;+ }+ }+ } else if constexpr (PackRow == 1) {+ for (int i = 0; i < row; i += 1) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 8) {+ auto bit1 = bit1ptr[(i * col + j * PackRow) / 8];+ dstptr[i * col + j * PackRow + 0] = bit1.a - FullRange - zptr[j + 0];+ dstptr[i * col + j * PackRow + 1] = bit1.b - FullRange - zptr[j + 1];+ dstptr[i * col + j * PackRow + 2] = bit1.c - FullRange - zptr[j + 2];+ dstptr[i * col + j * PackRow + 3] = bit1.d - FullRange - zptr[j + 3];+ dstptr[i * col + j * PackRow + 4] = bit1.e - FullRange - zptr[j + 4];+ dstptr[i * col + j * PackRow + 5] = bit1.f - FullRange - zptr[j + 5];+ dstptr[i * col + j * PackRow + 6] = bit1.g - FullRange - zptr[j + 6];+ dstptr[i * col + j * PackRow + 7] = bit1.h - FullRange - zptr[j + 7];+ }+ }+ } else if constexpr (PackRow == 2) {+ for (int i = 0; i < row; i += PackRow) {+ auto zptr = zpptr + (i + k_offset) / blocksize * ldzp + n_offset;+ for (int j = 0; j < col; j += 4) {+ auto bit1 = bit1ptr[(i * col + j * PackRow) / 8];+ auto zp = zptr[j] + FullRange;+ dstptr[i * col + j * PackRow + 0] = bit1.a - zp;+ dstptr[i * col + j * PackRow + 1] = bit1.b - zp;+ zp = zptr[j + 1] + FullRange;+ dstptr[i * col + j * PackRow + 2] = bit1.c - zp;+ dstptr[i * col + j * PackRow + 3] = bit1.d - zp;+ zp = zptr[j + 2] + FullRange;+ dstptr[i * col + j * PackRow + 4] = bit1.e - zp;+ dstptr[i * col + j * PackRow + 5] = bit1.f - zp;+ zp = zptr[j + 3] + FullRange;+ dstptr[i * col + j * PackRow + 6] = bit1.g - zp;+ dstptr[i * col + j * PackRow + 7] = bit1.h - zp;+ }+ }+ } else {+ static_assert(PackRow == 1 || PackRow == 2 || PackRow == 4);+ }+ } else {+ return decompress_s1_s8(bit1ptr, dstptr, size_t(row) * col, tmp, tmpsize);+ }+ return BTLA_CODE::Success;+}++static inline float f8_to_fp32(utils::f8 v, BTLA_DTYPE f8_t) {+ uint32_t sign_revert = v.x;+ uint32_t e_revert = v.x;+ uint32_t mantissa_revert = v.x;+ sign_revert <<= 24;+ sign_revert &= 0x80000000;+ auto ebits = utils::bestla_dtype_get_f8_ebits(f8_t);+ auto mantissabit = 7 - ebits;+ e_revert &= 0x7f;+ e_revert >>= mantissabit;+ e_revert = e_revert - static_cast<uint32_t>(std::pow(2, ebits - 1)) + 1 + 127;+ e_revert <<= 23;+ mantissa_revert <<= (23 - mantissabit);+ mantissa_revert &= 0x007fffff;+ uint32_t revert = sign_revert | e_revert | mantissa_revert;+ float* fp_v = reinterpret_cast<float*>(&revert);+ return *fp_v;+}++template <typename _DST_T, int _PACK_ROW, typename _S_T>+static inline BTLA_CODE decompress_kblock_f8_fp(utils::f8* srcptr, _DST_T* dstptr, int row, int col, int ld_src,+ int ld_dst, _S_T* scales, int k_offset, int kblock, int NPad,+ BTLA_DTYPE src_f8_type) {+ for (int i = 0; i < row; i++) {+ int kpos = (k_offset + i) / kblock;+ auto sptr = scales + kpos * NPad;+ for (int j = 0; j < col; j++) {+ auto fp_v = f8_to_fp32(srcptr[i * ld_src + j], src_f8_type);+ float scale;+ if constexpr (std::is_same_v<_S_T, utils::f8>) {+ int shared_exp = sptr[j / _PACK_ROW].x;+ scale = static_cast<float>(std::pow(2, shared_exp));+ } else if constexpr (std::is_same_v<_S_T, float>) {+ scale = scales[j / _PACK_ROW];+ } else {+ assert(0);+ }+ dstptr[i * ld_dst + j] = fp_v * scale;+ }+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE, typename DST_T>+inline BTLA_CODE decompress_kblock_s8_fp(int8_t* srcptr, DST_T* dstptr, int row, int col, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ for (int i = 0; i < row; i += PackRow) {+ int kpos = (k_offset + i) / blocksize * ldzp + n_offset;+ if (sdtype == BTLA_DTYPE::F32) {+ auto sptr = (float*)scales_ + kpos;+ for (int j = 0; j < col; j += 1) {+ auto scale = float(sptr[j]);+ auto zp = zero_points ? zero_points[kpos + j] : 0;+ for (int ir = 0; ir < PackRow; ir++) {+ float tmp = static_cast<float>(srcptr[i * col + j * PackRow + ir] - zp) * scale;+ dstptr[i * col + j * PackRow + ir] = static_cast<DST_T>(tmp);+ }+ }+ } else if (sdtype == BTLA_DTYPE::BF16) {+ auto sptr = (utils::bf16*)scales_ + kpos;+ for (int j = 0; j < col; j += 1) {+ auto scale = float(sptr[j]);+ auto zp = zero_points ? zero_points[kpos + j] : 0;+ for (int ir = 0; ir < PackRow; ir++) {+ float tmp = static_cast<float>(srcptr[i * col + j * PackRow + ir] - zp) * scale;+ dstptr[i * col + j * PackRow + ir] = static_cast<DST_T>(tmp);+ }+ }+ }+ }+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE, typename DST_T>+static inline BTLA_CODE decompress_kblock_s7_fp(utils::bit4x2* b4ptr, utils::bit2x4* b2ptr, utils::bit1x8* b1ptr,+ DST_T* dstptr, int row, int col, void* scales_, BTLA_DTYPE sdtype,+ int8_t* zero_points, int k_offset, int n_offset, int blocksize,+ int ldzp, int8_t* tmp, size_t tmpsize) {+ assert(tmpsize >= PackRow * NTILE);+ assert(NTILE == col);+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ decompress_kblock_s7_s8<PackRow, NTILE>(b4ptr, b2ptr, b1ptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset,+ k_offset, row, col, tmp, tmpsize);+ decompress_kblock_s8_fp<PackRow, NTILE>(tmps8ptr, dstptr, row, col, scales_, sdtype, nullptr, k_offset, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE, typename DST_T>+static inline BTLA_CODE decompress_kblock_s6_fp(utils::bit4x2* b4ptr, utils::bit2x4* b2ptr, DST_T* dstptr, int row,+ int col, void* scales_, BTLA_DTYPE sdtype, int8_t* zero_points,+ int k_offset, int n_offset, int blocksize, int ldzp, int8_t* tmp,+ size_t tmpsize) {+ assert(tmpsize >= PackRow * NTILE);+ assert(NTILE == col);+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ decompress_kblock_s6_s8<PackRow, NTILE>(b4ptr, b2ptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset, k_offset, row,+ col, tmp, tmpsize);+ decompress_kblock_s8_fp<PackRow, NTILE>(tmps8ptr, dstptr, row, col, scales_, sdtype, nullptr, k_offset, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE, typename DST_T>+static inline BTLA_CODE decompress_kblock_s5_fp(utils::bit4x2* b4ptr, utils::bit1x8* b1ptr, DST_T* dstptr, int row,+ int col, void* scales_, BTLA_DTYPE sdtype, int8_t* zero_points,+ int k_offset, int n_offset, int blocksize, int ldzp, int8_t* tmp,+ size_t tmpsize) {+ assert(tmpsize >= PackRow * NTILE);+ assert(NTILE == col);+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ decompress_kblock_s5_s8<PackRow, NTILE>(b4ptr, b1ptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset, k_offset, row,+ col, tmp, tmpsize);+ decompress_kblock_s8_fp<PackRow, NTILE>(tmps8ptr, dstptr, row, col, scales_, sdtype, nullptr, k_offset, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE, typename DST_T>+static inline BTLA_CODE decompress_kblock_s4_fp(utils::int4x2* srcptr, DST_T* dstptr, int row, int col, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ assert(tmpsize >= PackRow * NTILE);+ assert(NTILE == col);+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ decompress_kblock_s4_s8<PackRow, NTILE>(srcptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset, k_offset, row, col,+ tmp, tmpsize);+ decompress_kblock_s8_fp<PackRow, NTILE>(tmps8ptr, dstptr, row, col, scales_, sdtype, nullptr, k_offset, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE, typename DST_T>+static inline BTLA_CODE decompress_kblock_s3_fp(utils::bit2x4* b2ptr, utils::bit1x8* b1ptr, DST_T* dstptr, int row,+ int col, void* scales_, BTLA_DTYPE sdtype, int8_t* zero_points,+ int k_offset, int n_offset, int blocksize, int ldzp, int8_t* tmp,+ size_t tmpsize) {+ assert(tmpsize >= PackRow * NTILE);+ assert(NTILE == col);+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ decompress_kblock_s3_s8<PackRow, NTILE>(b2ptr, b1ptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset, k_offset, row,+ col, tmp, tmpsize);+ decompress_kblock_s8_fp<PackRow, NTILE>(tmps8ptr, dstptr, row, col, scales_, sdtype, nullptr, k_offset, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE, typename DST_T>+static inline BTLA_CODE decompress_kblock_s2_fp(utils::bit2x4* b2ptr, DST_T* dstptr, int row, int col, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ assert(tmpsize >= PackRow * NTILE);+ assert(NTILE == col);+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ decompress_kblock_s2_s8<PackRow, NTILE>(b2ptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset, k_offset, row, col,+ tmp, tmpsize);+ decompress_kblock_s8_fp<PackRow, NTILE>(tmps8ptr, dstptr, row, col, scales_, sdtype, nullptr, k_offset, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ return BTLA_CODE::Success;+}++template <int PackRow, int NTILE, typename DST_T>+static inline BTLA_CODE decompress_kblock_s1_fp(utils::bit1x8* b1ptr, DST_T* dstptr, int row, int col, void* scales_,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int blocksize, int ldzp, int8_t* tmp, size_t tmpsize) {+ assert(tmpsize >= PackRow * NTILE);+ assert(NTILE == col);+ const auto DstSize = row * NTILE * sizeof(DST_T);+ const auto S8Size = row * NTILE * sizeof(int8_t);+ auto tmps8ptr = (int8_t*)dstptr;+ tmps8ptr += DstSize - S8Size;+ decompress_kblock_s1_s8<PackRow, NTILE>(b1ptr, zero_points, tmps8ptr, blocksize, ldzp, n_offset, k_offset, row, col,+ tmp, tmpsize);+ decompress_kblock_s8_fp<PackRow, NTILE>(tmps8ptr, dstptr, row, col, scales_, sdtype, nullptr, k_offset, n_offset,+ blocksize, ldzp, tmp, tmpsize);+ return BTLA_CODE::Success;+}++template <BTLA_DTYPE S4_T, typename _DST_T, int _PACK_ROW>+static inline BTLA_CODE decompress_dq_kblock_s4_fp(utils::int4x2* srcptr, _DST_T* dstptr, int row, int col, int ld_src,+ int ld_dst, uint8_t* scales, float* dq_scale, int k_offset,+ int n_offset, int kblock, int dq_blk, int dq_offset_idx, int NPad,+ int N, void* tmp, size_t tmpsize) {+ auto sptr_base = scales + n_offset;+ for (int i = 0; i < row; i++) {+ int kpos = (k_offset + i) / kblock;+ auto sptr = sptr_base + kpos * NPad;+ for (int j = 0; j < col; j += 2) {+ auto tmp = srcptr[i * ld_src / 2 + j / 2];+ float scale0, scale1, dst0, dst1;+ int s0_idx, s1_idx;+ s0_idx = j / _PACK_ROW;+ s1_idx = (j + 1) / _PACK_ROW;+ auto dq_s0_idx = (n_offset + kpos * N + s0_idx) / dq_blk;+ auto dq_s1_idx = (n_offset + kpos * N + s1_idx) / dq_blk;+ scale0 = dq8_bnb_LUT[sptr[s0_idx]] * dq_scale[dq_s0_idx] + dq_scale[dq_offset_idx];+ scale1 = dq8_bnb_LUT[sptr[s1_idx]] * dq_scale[dq_s1_idx] + dq_scale[dq_offset_idx];+ dst0 = static_cast<float>(tmp.x - 8) * scale0;+ dst1 = static_cast<float>(tmp.y - 8) * scale1;+ dstptr[i * ld_dst + j + 0] = static_cast<_DST_T>(dst0);+ dstptr[i * ld_dst + j + 1] = static_cast<_DST_T>(dst1);+ }+ }+ return BTLA_CODE::Success;+}++static inline float fp4_bnb_unpack(uint8_t val) {+ float sign = (val & 0b1000) == 8 ? -1.0f : 1.0f;+ if ((val & 0b0100) == 4) // 0+ if ((val & 0b0010) == 2) // 01+ if ((val & 0b0001) == 1) // 111+ return 0.25000000f * sign; // 1111+ else+ return 0.16666667f * sign; // 1110+ else if ((val & 0b0001) == 1) // 110+ return 0.50000000f * sign; // 1101+ else+ return 0.33333333f * sign; // 1100+ else if ((val & 0b0010) == 2) // 10+ if ((val & 0b0001) == 1) // 101+ return 1.00000000f * sign; // 1011+ else+ return 0.66666667f * sign; // 1010+ else if ((val & 0b0001) == 1) // 100+ return 5.208333333e-03f * sign; // 1001+ else+ return 0.00000000f * sign; // 1000+}++static inline float fp4_bnb_dequantize(uint8_t val, float absmax) { return fp4_bnb_unpack(val) * absmax; }++static inline int8_t fp4_bnb_quantize(float x) {+ int sign = x < 0 ? 0b1000 : 0b0000;+ x = fabs(x);+ if (x > 0.29166667f)+ if (x > 0.583333f)+ if (x > 0.8333333f)+ return static_cast<int8_t>(0b0011 + sign);+ else+ return static_cast<int8_t>(0b0010 + sign);+ else if (x > 0.4166667f)+ return static_cast<int8_t>(0b101 + sign);+ else+ return static_cast<int8_t>(0b100 + sign);+ else if (x > 0.0859375f)+ if (x > 0.20833333f)+ return static_cast<int8_t>(0b0111 + sign);+ else+ return static_cast<int8_t>(0b0110 + sign);+ else if (x > 0.00260417f)+ return static_cast<int8_t>(0b0001 + sign);+ else+ return static_cast<int8_t>(0b0000 + sign);+}++static inline int8_t fp4_e2m1_quantize(float x) {+ // FP4 with bias of 1+ // first bit is a sign+ // subnormals+ // 0b000 = 0+ // 0b001 = 0.0625+ // 0b010 = 1+ // 0b011 = 1.5+ // 0b100 = 2+ // 0b101 = 3+ // 0b110 = 4+ // 0b111 = 6++ int sign = x < 0 ? 0b1000 : 0b0000;+ x = fabs(x);+ if (x > 1.75f / 6) {+ if (x > 3.5f / 6) {+ if (x > 5.f / 6)+ return static_cast<int8_t>(0b111 + sign); // 6+ else+ return static_cast<int8_t>(0b110 + sign); // 4+ } else {+ if (x > 2.5f / 6)+ return static_cast<int8_t>(0b101 + sign); // 3+ else+ return static_cast<int8_t>(0b100 + sign); // 2+ }+ } else {+ if (x > 0.53125f / 6) {+ if (x > 1.25f / 6)+ return static_cast<int8_t>(0b011 + sign); // 1.5+ else+ return static_cast<int8_t>(0b010 + sign); // 1+ } else {+ if (x > 0.03125f / 6)+ return static_cast<int8_t>(0b0001 + sign); // 0.0625+ else+ return static_cast<int8_t>(0b0000 + sign); // 0+ }+ }+}++static inline float fp4_e2m1_unpack(uint8_t val) {+ float sign = (val & 0b1000) == 8 ? -1.0f : 1.0f;+ if ((val & 0b0100) == 4) // 0+ if ((val & 0b0010) == 2) // 01+ if ((val & 0b0001) == 1) // 111+ return 1.f * sign; // 1111+ else+ return 0.6666666666666666f * sign; // 1110+ else if ((val & 0b0001) == 1) // 110+ return 0.5f * sign; // 1101+ else+ return 0.3333333333333333f * sign; // 1100+ else if ((val & 0b0010) == 2) // 10+ if ((val & 0b0001) == 1) // 101+ return 0.25f * sign; // 1011+ else+ return 0.16666666666666666f * sign; // 1010+ else if ((val & 0b0001) == 1) // 100+ return 0.010416666666666666f * sign; // 1001+ else+ return 0.00000000f * sign; // 1000+}++static inline float fp4_e2m1_dequantize(uint8_t val, float absmax) { return fp4_e2m1_unpack(val) * absmax; }++static inline float nf4_unpack(int8_t val) {+ if ((val & 0b1000) == 8)+ if ((val & 0b0100) == 4) // 1+ if ((val & 0b0010) == 2) // 11+ if ((val & 0b0001) == 1) // 111+ return 1.0f;+ else+ return 0.7229568362236023f;+ else if ((val & 0b0001) == 1) // 110+ return 0.5626170039176941f;+ else+ return 0.44070982933044434f;+ else if ((val & 0b0010) == 2) // 10+ if ((val & 0b0001) == 1) // 101+ return 0.33791524171829224f;+ else+ return 0.24611230194568634f;+ else if ((val & 0b0001) == 1) // 100+ return 0.16093020141124725f;+ else+ return 0.07958029955625534f;++ else if ((val & 0b0100) == 4) // 0+ if ((val & 0b0010) == 2) // 01+ if ((val & 0b0001) == 1) // 011+ return -1.f;+ else+ return -0.09105003625154495f;+ else if ((val & 0b0001) == 1) // 010+ return -0.18477343022823334f;+ else+ return -0.28444138169288635f;+ else if ((val & 0b0010) == 2) // 00+ if ((val & 0b0001) == 1) // 001+ return -0.39491748809814453f;+ else+ return -0.5250730514526367f;+ else if ((val & 0b0001) == 1) // 000+ return -0.6961928009986877f;+ else+ return 0.f;+}++static inline float nf4_dequantize(int8_t val, float absmax) { return nf4_unpack(val) * absmax; }++// Note: In the BNB Nf4 definition, 0 has a non-zero value after dequantization,+// but BTLA uses 0 for padding, which leads to calculation errors. We ultimately+// choose to swap the binary bits of -1 and 0 in Nf4 to avoid this conflict.+static inline int8_t nf4_quantize(float x) {+ if (x > 0.03979014977812767f)+ if (x > 0.3893125355243683f) // 1+ if (x > 0.6427869200706482f) // 11+ if (x > 0.8614784181118011f) // 111+ return 0b1111;+ else+ return 0b1110;+ else if (x > 0.5016634166240692f) // 110+ return 0b1101;+ else+ return 0b1100;+ else if (x > 0.2035212516784668f) // 10+ if (x > 0.2920137718319893f) // 101+ return 0b1011;+ else+ return 0b1010;+ else if (x > 0.1202552504837513f) // 100+ return 0b1001;+ else+ return 0b1000;+ else if (x > -0.33967943489551544f) // 0+ if (x > -0.13791173323988914f) // 01+ if (x > -0.045525018125772476f) // 011+ return 0b0000;+ else+ return 0b0110;+ else if (x > -0.23460740596055984f) // 010+ return 0b0101;+ else+ return 0b0100;+ else if (x > -0.6106329262256622f) // 00+ if (x > -0.4599952697753906f) // 001+ return 0b0011;+ else+ return 0b0010;+ else if (x > -0.8480964004993439f) // 000+ return 0b0001;+ else+ return 0b0111;+}++template <BTLA_DTYPE F4_T>+static inline float f4_unpack(int8_t v) {+ static_assert(F4_T == BTLA_DTYPE::F4_BNB || F4_T == BTLA_DTYPE::F4_NF4 || F4_T == BTLA_DTYPE::F4_E2M1,+ "Unsupported F4 type");+ switch (F4_T) {+ case BTLA_DTYPE::F4_BNB:+ return fp4_bnb_unpack(v);+ case BTLA_DTYPE::F4_NF4:+ return nf4_unpack(v);+ case BTLA_DTYPE::F4_E2M1:+ return fp4_e2m1_unpack(v);+ default:+ break;+ }+ return std::numeric_limits<float>::quiet_NaN();+}++template <BTLA_DTYPE F4_T>+static inline float f4_dequantize(int8_t v, float scale) {+ static_assert(F4_T == BTLA_DTYPE::F4_BNB || F4_T == BTLA_DTYPE::F4_NF4 || F4_T == BTLA_DTYPE::F4_E2M1,+ "Unsupported F4 type");+ return f4_unpack<F4_T>(v) * scale;+}++template <BTLA_DTYPE F4_T>+static inline int8_t f4_quantize(float x) {+ static_assert(F4_T == BTLA_DTYPE::F4_BNB || F4_T == BTLA_DTYPE::F4_NF4 || F4_T == BTLA_DTYPE::F4_E2M1,+ "Unsupported F4 type");+ switch (F4_T) {+ case BTLA_DTYPE::F4_BNB:+ return fp4_bnb_quantize(x);+ case BTLA_DTYPE::F4_NF4:+ return nf4_quantize(x);+ case BTLA_DTYPE::F4_E2M1:+ return fp4_e2m1_quantize(x);+ default:+ break;+ }+ return static_cast<int8_t>(0);+}++template <BTLA_DTYPE F4_T, typename _DST_T, int _PACK_ROW, typename _S_T>+static inline BTLA_CODE decompress_kblock_f4_fp(utils::f4x2* srcptr, _DST_T* dstptr, int row, int col, int ld_src,+ int ld_dst, _S_T* scales, int k_offset, int kblock, int NPad,+ int8_t* tmp, size_t tmpsize) {+ for (int i = 0; i < row; i++) {+ int kpos = (k_offset + i) / kblock;+ auto sptr = scales + kpos * NPad;+ for (int j = 0; j < col; j += 2) {+ auto tmp = srcptr[i * ld_src / 2 + j / 2];+ float scale0, scale1, dst0, dst1;+ int s0_idx, s1_idx;+ s0_idx = j / _PACK_ROW;+ s1_idx = (j + 1) / _PACK_ROW;+ scale0 = static_cast<float>(sptr[s0_idx]);+ scale1 = static_cast<float>(sptr[s1_idx]);+ dst0 = f4_dequantize<F4_T>(tmp.x, scale0);+ dst1 = f4_dequantize<F4_T>(tmp.y, scale1);+ dstptr[i * ld_dst + j + 0] = static_cast<_DST_T>(dst0);+ dstptr[i * ld_dst + j + 1] = static_cast<_DST_T>(dst1);+ }+ }+ return BTLA_CODE::Success;+}++template <BTLA_DTYPE F4_T, int _PACK_ROW, typename _DST_T, typename SCA_T>+static inline BTLA_CODE decompress_dq_kblock_f4_fp(utils::f4x2* srcptr, _DST_T* dstptr, int row, int col, int ld_src,+ int ld_dst, SCA_T* scales, float* dq_scale, int k_offset,+ int n_offset, int kblock, int dq_blk, int dq_offset_idx, int NPad,+ int N, void* tmp, size_t tmpsize) {+ auto sptr_base = scales + n_offset;+ for (int i = 0; i < row; i++) {+ int kpos = (k_offset + i) / kblock;+ auto sptr = sptr_base + kpos * NPad;+ for (int j = 0; j < col; j += 2) {+ auto tmp = srcptr[i * ld_src / 2 + j / 2];+ float scale0, scale1, dst0, dst1;+ int s0_idx, s1_idx;+ s0_idx = j / _PACK_ROW;+ s1_idx = (j + 1) / _PACK_ROW;+ auto dq_s0_idx = (n_offset + kpos * N + s0_idx) / dq_blk;+ auto dq_s1_idx = (n_offset + kpos * N + s1_idx) / dq_blk;+ scale0 = dq8_bnb_LUT[sptr[s0_idx]] * dq_scale[dq_s0_idx] + dq_scale[dq_offset_idx];+ scale1 = dq8_bnb_LUT[sptr[s1_idx]] * dq_scale[dq_s1_idx] + dq_scale[dq_offset_idx];+ dst0 = ref::f4_dequantize<F4_T>(tmp.x, scale0);+ dst1 = ref::f4_dequantize<F4_T>(tmp.y, scale1);+ dstptr[i * ld_dst + j + 0] = static_cast<_DST_T>(dst0);+ dstptr[i * ld_dst + j + 1] = static_cast<_DST_T>(dst1);+ }+ }+ return BTLA_CODE::Success;+}++template <BTLA_DTYPE F4_T, typename _DST_T>+static inline BTLA_CODE decompress_kblock_f4_fp_noscale(utils::f4x2* srcptr, _DST_T* dstptr, int row, int col,+ int ld_src, int ld_dst, int8_t* tmp, size_t tmpsize) {+ for (int i = 0; i < row; i++) {+ for (int j = 0; j < col; j += 2) {+ auto tmp = srcptr[i * ld_src / 2 + j / 2];+ dstptr[i * ld_dst + j + 0] = static_cast<_DST_T>(f4_unpack<F4_T>(tmp.x));+ dstptr[i * ld_dst + j + 1] = static_cast<_DST_T>(f4_unpack<F4_T>(tmp.y));+ }+ }+ return BTLA_CODE::Success;+}++template <typename _DST_T>+static inline BTLA_CODE decompress_kblock_f8_fp_noscale(utils::f8* srcptr, _DST_T* dstptr, int row, int col, int ld_src,+ int ld_dst, BTLA_DTYPE src_f8_t) {+ for (int i = 0; i < row; i++) {+ for (int j = 0; j < col; j++) {+ dstptr[i * ld_dst + j] = f8_to_fp32(srcptr[i * ld_src + j], src_f8_t);+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE memcpy2d_dw2highw(const void* srcptr, void* dstptr, int row, int col, int srcstride,+ int dststride) {+ auto bsrcptr = (char*)srcptr;+ auto bdstptr = (char*)dstptr;+ for (int i = 0; i < row; i++) {+ for (int j = 0; j < col; j++) {+ std::memcpy(bdstptr + i * dststride + j * sizeof(utils::bf16), bsrcptr + i * srcstride + j * sizeof(float) + 2,+ sizeof(utils::bf16));+ }+ }+ return BTLA_CODE::Success;+}++template <typename _SRC_T, typename _DST_T>+static inline BTLA_CODE memcpy2d(const _SRC_T* srcptr, _DST_T* dstptr, int row, int col, int srcstride, int dststride) {+ auto bsrcptr = (const char*)srcptr;+ auto bdstptr = (char*)dstptr;+ for (int i = 0; i < row; i++) {+ if constexpr (std::is_same_v<_SRC_T, _DST_T>) {+ std::memcpy(bdstptr + i * dststride, bsrcptr + i * srcstride, col);+ } else if constexpr (std::is_same_v<_SRC_T, float> &&+ (std::is_same_v<_DST_T, utils::bf16> || std::is_same_v<_DST_T, utils::fp16>)) {+ for (int j = 0; j < col; j += sizeof(_SRC_T))+ dstptr[(i * dststride + j / 2) / sizeof(_DST_T)] =+ static_cast<_DST_T>(srcptr[(i * srcstride + j) / sizeof(_SRC_T)]);+ } else if constexpr ((std::is_same_v<_SRC_T, utils::bf16> || std::is_same_v<_SRC_T, utils::fp16>) &&+ std::is_same_v<_DST_T, float>) {+ for (int j = 0; j < col; j += sizeof(_SRC_T))+ dstptr[(i * dststride + j * 2) / sizeof(_DST_T)] =+ static_cast<_DST_T>(srcptr[(i * srcstride + j) / sizeof(_SRC_T)]);+ } else {+ assert(0);+ }+ }+ return BTLA_CODE::Success;+}++template <BTLA_ELTWISEOP OP_T>+static inline float postop(float x, void* const_elt_v) {+ if constexpr (OP_T == BTLA_ELTWISEOP::GELU) {+ return 0.5f * x * (1.f + tanhf(0.7978845834732056f * (x + 0.044714998453855515f * x * x * x)));+ }+ if constexpr (OP_T == BTLA_ELTWISEOP::SWISH) {+ return x / (1 + exp(-x));+ }+ assert(0);+ return std::numeric_limits<float>::infinity();+}++template <typename _SRC_T, typename _DST_T, BTLA_ELTWISEOP OP_T>+static inline BTLA_CODE memcpy2d_withop(const _SRC_T* srcptr, _DST_T* dstptr, int row, int col, int srcstep,+ int dststep, void* const_elt_v) {+ for (int i = 0; i < row; i++) {+ for (int j = 0; j < col; j += 1) {+ float v = srcptr[i * srcstep + j];+ v = postop<OP_T>(v, const_elt_v);+ dstptr[i * dststep + j] = v;+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE get2d_e8m0_scale(const void* srcptr, void* dstptr, int row, int col, int srcstride,+ int dststride) {+ auto f8_v = (const utils::f8*)srcptr;+ auto f32_v = (float*)dstptr;+ auto f8_stride = srcstride / sizeof(utils::f8);+ auto f32_stride = dststride / sizeof(float);+ auto col_elt = col / sizeof(utils::f8);+ for (int i = 0; i < row; i++) {+ for (int j = 0; j < col_elt; j++) {+ f32_v[i * f32_stride + j] = static_cast<float>(std::pow(2, f8_v[i * f8_stride + j].x));+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE quantize_f32_sign_int_rowblock(const float* srcptr, int8_t* dstptr, int row, int col,+ int ld_src, int ld_dst, float* scales, int8_t* zero_points,+ int blocksize, BTLA_DTYPE qtype) {+ int raw_blocksize = blocksize;+ auto NBits = utils::bestla_dtype_bits(qtype);+ int FullValue = 1 << (NBits - 1);+ int SymValue = FullValue - 1;+ auto clip = [&](int s) {+ s = std::max(s, -FullValue);+ s = std::min(s, SymValue);+ return s;+ };+ for (int i = 0; i < col; i++) {+ int align_row_loop = row / blocksize * blocksize;+ int j = 0;+ auto s8_calc_store_scale_and_quantv_sym = [&](int blocksize) {+ float maxval = std::numeric_limits<float>::min();+ for (size_t ij = 0; ij < blocksize; ij++) {+ maxval = std::max(maxval, std::abs(srcptr[(j + ij) * ld_src + i]));+ }+ float scale = maxval / 127;+ float rscale = 1.f / scale;+ scales[j / raw_blocksize * ld_dst + i] = scale;+ for (size_t ij = 0; ij < blocksize; ij++) {+ dstptr[(j + ij) * ld_dst + i] = utils::cast<float, int8_t>(srcptr[(j + ij) * ld_src + i] * rscale);+ }+ };+ auto s8_calc_store_scale_and_quantv_asym = [&](int blocksize) {+ float maxval = 0.f;+ float minval = 0.f;+ for (size_t ij = 0; ij < blocksize; ij++) {+ maxval = std::max(maxval, srcptr[(j + ij) * ld_src + i]);+ minval = std::min(minval, srcptr[(j + ij) * ld_src + i]);+ }+ float scale = (maxval - minval) / 255;+ float rscale = 1.f / scale;+ scales[j / raw_blocksize * ld_dst + i] = scale;+ int8_t bzp = clip(utils::cast<float, int>((0 - minval) * rscale) - 128);+ zero_points[j / raw_blocksize * ld_dst + i] = bzp;+ for (size_t ij = 0; ij < blocksize; ij++) {+ dstptr[(j + ij) * ld_dst + i] = clip(utils::cast<float, int>((srcptr[(j + ij) * ld_src + i]) * rscale) + bzp);+ }+ };+ auto sNauto_calc_store_scale_and_quantv_sym = [&](int blocksize) {+ float maxval = std::numeric_limits<float>::min();+ float minval = std::numeric_limits<float>::max();+ float absmax = 0;+ for (size_t ij = 0; ij < blocksize; ij++) {+ maxval = std::max(maxval, srcptr[(j + ij) * ld_src + i]);+ minval = std::min(minval, srcptr[(j + ij) * ld_src + i]);+ absmax = std::max(absmax, std::abs(srcptr[(j + ij) * ld_src + i]));+ }+ float NVal = SymValue + 0.5f;+ auto sum = maxval + minval;+ if (abs(sum) >= absmax / FullValue) {+ NVal = sum > 0.f ? static_cast<float>(-FullValue) : static_cast<float>(FullValue);+ }+ float scale = absmax / NVal;+ float rscale = 1.f / scale;+ scales[j / raw_blocksize * ld_dst + i] = scale;+ for (size_t ij = 0; ij < blocksize; ij++) {+ dstptr[(j + ij) * ld_dst + i] = clip(utils::cast<float, int8_t>(srcptr[(j + ij) * ld_src + i] * rscale));+ }+ };++ auto sNauto_calc_store_scale_and_quantv_asym = [&](int blocksize) {+ float maxval = 0.f;+ float minval = 0.f;+ for (size_t ij = 0; ij < blocksize; ij++) {+ maxval = std::max(maxval, srcptr[(j + ij) * ld_src + i]);+ minval = std::min(minval, srcptr[(j + ij) * ld_src + i]);+ }+ float scale = (maxval - minval) / ((1 << NBits) - 1);+ float rscale = 1.f / scale;+ scales[j / raw_blocksize * ld_dst + i] = scale;+ int bzp = utils::cast<float, int>((0 - minval) * rscale) - FullValue;++ bzp = clip(bzp);+ zero_points[j / raw_blocksize * ld_dst + i] = static_cast<int8_t>(bzp);+ for (size_t ij = 0; ij < blocksize; ij++) {+ auto tmp = utils::cast<float, int>((srcptr[(j + ij) * ld_src + i]) * rscale) + bzp;+ tmp = clip(tmp);+ dstptr[(j + ij) * ld_dst + i] = tmp;+ }+ };+ auto dispatch_calc = [&](int blocksize) {+ switch (qtype) {+ case BTLA_DTYPE::S8:+ case BTLA_DTYPE::S2_CLIP:+ case BTLA_DTYPE::S1_CLIP:+ case BTLA_DTYPE::S3_CLIP:+ case BTLA_DTYPE::S4_CLIP:+ case BTLA_DTYPE::S5_CLIP:+ case BTLA_DTYPE::S6_CLIP:+ case BTLA_DTYPE::S7_CLIP:+ if (zero_points == nullptr) {+ sNauto_calc_store_scale_and_quantv_sym(blocksize);+ } else {+ sNauto_calc_store_scale_and_quantv_asym(blocksize);+ }+ break;+ default:+ assert(false);+ break;+ }+ };++ for (; j < align_row_loop; j += blocksize) dispatch_calc(blocksize);+ if (j < row) dispatch_calc(row - align_row_loop);+ }+ return BTLA_CODE::Success;+}++template <BTLA_DTYPE F8_T>+static inline int8_t f8_mx_quantize(float v, float scale, BTLA_DTYPE scale_dtype) {+ if (scale_dtype == BTLA_DTYPE::F8_E8M0) {+ v /= static_cast<float>(std::pow(2, scale));+ } else {+ v /= scale;+ }+ auto ebits = utils::bestla_dtype_get_f8_ebits(F8_T);+ auto quant_mantissa = utils::bestla_dtype_get_f8_quant_mbits(F8_T);+ auto store_mantissa = 7 - ebits;+ auto private_exp = std::floor(std::log2(std::abs(v == 0 ? v + 1 : v)));+ auto min_exp = static_cast<float>(-1 * (std::pow(2, ebits - 1)) + 2);+ private_exp = private_exp < min_exp ? min_exp : private_exp;++ // Scale up so appropriate number of bits are in the integer portion of the+ // number+ v = static_cast<float>(v / std::pow(2, private_exp) * std::pow(2, quant_mantissa - 2));+ auto sign = v > 0 ? 1 : -1;+ v = sign * static_cast<float>(std::floor(std::abs(v) + 0.5));+ // Undo scaling+ v = static_cast<float>(v / std::pow(2, quant_mantissa - 2) * std::pow(2, private_exp));++ // saturate normals.+ auto max_norm = utils::get_mxfp_maxnorm(F8_T, ebits, quant_mantissa);+ v = std::clamp(v, -1 * max_norm, max_norm);+ uint32_t* shift_v = reinterpret_cast<uint32_t*>(&v);+ // get sign;+ char* p = reinterpret_cast<char*>(&v);+ uint8_t store_signbit = (*(p + 3) & 0x80);+ *shift_v <<= 1;+ uint8_t store_ebit = (*(p + 3) & 0xFF);+ store_ebit = store_ebit - 127 + static_cast<uint8_t>(std::pow(2, ebits - 1)) - 1;+ if (store_ebit > 15 && F8_T == BTLA_DTYPE::F8_E4M3) store_ebit = 0;+ if (store_ebit > 31 && F8_T == BTLA_DTYPE::F8_E5M2) store_ebit = 0;+ store_ebit <<= store_mantissa;+ *shift_v <<= 8;+ int8_t ox80_shift = -128 >> (store_mantissa - 1);+ uint8_t store_mantissabit = (*(p + 3) & ox80_shift);+ store_mantissabit >>= (1 + ebits);+ auto ret = store_signbit | store_ebit | store_mantissabit;+ return ret;+}++template <BTLA_DTYPE F8_T>+static inline BTLA_CODE quantize_f32_f8_rowblock_mxscale(const float* srcptr, int8_t* dstptr, int row, int col,+ int ld_src, int ld_dst, float* scales, int blocksize,+ BTLA_DTYPE scale_dtype) {+ for (int i = 0; i < col; i++) {+ int align_row_loop = row / blocksize * blocksize;+ int j = 0;+ auto f8_blk_quant = [&](int blksize) {+ float scale = std::numeric_limits<float>::min();+ for (size_t ij = 0; ij < blksize; ij++) {+ scale = std::max(scale, std::abs(srcptr[(j + ij) * ld_src + i]));+ }+ if (scale_dtype == BTLA_DTYPE::F8_E8M0) {+ if (scale == 0) scale += std::abs(std::numeric_limits<float>::min());+ scale = std::floor(std::log2(scale));+ auto ebits = utils::bestla_dtype_get_f8_ebits(F8_T);+ auto emax = static_cast<float>(std::pow(2, ebits - 1));+ if (F8_T == BTLA_DTYPE::F8_E5M2) emax -= 1;+ scale -= emax;+ auto scale_max = static_cast<float>(std::pow(2, 7)) - 1; // e8m0 scale type.+ scale = scale < (-1 * scale_max) ? (-1 * scale_max) : scale;+ } else if (scale_dtype == BTLA_DTYPE::F32) {+ scale /= utils::get_mxfp_maxnorm(F8_T, utils::bestla_dtype_get_f8_ebits(F8_T),+ utils::bestla_dtype_get_f8_quant_mbits(F8_T));+ } else {+ assert(0);+ }+ scales[j / blocksize * ld_dst + i] = scale;+ for (size_t ij = 0; ij < blksize; ij++) {+ dstptr[(j + ij) * ld_dst + i] = f8_mx_quantize<F8_T>(srcptr[(j + ij) * ld_src + i], scale, scale_dtype);+ }+ };+ for (; j < align_row_loop; j += blocksize) f8_blk_quant(blocksize);+ if (j < row) f8_blk_quant(row - align_row_loop);+ }+ return BTLA_CODE::Success;+}++template <BTLA_DTYPE F4_T>+static inline BTLA_CODE quantize_f32_f4_rowblock(const float* srcptr, int8_t* dstptr, int row, int col, int ld_src,+ int ld_dst, float* scales, int blocksize) {+ int raw_blocksize = blocksize;+ for (int i = 0; i < col; i++) {+ int align_row_loop = row / blocksize * blocksize;+ int j = 0;+ auto calc_store_scale_and_quantv_sym = [&](int blocksize) {+ float absmax = std::numeric_limits<float>::min();+ for (size_t ij = 0; ij < blocksize; ij++) {+ absmax = std::max(absmax, std::abs(srcptr[(j + ij) * ld_src + i]));+ }+ scales[j / raw_blocksize * ld_dst + i] = absmax;+ for (size_t ij = 0; ij < blocksize; ij++) {+ dstptr[(j + ij) * ld_dst + i] = f4_quantize<F4_T>(srcptr[(j + ij) * ld_src + i] * (1.f / absmax));+ }+ };+ for (; j < align_row_loop; j += blocksize) calc_store_scale_and_quantv_sym(blocksize);+ if (j < row) calc_store_scale_and_quantv_sym(row - align_row_loop);+ }+ return BTLA_CODE::Success;+}++template <typename SRC_T>+static inline BTLA_CODE quantize_fp_u8_colblock(int row, int col, const SRC_T* srcptr, int ld_src, uint8_t* dstptr,+ int ld_dst, float* scales, int ld_scale, uint8_t* zps, int blocksize,+ float* blkreduce) {+ int colblk = utils::padto_le(col, blocksize);+ for (int i = 0; i < row; i++) {+ size_t j = 0;+ for (; j < colblk; j += blocksize) {+ float maxval = std::numeric_limits<float>::min();+ float minval = 0.f;+ for (size_t ij = 0; ij < blocksize; ij++) {+ auto fsrc = static_cast<float>(srcptr[(j + ij) + i * ld_src]);+ maxval = std::max(fsrc, maxval);+ minval = std::min(fsrc, minval);+ }+ float scale = (maxval - minval) / 255;+ uint8_t zp = utils::cast<float, uint8_t>((0 - minval) / scale);+ float rscale = 1.f / scale;+ scales[j / blocksize + i * ld_scale] = scale;+ zps[j / blocksize + i * ld_scale] = zp;+ int sum = 0;+ auto zpf = static_cast<float>(zp);+ for (size_t ij = 0; ij < blocksize; ij++) {+ auto fsrc = static_cast<float>(srcptr[(j + ij) + i * ld_src]);+ auto qtmp = utils::cast<float, int>(fsrc * rscale);+ sum += qtmp;+ dstptr[(j + ij) + i * ld_dst] = utils::cast<float, uint8_t>(zpf + qtmp);+ }+ if (blkreduce) {+ blkreduce[j / blocksize + i * ld_scale] = sum * scale;+ }+ }+ if (j < col) {+ float maxval = 0.f;+ float minval = 0.f;+ for (size_t ij = j; ij < col; ij++) {+ auto fsrc = static_cast<float>(srcptr[(ij) + i * ld_src]);+ maxval = std::max(fsrc, maxval);+ minval = std::min(fsrc, minval);+ }+ float scale = (maxval - minval) / 255;+ uint8_t zp = utils::cast<float, uint8_t>((0 - minval) / scale);+ float rscale = 1.f / scale;+ scales[j / blocksize + i * ld_scale] = scale;+ zps[j / blocksize + i * ld_scale] = zp;+ int sum = 0;+ auto zpf = float(zp);+ for (size_t ij = j; ij < col; ij++) {+ auto fsrc = static_cast<float>(srcptr[(ij) + i * ld_src]);+ auto qtmp = utils::cast<float, int>(fsrc * rscale);+ sum += qtmp;+ dstptr[(ij) + i * ld_dst] = utils::cast<float, uint8_t>(zpf + qtmp);+ }+ if (blkreduce) {+ blkreduce[j / blocksize + i * ld_scale] = sum * scale;+ }+ }+ }+ return BTLA_CODE::Success;+}++template <typename SRC_T>+static inline BTLA_CODE quantize_fp_s8_colblock(int row, int col, const SRC_T* srcptr, int ld_src, int8_t* dstptr,+ int ld_dst, float* scales, int ld_scale, int blocksize, float* reduce) {+ int colblk = utils::padto_le(col, blocksize);+ for (int i = 0; i < row; i++) {+ size_t j = 0;+ for (; j < colblk; j += blocksize) {+ float absmaxval = std::numeric_limits<float>::min();+ for (size_t ij = 0; ij < blocksize; ij++) {+ auto fsrc = static_cast<float>(srcptr[(j + ij) + i * ld_src]);+ absmaxval = std::max(std::abs(fsrc), absmaxval);+ }+ float scale = absmaxval / 127;+ float rscale = 1.f / scale;+ int sum = 0;+ scales[j / blocksize + i * ld_scale] = scale;+ for (size_t ij = 0; ij < blocksize; ij++) {+ auto fsrc = static_cast<float>(srcptr[(j + ij) + i * ld_src]);+ auto tmp = utils::cast<float, int8_t>(fsrc * rscale);+ dstptr[(j + ij) + i * ld_dst] = tmp;+ sum += tmp;+ }+ if (reduce) reduce[j / blocksize + i * ld_scale] = sum * scale;+ }+ if (j < col) {+ float absmaxval = std::numeric_limits<float>::min();+ for (size_t ij = j; ij < col; ij++) {+ auto fsrc = static_cast<float>(srcptr[(ij) + i * ld_src]);+ absmaxval = std::max(std::abs(fsrc), absmaxval);+ }+ float scale = absmaxval / 127;+ float rscale = 1.f / scale;+ scales[j / blocksize + i * ld_scale] = scale;+ int sum = 0;+ for (size_t ij = j; ij < col; ij++) {+ auto fsrc = static_cast<float>(srcptr[(ij) + i * ld_src]);+ dstptr[(ij) + i * ld_dst] = utils::cast<float, int8_t>(fsrc * rscale);+ sum += dstptr[(ij) + i * ld_dst];+ }+ if (reduce) reduce[j / blocksize + i * ld_scale] = sum * scale;+ }+ }+ return BTLA_CODE::Success;+}++static inline uint8_t get_dq8_bnb(float v) {+ int left = 0;+ int right = 255;+ while (left <= right) {+ int mid = left + (right - left) / 2;+ if (dq8_bnb_LUT[mid] == v) {+ return mid;+ } else if (dq8_bnb_LUT[mid] < v) {+ left = mid + 1;+ } else {+ right = mid - 1;+ }+ }+ if (right < 0) {+ return 0;+ } else if (left >= 256) {+ return 255;+ } else {+ return (v - dq8_bnb_LUT[right] < dq8_bnb_LUT[left] - v) ? right : left;+ }+}+template <bool QDQ_SCALE>+static inline BTLA_CODE dq8_bnb_double_quant(float* scale, size_t scale_size, int dq_blocksize, float* dq_buf) {+ float offset = 0.f;+ for (size_t i = 0; i < scale_size; i++) offset += scale[i];+ offset /= scale_size;+ dq_buf[utils::updiv(scale_size, dq_blocksize)] = offset; // store zp.+ size_t align_blk_size = scale_size / dq_blocksize * dq_blocksize;+ size_t i = 0;+ auto calc_scale = [&](size_t blksize) {+ float absmax = std::numeric_limits<float>::min();+ for (size_t j = 0; j < blksize; j++) {+ scale[i + j] -= offset;+ absmax = std::max(absmax, std::abs(scale[i + j]));+ }+ for (size_t j = 0; j < blksize; j++) {+ scale[i + j] /= absmax;+ scale[i + j] = get_dq8_bnb(scale[i + j]);+ if constexpr (QDQ_SCALE) {+ scale[i + j] = dq8_bnb_LUT[static_cast<int>(scale[i + j])];+ scale[i + j] *= absmax;+ scale[i + j] += offset;+ }+ }+ return absmax;+ };+ for (; i < align_blk_size; i += dq_blocksize) dq_buf[i / dq_blocksize] = calc_scale(dq_blocksize);+ if (i < scale_size) dq_buf[i / dq_blocksize + 1] = calc_scale(scale_size - i);+ return BTLA_CODE::Success;+}++static inline BTLA_CODE dq8_get_fp_scale(uint8_t* src, float* dst, int row, int col, int scale_offset, int dq_blk,+ int dq_offset_idx, float* dq_scale, int src_stride, int dst_stride,+ bool zeropadding, int mN) {+ for (int i = 0; i < row; i++) {+ for (int j = 0; j < col; j++) {+ auto dq_s_idx = (i * mN + scale_offset + j) / dq_blk;+ dst[i * dst_stride + j] = dq8_bnb_LUT[src[i * src_stride + j]] * dq_scale[dq_s_idx] + dq_scale[dq_offset_idx];+ }+ }+ if (zeropadding) assert(0);+ return BTLA_CODE::Success;+}++static inline BTLA_CODE alphabeta_f32_f32(const float alpha, const float* srcptr, const int srcstep, const float beta,+ const float* src1ptr, const int src1step, float* dstptr, const int dststep,+ const int M, const int N) {+ if (beta != 0.f) {+ for (int i = 0; i < M; i++) {+ for (int j = 0; j < N; j++) {+ dstptr[i * dststep + j] = alpha * srcptr[i * srcstep + j] + beta * src1ptr[i * src1step + j];+ }+ }+ return BTLA_CODE::Success;+ }+ for (int i = 0; i < M; i++) {+ for (int j = 0; j < N; j++) {+ dstptr[i * dststep + j] = alpha * srcptr[i * srcstep + j];+ }+ }+ return BTLA_CODE::Success;+}+template <typename SCA_T>+static inline BTLA_CODE accum_alphaN_f32_f32(const SCA_T* alpha, const float* srcptr, const int srcstep, float* dstptr,+ const int dststep, const int M, const int N) {+ for (size_t i = 0; i < M; i++) {+ for (size_t j = 0; j < N; j++) {+ if constexpr (!std::is_same_v<SCA_T, utils::f8>) {+ dstptr[i * dststep + j] = static_cast<float>(alpha[j]) * srcptr[i * srcstep + j] + dstptr[i * dststep + j];+ } else {+ dstptr[i * dststep + j] =+ std::pow(2, alpha[j].x) * srcptr[i * srcstep + j] + dstptr[i * dststep + j]; // e8m0 scale.+ }+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE accum_f32_f32(const float* srcptr, const int srcstep, float* dstptr, const int dststep,+ const int M, const int N) {+ for (size_t i = 0; i < M; i++) {+ for (size_t j = 0; j < N; j++) {+ dstptr[i * dststep + j] = srcptr[i * srcstep + j] + dstptr[i * dststep + j];+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE quanout_s32_u32(const float alpha, const int32_t* srcptr, const int srcstep, uint8_t* dstptr,+ const int dststep, const int M, const int N, float scaleSrc, float scaleDst,+ int zpDst) {+ float factor = alpha * scaleSrc / scaleDst;+ for (int i = 0; i < M; i++) {+ for (int j = 0; j < N; j++) {+ float fsrc = static_cast<float>(srcptr[i * srcstep + j]) * factor;+ dstptr[i * dststep + j] = utils::cast<float, uint8_t>(fsrc + static_cast<float>(zpDst));+ }+ }+ return BTLA_CODE::Success;+}++template <typename SCAB_T>+static inline BTLA_CODE dequant_s32_fp32(const int32_t* srcptr, const int srcstep, float* dstptr, const int dststep,+ const int M, const int N, const float* scaleA, const int ldsa,+ const SCAB_T* scaleB) {+ for (int i = 0; i < M; i++) {+ float scale = scaleA[i * ldsa];+ for (int j = 0; j < N; j++) {+ float fsrc = static_cast<float>(srcptr[i * srcstep + j]) * static_cast<float>(scaleB[j]) * scale;+ dstptr[i * dststep + j] = fsrc;+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE minmax_f32_kblock(const float* srcptr, int row, int col, int ld_src, float* minmaxptr,+ int ld_minmax, int fsize_minmax, int blocksize) {+ for (int i = 0; i < row; i++) {+ if (col >= blocksize) {+ for (int icol = 0; icol < col; icol += blocksize) {+ float maxval = std::numeric_limits<float>::min();+ float minval = std::numeric_limits<float>::max();+ for (int ii = 0; ii < blocksize; ii++) {+ maxval = std::max(srcptr[i * ld_src + icol + ii], maxval);+ minval = std::min(srcptr[i * ld_src + icol + ii], minval);+ }+ auto colptr = &minmaxptr[i * ld_minmax + icol / blocksize * fsize_minmax];+ colptr[0] = minval;+ colptr[1] = maxval;+ }+ } else {+ float maxval = std::numeric_limits<float>::min();+ float minval = std::numeric_limits<float>::max();+ for (int icol = 0; icol < col; icol++) {+ maxval = std::max(srcptr[i * ld_src + icol], maxval);+ minval = std::min(srcptr[i * ld_src + icol], minval);+ }+ minmaxptr[i * ld_minmax + 0] = minval;+ minmaxptr[i * ld_minmax + 1] = maxval;+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE accumulate_dequantize_s32_f32(const int32_t* srcptr, float* dstptr, float alpha, float beta,+ int row, int col, int ld_src, int ld_dst, float* ascales,+ int ldas, float* wscales) {+ for (int irow = 0; irow < row; irow++) {+ for (int icol = 0; icol < col; icol++) {+ float scale = ascales[irow * ldas] * wscales[icol] * alpha;+ dstptr[irow * ld_dst + icol] = scale * srcptr[irow * ld_src + icol] + beta * dstptr[irow * ld_dst + icol];+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE broadcast_u8(int num, const uint8_t& srcval, uint8_t* dstptr) {+ int i = 0;+ for (; i < num; i++) {+ dstptr[i] = srcval;+ }+ return BTLA_CODE::Success;+}++template <typename _RT>+static inline BTLA_CODE quant_s8_row_reduce_sum(const int8_t* srcptr, int ldsrc, const float* scales,+ const int8_t* zero_points, int row, int col, _RT* reduce) {+ std::memset(reduce, 0, sizeof(reduce[0]) * col);+ for (int i = 0; i < row; i++) {+ for (int j = 0; j < col; j++) {+ if (zero_points != nullptr) {+ reduce[j] += static_cast<_RT>((static_cast<float>(srcptr[i * ldsrc + j]) - static_cast<float>(zero_points[j])) *+ static_cast<float>(scales[j]));+ } else {+ reduce[j] += static_cast<_RT>(srcptr[i * ldsrc + j] * scales[j]);+ }+ }+ }+ return BTLA_CODE::Success;+}++template <typename _SRCT, typename _DSTT>+static inline BTLA_CODE row_reduce_sum(const _SRCT* srcptr, int ldsrc, int row, int col, _DSTT* reduce) {+ for (int j = 0; j < col; j++) {+ _DSTT tmp = 0;+ for (int i = 0; i < row; i++) {+ tmp += static_cast<_DSTT>(srcptr[i * ldsrc + j]);+ }+ reduce[j] = tmp;+ }+ return BTLA_CODE::Success;+}++template <typename SRC_T>+static inline BTLA_CODE col_block_reduce_sum(const SRC_T* srcptr, int ldsrc, int row, int col, int blocksize,+ float* reduce, int ldr) {+ for (int i = 0; i < row; i++) {+ for (int j = 0; j < col; j += blocksize) {+ auto tmp = 0.f;+ for (size_t jj = 0; jj < blocksize; jj++) {+ if (j + jj < col) {+ tmp += srcptr[i * ldsrc + j + jj];+ }+ }+ reduce[i * ldr + j / blocksize] = tmp;+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE remove_act_zeropoint_bias(float* accptr, int ldacc, int row, int col, uint8_t* zps,+ float* scales, int lds, const float* reduce) {+ for (int i = 0; i < row; i++) {+ auto zpf = static_cast<float>(zps[i * lds]) * scales[i * lds];+ for (int j = 0; j < col; j++) {+ accptr[i * ldacc + j] -= zpf * reduce[j];+ }+ }+ return BTLA_CODE::Success;+}++static inline BTLA_CODE remove_wei_zeropoint_bias(float* accptr, int ldacc, int row, int col, int8_t* zps,+ float* scales, int lds, const float* reduce) {+ for (int i = 0; i < row; i++) {+ auto reducef = reduce[i * lds];+ for (int j = 0; j < col; j++) {+ accptr[i * ldacc + j] -= static_cast<float>(zps[j]) * scales[j] * reducef;+ }+ }+ return BTLA_CODE::Success;+}++template <bool ZPA, bool ZPB>+static inline BTLA_CODE remove_zeropoint_bias(int32_t* accptr, int ldacc, int row, int col, uint8_t* zpa, int8_t* zpb,+ int lda, int k, const int32_t* reducea, const int32_t* reduceb) {+ for (int i = 0; i < row; i++) {+ int32_t vreducea = ZPB ? reducea[i * lda] : 0;+ uint8_t vzpa = ZPA ? zpa[i * lda] : 0;+ for (int j = 0; j < col; j++) {+ auto vzpb = ZPB ? zpb[j] : 0;+ accptr[i * ldacc + j] -= ZPB ? vzpb * vreducea : 0;+ accptr[i * ldacc + j] -= ZPA ? vzpa * reduceb[j] : 0;+ accptr[i * ldacc + j] -= ZPB && ZPA ? vzpa * vzpb * k : 0;+ }+ }+ return BTLA_CODE::Success;+}++template <typename T>+static inline BTLA_CODE layernorm(const T* srcptr, const T* scaleptr, const T* biasptr, T epsilon, int norm_size,+ T* dstptr, T* mean_out, T* mean_square_out, bool simplified) {+ T mean = 0;+ T mean_square = 0;++ for (int h = 0; h < norm_size; h++) {+ mean += srcptr[h];+ mean_square += srcptr[h] * srcptr[h];+ }++ mean = mean / norm_size;+ if (simplified) {+ mean_square = std::sqrt(mean_square / norm_size + epsilon);+ } else {+ mean_square = std::sqrt(mean_square / norm_size - mean * mean + epsilon);+ }+ float inv_mean_square = 1.f / mean_square;+ if (simplified) {+ if (scaleptr) {+ for (int h = 0; h < norm_size; h++) {+ dstptr[h] = srcptr[h] * inv_mean_square * scaleptr[h];+ }+ } else {+ for (int h = 0; h < norm_size; h++) {+ dstptr[h] = srcptr[h] * inv_mean_square;+ }+ }+ } else {+ if (scaleptr) {+ if (biasptr == nullptr) {+ for (int h = 0; h < norm_size; h++) {+ dstptr[h] = (srcptr[h] - mean) * inv_mean_square * scaleptr[h];+ }+ } else {+ for (int h = 0; h < norm_size; h++) {+ dstptr[h] = (srcptr[h] - mean) * inv_mean_square * scaleptr[h] + biasptr[h];+ }+ }+ } else {+ for (int h = 0; h < norm_size; h++) {+ dstptr[h] = (srcptr[h] - mean) * inv_mean_square;+ }+ }+ }++ if (mean_out) {+ *mean_out = mean;+ }+ if (mean_square_out) {+ *mean_square_out = mean_square;+ }+ return BTLA_CODE::Success;+}++inline float exp_ps_0_1(float x) {+ static const float log2e = std::log2(std::exp(1.f));+ static const float ln2 = std::log(2.f);+ const float x1 = x * log2e + .5f;+ const float z = std::floor(x1);+ const float f = x1 - z;+ constexpr std::array<float, 3> coeff{0.240226507f, 0.452920674f, 0.713483036f};+ // same as a * std::pow(2, z) but more precise+ return ldexpf(coeff[0] * f * f + coeff[1] * f + coeff[2], static_cast<int>(z));+}++template <BTLA_DTYPE S3_T, typename _DST_T>+static inline BTLA_CODE decompress_kblock_s3_s8fp(utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr, _DST_T* dstptr,+ int interleave_n_offset, int unpack_elt, int8_t* tmp,+ size_t tmpsize) {+ auto head_ignore_num = interleave_n_offset % 128;+ auto bit3_interleave_decompress_pack128 = [&](utils::bit2x4* src1, utils::bit1x8* src2, int8_t* dst) {+ auto b2ptr = reinterpret_cast<uint8_t*>(src1);+ for (size_t i = 0; i < 128; i += 8) {+ auto bit1off = i >> 3;+ auto bit2off = (i >> 5) << 1;+ auto byteoff = i % 32;+ uint8_t bit1 = *(uint8_t*)(src2 + bit1off);+ for (size_t j = 0; j < 8; j++) {+ uint8_t bit2 = *(b2ptr + byteoff + j);+ bit2 >>= bit2off;+ uint8_t dst8 = ((bit2 & 0x3)) | ((bit1 & 0x1) << 2);+ bit1 >>= 1;+ dst[i + j] = (*(int8_t*)&dst8) - 4;+ }+ }+ };++ assert(head_ignore_num % 8 == 0);++ auto base_bit2ptr = bit2ptr - head_ignore_num / 4;+ auto base_bit1ptr = bit1ptr - head_ignore_num / 8;+ int compress_wei_ptr_offset = 0;+ int8_t* s8_ptr = reinterpret_cast<int8_t*>(tmp);+ auto head_write_num = 128 - head_ignore_num;+ if (head_ignore_num != 0) {+ bit3_interleave_decompress_pack128(base_bit2ptr, base_bit1ptr, tmp);+ for (int i = 0; i < head_write_num; i++) dstptr[i] = s8_ptr[head_ignore_num + i];+ compress_wei_ptr_offset += head_write_num;+ }++ auto body_loop = (unpack_elt - head_write_num % 128) / 128;+ auto tail_proc_num = (unpack_elt - head_write_num % 128) % 128;+ for (size_t i = 0; i < body_loop; i++) {+ bit3_interleave_decompress_pack128(bit2ptr + compress_wei_ptr_offset / 4 + i * 32,+ bit1ptr + compress_wei_ptr_offset / 8 + i * 16, tmp);+ for (int j = 0; j < 128; j++) dstptr[compress_wei_ptr_offset + i * 128 + j] = tmp[j];+ }+ compress_wei_ptr_offset += body_loop * 128;+ if (tail_proc_num > 0) {+ bit3_interleave_decompress_pack128(base_bit2ptr, base_bit1ptr, tmp);+ bit3_interleave_decompress_pack128(bit2ptr + compress_wei_ptr_offset / 4, bit1ptr + compress_wei_ptr_offset / 8,+ tmp);+ for (int i = 0; i < tail_proc_num; i++) dstptr[compress_wei_ptr_offset + i] = s8_ptr[i];+ }+ return BTLA_CODE::Success;+}++template <BTLA_DTYPE _S3_T, typename _DST_T, int _PACK_ROW, typename _ST>+static inline BTLA_CODE decompress_kblock_bit3_packrow_fp(utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr,+ _DST_T* dstptr, int interleave_n_offset, int row, int col,+ _ST* scales, int8_t* zero_points, int k_offset, int kblock,+ int NPad, void* tmp, size_t tmpsize) {+ auto unpack_elt = row * col;+ decompress_kblock_s3_s8fp<_S3_T>(bit2ptr, bit1ptr, dstptr, interleave_n_offset, unpack_elt,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ // TODO(zhe): simd version+ for (int i = 0; i < row; i++) {+ int kpos = (k_offset + i) / kblock;+ auto sptr = scales + kpos * NPad;+ for (int j = 0; j < col; j++) {+ float tmp = static_cast<float>(dstptr[i * col + j]);+ if (zero_points != nullptr) tmp -= static_cast<float>(zero_points[kpos * NPad + j / _PACK_ROW]);+ dstptr[i * col + j] = static_cast<_DST_T>(tmp * sptr[j / _PACK_ROW]);+ }+ }++ return BTLA_CODE::Success;+}++template <BTLA_DTYPE S2_T, typename _DST_T>+static inline BTLA_CODE decompress_kblock_s2_s8fp(utils::bit2x4* bit2ptr, _DST_T* dstptr, size_t unpack_elt,+ int8_t* tmp, size_t tmpsize) {+ for (size_t i = 0; i < unpack_elt; i += 4) {+ auto tmp = bit2ptr[i / 4];+ dstptr[i + 0] = _DST_T(tmp.a - 2);+ dstptr[i + 1] = _DST_T(tmp.b - 2);+ dstptr[i + 2] = _DST_T(tmp.c - 2);+ dstptr[i + 3] = _DST_T(tmp.d - 2);+ }+ return BTLA_CODE::Success;+}++template <BTLA_DTYPE _S2_T, typename _DST_T, int _PACK_ROW, typename _ST>+static inline BTLA_CODE decompress_kblock_bit2_packrow_fp(utils::bit2x4* bit2ptr, _DST_T* dstptr, int row, int col,+ _ST* scales, int8_t* zero_points, int k_offset, int kblock,+ int NPad, void* tmp, size_t tmpsize) {+ auto unpack_elt = row * col;+ decompress_kblock_s2_s8fp<_S2_T>(bit2ptr, dstptr, unpack_elt, reinterpret_cast<int8_t*>(tmp), tmpsize);+ // TODO(zhe): simd version+ for (int i = 0; i < row; i++) {+ int kpos = (k_offset + i) / kblock;+ auto sptr = scales + kpos * NPad;+ for (int j = 0; j < col; j++) {+ float tmp = static_cast<float>(dstptr[i * col + j]);+ if (zero_points != nullptr) tmp -= static_cast<float>(zero_points[kpos * NPad + j / _PACK_ROW]);+ dstptr[i * col + j] = static_cast<_DST_T>(tmp * sptr[j / _PACK_ROW]);+ }+ }++ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_4bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto a8ptr = A.aptr;+ auto b4ptr = B.b4ptr;+ auto asptr = A.sptr;+ auto azptr = A.zpptr;+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ if (B.zpptr) {+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += 4) {+ for (int im = 0; im < MTILE; im++) {+ int azp = azptr[ib + im * A.ldzp];+ float ascale = asptr[ib + im * A.ldzp];+ for (int in = 0; in < NTILE; in++) {+ auto bv0 = *(utils::int4x2*)(b4ptr + in * 2);+ auto bv1 = *(utils::int4x2*)(b4ptr + in * 2 + 1);+ auto vscale = ascale * (float)bsptr[in];+ int bzp = bzptr[in] + 8;+ accf[im * NTILE + in] += int(a8ptr[0 + im * A.lda] - azp) * (bv0.x - bzp) * vscale;+ accf[im * NTILE + in] += int(a8ptr[1 + im * A.lda] - azp) * (bv0.y - bzp) * vscale;+ accf[im * NTILE + in] += int(a8ptr[2 + im * A.lda] - azp) * (bv1.x - bzp) * vscale;+ accf[im * NTILE + in] += int(a8ptr[3 + im * A.lda] - azp) * (bv1.y - bzp) * vscale;+ }+ }+ a8ptr += 4;+ b4ptr += NTILE * 2;+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += 4) {+ for (int im = 0; im < MTILE; im++) {+ int azp = azptr[ib + im * A.ldzp];+ float ascale = asptr[ib + im * A.ldzp];+ for (int in = 0; in < NTILE; in++) {+ auto bv0 = *(utils::int4x2*)(b4ptr + in * 2);+ auto bv1 = *(utils::int4x2*)(b4ptr + in * 2 + 1);+ auto vscale = ascale * (float)bsptr[in];+ accf[im * NTILE + in] += int(a8ptr[0 + im * A.lda] - azp) * (bv0.x - 8) * vscale;+ accf[im * NTILE + in] += int(a8ptr[1 + im * A.lda] - azp) * (bv0.y - 8) * vscale;+ accf[im * NTILE + in] += int(a8ptr[2 + im * A.lda] - azp) * (bv1.x - 8) * vscale;+ accf[im * NTILE + in] += int(a8ptr[3 + im * A.lda] - azp) * (bv1.y - 8) * vscale;+ }+ }+ a8ptr += 4;+ b4ptr += NTILE * 2;+ }+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_4bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto a8ptr = reinterpret_cast<int8_t*>(A.aptr);+ auto b4ptr = B.b4ptr;+ auto asptr = A.sptr;+ auto azptr = A.zpptr;+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ if (B.zpptr) {+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += 4) {+ for (int im = 0; im < MTILE; im++) {+ float ascale = asptr[ib + im * A.ldzp];+ for (int in = 0; in < NTILE; in++) {+ auto bv0 = *(utils::int4x2*)(b4ptr + in * 2);+ auto bv1 = *(utils::int4x2*)(b4ptr + in * 2 + 1);+ auto vscale = ascale * (float)bsptr[in];+ auto bzp = bzptr[in] + 8;+ accf[im * NTILE + in] += int(a8ptr[0 + im * A.lda]) * (bv0.x - bzp) * vscale;+ accf[im * NTILE + in] += int(a8ptr[1 + im * A.lda]) * (bv0.y - bzp) * vscale;+ accf[im * NTILE + in] += int(a8ptr[2 + im * A.lda]) * (bv1.x - bzp) * vscale;+ accf[im * NTILE + in] += int(a8ptr[3 + im * A.lda]) * (bv1.y - bzp) * vscale;+ }+ }+ a8ptr += 4;+ b4ptr += NTILE * 2;+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += 4) {+ for (int im = 0; im < MTILE; im++) {+ float ascale = asptr[ib + im * A.ldzp];+ for (int in = 0; in < NTILE; in++) {+ auto bv0 = *(utils::int4x2*)(b4ptr + in * 2);+ auto bv1 = *(utils::int4x2*)(b4ptr + in * 2 + 1);+ auto vscale = ascale * (float)bsptr[in];+ accf[im * NTILE + in] += int(a8ptr[0 + im * A.lda]) * (bv0.x - 8) * vscale;+ accf[im * NTILE + in] += int(a8ptr[1 + im * A.lda]) * (bv0.y - 8) * vscale;+ accf[im * NTILE + in] += int(a8ptr[2 + im * A.lda]) * (bv1.x - 8) * vscale;+ accf[im * NTILE + in] += int(a8ptr[3 + im * A.lda]) * (bv1.y - 8) * vscale;+ }+ }+ a8ptr += 4;+ b4ptr += NTILE * 2;+ }+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_4bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto b4ptr = B.b4ptr;+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ if (B.zpptr) {+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += 1) {+ for (int im = 0; im < MTILE; im++) {+ auto aval = A[ib * blocksize + ik + im * lda];+ for (int in = 0; in < NTILE; in += 2) {+ auto bv0 = *(utils::int4x2*)(b4ptr + in / 2);+ accf[im * NTILE + in + 0] += aval * (bv0.x - 8 - bzptr[in + 0]) * (float)bsptr[in + 0];+ accf[im * NTILE + in + 1] += aval * (bv0.y - 8 - bzptr[in + 1]) * (float)bsptr[in + 1];+ }+ }+ b4ptr += NTILE / 2;+ }+ } else {+ for (int ik = 0; ik < blocksize; ik += 1) {+ for (int im = 0; im < MTILE; im++) {+ auto aval = A[ib * blocksize + ik + im * lda];+ for (int in = 0; in < NTILE; in += 2) {+ auto bv0 = *(utils::int4x2*)(b4ptr + in / 2);+ accf[im * NTILE + in + 0] += aval * (bv0.x - 8) * (float)bsptr[in + 0];+ accf[im * NTILE + in + 1] += aval * (bv0.y - 8) * (float)bsptr[in + 1];+ }+ }+ b4ptr += NTILE / 2;+ }+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE>+static inline BTLA_CODE gemv_3bit_u8s8_fp32_align128(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B,+ float* C, int k, int ld_scaleb, int blocksize, int8_t* tmp,+ size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE];+ std::memset(accf, 0, sizeof(accf));+ auto a8ptr = A.aptr;+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);+ auto asptr = A.sptr;+ auto azptr = A.zpptr;+ int constexpr EltPadding = 128;+ static_assert(NTILE % 8 == 0);+ int constexpr KTILE = 4;+ int constexpr UnpackElt = EltPadding / 8 / KTILE;+ int8_t UnpackBuf[UnpackElt * NTILE * KTILE];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * ld_scaleb;+ int acci[NTILE];+ std::memset(acci, 0, sizeof(acci));+ int wacc[NTILE];+ std::memset(wacc, 0, sizeof(wacc));+ for (int ik = 0; ik < blocksize; ik += KTILE * UnpackElt) {+ decompress_kblock_s3_s8fp<BTLA_DTYPE::S3_CLIP, int8_t>(b2ptr, b1ptr, UnpackBuf, ik * NTILE,+ NTILE * KTILE * UnpackElt, tmp, tmpsize);+ for (int iu = 0; iu < UnpackElt; iu++) {+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < KTILE; ikt++) {+ auto bval = UnpackBuf[iu * NTILE * KTILE + in * KTILE + ikt];+ acci[in] += int(a8ptr[iu * KTILE + ikt]) * bval;+ wacc[in] += bval;+ }+ }+ }++ b2ptr += KTILE * UnpackElt * NTILE / 4;+ b1ptr += KTILE * UnpackElt * NTILE / 8;+ a8ptr += KTILE * UnpackElt;+ }+ float scale = asptr[ib];+ int zp = azptr[ib];+ for (int in = 0; in < NTILE; in++) {+ auto tmp = float(acci[in] - zp * wacc[in]);+ tmp = tmp * (scale * (float)bsptr[in]);+ accf[in] += tmp;+ }+ }+ for (int in = 0; in < NTILE; in++) {+ C[in] = accf[in];+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE>+static inline BTLA_CODE gemv_3bit_s8s8_fp32_align128(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B,+ float* C, int k, int ld_scaleb, int blocksize, int8_t* tmp,+ size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE];+ std::memset(accf, 0, sizeof(accf));+ auto a8ptr = reinterpret_cast<int8_t*>(A.aptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);+ auto asptr = A.sptr;+ int constexpr EltPadding = 128;+ static_assert(NTILE % 8 == 0);+ int constexpr KTILE = 4;+ int constexpr UnpackElt = EltPadding / 8 / KTILE;+ int8_t UnpackBuf[UnpackElt * NTILE * KTILE];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * ld_scaleb;+ int acci[NTILE];+ std::memset(acci, 0, sizeof(acci));+ for (int ik = 0; ik < blocksize; ik += KTILE * UnpackElt) {+ decompress_kblock_s3_s8fp<BTLA_DTYPE::S3_CLIP, int8_t>(b2ptr, b1ptr, UnpackBuf, ik * NTILE,+ NTILE * KTILE * UnpackElt, tmp, tmpsize);+ for (int iu = 0; iu < UnpackElt; iu++) {+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < KTILE; ikt++) {+ auto bval = UnpackBuf[iu * NTILE * KTILE + in * KTILE + ikt];+ acci[in] += int(a8ptr[iu * KTILE + ikt]) * bval;+ }+ }+ }+ b2ptr += KTILE * UnpackElt * NTILE / 4;+ b1ptr += KTILE * UnpackElt * NTILE / 8;+ a8ptr += KTILE * UnpackElt;+ }++ float scale = asptr[ib];+ for (int in = 0; in < NTILE; in++) {+ auto tmp = float(acci[in]);+ tmp = tmp * (scale * (float)bsptr[in]);+ accf[in] += tmp;+ }+ }+ for (int in = 0; in < NTILE; in++) {+ C[in] = accf[in];+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_2bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto a8ptr = A.aptr;+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ int constexpr KTILE = 4;+ int8_t UnpackBuf[NTILE * KTILE];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ decompress_kblock_s2_s8<4, NTILE>(b2ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0, 0, KTILE,+ NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ float ascale = A.sptr[ib + im * A.ldzp];+ auto azp = A.zpptr[ib + im * A.ldzp];+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < KTILE; ikt++) {+ auto bval = (UnpackBuf[in * KTILE + ikt]) * (float)bsptr[in];+ auto aval = int(a8ptr[ikt + im * A.lda] - azp) * ascale;+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b2ptr += KTILE * NTILE / 4;+ a8ptr += KTILE;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}+template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_2bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto a8ptr = (int8_t*)A.aptr;+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ int constexpr KTILE = 4;+ int8_t UnpackBuf[NTILE * KTILE];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ decompress_kblock_s2_s8<4, NTILE>(b2ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0, 0, KTILE,+ NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ float ascale = A.sptr[ib + im * A.ldzp];+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < KTILE; ikt++) {+ auto bval = (UnpackBuf[in * KTILE + ikt]) * (float)bsptr[in];+ auto aval = int(a8ptr[ikt + im * A.lda]) * ascale;+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b2ptr += KTILE * NTILE / 4;+ a8ptr += KTILE;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_2bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ int constexpr KTILE = 1;+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);+ int8_t UnpackBuf[NTILE * Unroll];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ decompress_kblock_s2_s8fp<BTLA_DTYPE::S2_CLIP, int8_t>(b2ptr, UnpackBuf, NTILE * Unroll, tmp, tmpsize);+ if (B.zpptr) {+ for (int ikt = 0; ikt < Unroll; ikt++) {+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ auto bval = (UnpackBuf[in + ikt * NTILE] - bzptr[in]) * (float)bsptr[in];+ auto aval = A[ikt + im * lda];+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ } else {+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < Unroll; ikt++) {+ auto bval = (UnpackBuf[in + ikt * NTILE]) * (float)bsptr[in];+ auto aval = A[ikt + im * lda];+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ }+ b2ptr += Unroll * NTILE / 4;+ A += Unroll;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_1bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);+ int constexpr KTILE = 1;+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);+ int8_t UnpackBuf[NTILE * Unroll];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ decompress_kblock_s1_s8<1, NTILE>(b1ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0, 0, Unroll,+ NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < Unroll; ikt++) {+ auto bval = (UnpackBuf[in + ikt * NTILE]) * (float)bsptr[in];+ auto aval = A[ikt + im * lda];+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b1ptr += Unroll * NTILE / 8;+ A += Unroll;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_1bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto a8ptr = A.aptr;+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);+ int constexpr KTILE = 4;+ int8_t UnpackBuf[NTILE * KTILE];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ decompress_kblock_s1_s8<4, NTILE>(b1ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0, 0, KTILE,+ NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ float ascale = A.sptr[ib + im * A.ldzp];+ auto azp = A.zpptr[ib + im * A.ldzp];+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < KTILE; ikt++) {+ auto bval = (UnpackBuf[in * KTILE + ikt]) * (float)bsptr[in];+ auto aval = int(a8ptr[ikt + im * A.lda] - azp) * ascale;+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b1ptr += KTILE * NTILE / 8;+ a8ptr += KTILE;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_1bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto a8ptr = (int8_t*)A.aptr;+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);+ int constexpr KTILE = 4;+ int8_t UnpackBuf[NTILE * KTILE];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ decompress_kblock_s1_s8<4, NTILE>(b1ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0, 0, KTILE,+ NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ float ascale = A.sptr[ib + im * A.ldzp];+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < KTILE; ikt++) {+ auto bval = (UnpackBuf[in * KTILE + ikt]) * (float)bsptr[in];+ auto aval = int(a8ptr[ikt + im * A.lda]) * ascale;+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b1ptr += KTILE * NTILE / 8;+ a8ptr += KTILE;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_3bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);+ int constexpr KTILE = 1;+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);+ int8_t UnpackBuf[NTILE * Unroll];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ decompress_kblock_s3_s8<1, NTILE>(b2ptr, b1ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0, 0,+ Unroll, NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < Unroll; ikt++) {+ auto bval = (UnpackBuf[in + ikt * NTILE]) * (float)bsptr[in];+ auto aval = A[ikt + im * lda];+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b2ptr += Unroll * NTILE / 4;+ b1ptr += Unroll * NTILE / 8;+ A += Unroll;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_3bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto a8ptr = A.aptr;+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);+ int constexpr KTILE = 4;+ int8_t UnpackBuf[NTILE * KTILE];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ decompress_kblock_s3_s8<4, NTILE>(b2ptr, b1ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0, 0,+ KTILE, NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ float ascale = A.sptr[ib + im * A.ldzp];+ auto azp = A.zpptr[ib + im * A.ldzp];+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < KTILE; ikt++) {+ auto bval = (UnpackBuf[in * KTILE + ikt]) * (float)bsptr[in];+ auto aval = int(a8ptr[ikt + im * A.lda] - azp) * ascale;+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b2ptr += KTILE * NTILE / 4;+ b1ptr += KTILE * NTILE / 8;+ a8ptr += KTILE;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_3bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto a8ptr = (int8_t*)A.aptr;+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);+ int constexpr KTILE = 4;+ int8_t UnpackBuf[NTILE * KTILE];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ decompress_kblock_s3_s8<4, NTILE>(b2ptr, b1ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0, 0,+ KTILE, NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ float ascale = A.sptr[ib + im * A.ldzp];+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < KTILE; ikt++) {+ auto bval = (UnpackBuf[in * KTILE + ikt]) * (float)bsptr[in];+ auto aval = int(a8ptr[ikt + im * A.lda]) * ascale;+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b2ptr += KTILE * NTILE / 4;+ b1ptr += KTILE * NTILE / 8;+ a8ptr += KTILE;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_6bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ int constexpr KTILE = 1;+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);+ int8_t UnpackBuf[NTILE * Unroll];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ decompress_kblock_s6_s8<1, NTILE>(b4ptr, b2ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0, 0,+ Unroll, NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < Unroll; ikt++) {+ auto bval = (UnpackBuf[in + ikt * NTILE]) * (float)bsptr[in];+ auto aval = A[ikt + im * lda];+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b4ptr += Unroll * NTILE / 2;+ b2ptr += Unroll * NTILE / 4;+ A += Unroll;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_6bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto a8ptr = A.aptr;+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ int constexpr KTILE = 4;+ int8_t UnpackBuf[NTILE * KTILE];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ decompress_kblock_s6_s8<4, NTILE>(b4ptr, b2ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0, 0,+ KTILE, NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ float ascale = A.sptr[ib + im * A.ldzp];+ auto azp = A.zpptr[ib + im * A.ldzp];+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < KTILE; ikt++) {+ auto bval = (UnpackBuf[in * KTILE + ikt]) * (float)bsptr[in];+ auto aval = int(a8ptr[ikt + im * A.lda] - azp) * ascale;+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b4ptr += KTILE * NTILE / 2;+ b2ptr += KTILE * NTILE / 4;+ a8ptr += KTILE;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_6bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto a8ptr = (int8_t*)A.aptr;+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ int constexpr KTILE = 4;+ int8_t UnpackBuf[NTILE * KTILE];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ decompress_kblock_s6_s8<4, NTILE>(b4ptr, b2ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0, 0,+ KTILE, NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ float ascale = A.sptr[ib + im * A.ldzp];+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < KTILE; ikt++) {+ auto bval = (UnpackBuf[in * KTILE + ikt]) * (float)bsptr[in];+ auto aval = int(a8ptr[ikt + im * A.lda]) * ascale;+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b4ptr += KTILE * NTILE / 2;+ b2ptr += KTILE * NTILE / 4;+ a8ptr += KTILE;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_5bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);+ int constexpr KTILE = 1;+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);+ int8_t UnpackBuf[NTILE * Unroll];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ decompress_kblock_s5_s8<1, NTILE>(b4ptr, b1ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0, 0,+ Unroll, NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < Unroll; ikt++) {+ auto bval = (UnpackBuf[in + ikt * NTILE]) * (float)bsptr[in];+ auto aval = A[ikt + im * lda];+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b4ptr += Unroll * NTILE / 2;+ b1ptr += Unroll * NTILE / 8;+ A += Unroll;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_5bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto a8ptr = A.aptr;+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);+ int constexpr KTILE = 4;+ int8_t UnpackBuf[NTILE * KTILE];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ decompress_kblock_s5_s8<4, NTILE>(b4ptr, b1ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0, 0,+ KTILE, NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ float ascale = A.sptr[ib + im * A.ldzp];+ auto azp = A.zpptr[ib + im * A.ldzp];+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < KTILE; ikt++) {+ auto bval = (UnpackBuf[in * KTILE + ikt]) * (float)bsptr[in];+ auto aval = int(a8ptr[ikt + im * A.lda] - azp) * ascale;+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b4ptr += KTILE * NTILE / 2;+ b1ptr += KTILE * NTILE / 8;+ a8ptr += KTILE;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_5bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto a8ptr = (int8_t*)A.aptr;+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);+ int constexpr KTILE = 4;+ int8_t UnpackBuf[NTILE * KTILE];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ decompress_kblock_s5_s8<4, NTILE>(b4ptr, b1ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0, 0,+ KTILE, NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ float ascale = A.sptr[ib + im * A.ldzp];+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < KTILE; ikt++) {+ auto bval = (UnpackBuf[in * KTILE + ikt]) * (float)bsptr[in];+ auto aval = int(a8ptr[ikt + im * A.lda]) * ascale;+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b4ptr += KTILE * NTILE / 2;+ b1ptr += KTILE * NTILE / 8;+ a8ptr += KTILE;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_7bit_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);+ int constexpr KTILE = 1;+ int constexpr Unroll = 4;+ assert((blocksize % 4) == 0);+ assert(tmpsize >= NTILE * Unroll);+ int8_t UnpackBuf[NTILE * Unroll];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += Unroll) {+ decompress_kblock_s7_s8<1, NTILE>(b4ptr, b2ptr, b1ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0,+ 0, Unroll, NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < Unroll; ikt++) {+ auto bval = (UnpackBuf[in + ikt * NTILE]) * (float)bsptr[in];+ auto aval = A[ikt + im * lda];+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b4ptr += Unroll * NTILE / 2;+ b2ptr += Unroll * NTILE / 4;+ b1ptr += Unroll * NTILE / 8;+ A += Unroll;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_7bit_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto a8ptr = A.aptr;+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);+ int constexpr KTILE = 4;+ int8_t UnpackBuf[NTILE * KTILE];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ decompress_kblock_s7_s8<4, NTILE>(b4ptr, b2ptr, b1ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0,+ 0, KTILE, NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ float ascale = A.sptr[ib + im * A.ldzp];+ auto azp = A.zpptr[ib + im * A.ldzp];+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < KTILE; ikt++) {+ auto bval = (UnpackBuf[in * KTILE + ikt]) * (float)bsptr[in];+ auto aval = int(a8ptr[ikt + im * A.lda] - azp) * ascale;+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b4ptr += KTILE * NTILE / 2;+ b2ptr += KTILE * NTILE / 4;+ b1ptr += KTILE * NTILE / 8;+ a8ptr += KTILE;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename ScaleT, int NTILE, int MTILE>+static inline BTLA_CODE gemv_7bit_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, int8_t* tmp, size_t tmpsize) {+ int blks = k / blocksize;+ float accf[NTILE * MTILE];+ std::memset(accf, 0, sizeof(accf));+ auto a8ptr = (int8_t*)A.aptr;+ auto b4ptr = reinterpret_cast<utils::bit4x2*>(B.b4ptr);+ auto b2ptr = reinterpret_cast<utils::bit2x4*>(B.b2ptr);+ auto b1ptr = reinterpret_cast<utils::bit1x8*>(B.b1ptr);+ int constexpr KTILE = 4;+ int8_t UnpackBuf[NTILE * KTILE];+ for (int ib = 0; ib < blks; ib += 1) {+ auto bsptr = B.sptr + ib * B.ldzp;+ auto bzptr = B.zpptr + ib * B.ldzp;+ for (int ik = 0; ik < blocksize; ik += KTILE) {+ decompress_kblock_s7_s8<4, NTILE>(b4ptr, b2ptr, b1ptr, B.zpptr ? bzptr : nullptr, UnpackBuf, blocksize, B.ldzp, 0,+ 0, KTILE, NTILE, tmp, tmpsize);+ for (int im = 0; im < MTILE; im++) {+ float ascale = A.sptr[ib + im * A.ldzp];+ for (int in = 0; in < NTILE; in++) {+ for (int ikt = 0; ikt < KTILE; ikt++) {+ auto bval = (UnpackBuf[in * KTILE + ikt]) * (float)bsptr[in];+ auto aval = int(a8ptr[ikt + im * A.lda]) * ascale;+ accf[im * NTILE + in] += aval * bval;+ }+ }+ }+ b4ptr += KTILE * NTILE / 2;+ b2ptr += KTILE * NTILE / 4;+ b1ptr += KTILE * NTILE / 8;+ a8ptr += KTILE;+ }+ }+ for (int im = 0; im < MTILE; im++) {+ for (int in = 0; in < NTILE; in++) {+ C[in + im * ldc] = accf[im * NTILE + in];+ }+ }+ return BTLA_CODE::Success;+}++template <typename T>+static inline BTLA_CODE mul(const T* src0ptr, const T* src1ptr, T* dstptr, size_t size) {+ for (size_t i = 0; i < size; i++) {+ float tmp = float(src0ptr[i]) * float(src1ptr[i]);+ dstptr[i] = tmp;+ }+ return BTLA_CODE::Success;+}++template <typename T>+static inline BTLA_CODE inv(const T* src0ptr, T* dstptr, size_t size) {+ for (size_t i = 0; i < size; i++) {+ float tmp = 1.f / float(src0ptr[i]);+ dstptr[i] = tmp;+ }+ return BTLA_CODE::Success;+}++template <typename T>+static inline BTLA_CODE add(const T* src0ptr, const T* src1ptr, T* dstptr, size_t size) {+ for (size_t i = 0; i < size; i++) {+ float tmp = float(src0ptr[i]) + float(src1ptr[i]);+ dstptr[i] = tmp;+ }+ return BTLA_CODE::Success;+}++template <typename T_DST>+static inline BTLA_CODE scale_exp_acc_sum_fp32(const float* src, const int src_step, T_DST* dst, int ld_dst,+ float* dst_sum, const int M_offset, const int N_offset, const int M,+ const int N, float scale, int causal_offset, void* /* tmpcache */,+ size_t /* cachesize */) {+ for (int i = 0; i < M; ++i) {+ const auto N_unmasked = std::min(N, causal_offset < 0 ? INT32_MAX : i + M_offset - N_offset + causal_offset + 1);+ for (int j = 0; j < N_unmasked; ++j) {+ const auto exp_ = expf(src[i * src_step + j] * scale);+ dst[i * ld_dst + j] = static_cast<T_DST>(exp_);+ dst_sum[i] += exp_;+ }+ if (N_unmasked < utils::padto(N, 64))+ memset(dst + i * ld_dst + N_unmasked, 0, sizeof(*dst) * (utils::padto(N, 64) - N_unmasked));+ }+ return BTLA_CODE::Success;+}++template <typename T_SRC, typename T_DST>+static inline BTLA_CODE scale_track_max(const T_SRC* src, const int src_step, T_DST* dst, T_DST* dst_max, int ld_dst,+ const int M_offset, const int N_offset, const int M, const int N, float scale,+ int causal_offset, float alibi_slope, float tanh_scale, int padding_type,+ void* tmpcache, size_t cachesize) {+ for (int i = 0; i < M; ++i) {+ const auto N_unmasked = std::min(N, padding_type == 0 ? INT32_MAX+ : padding_type == 1 ? i + M_offset - N_offset + causal_offset + 1+ : causal_offset - N_offset);+ for (int j = 0; j < N_unmasked; ++j) {+ const auto val_ = float(src[i * src_step + j]) * scale;+ dst[i * ld_dst + j] = static_cast<T_DST>(val_);+ dst_max[i] = std::max(dst_max[i], val_);+ }+ }+ return BTLA_CODE::Success;+}+} // namespace ref+} // namespace kernel+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_wrapper.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_wrapper.hnew file mode 100644index 000000000..3248e9529--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/kernel_wrapper.h@@ -0,0 +1,2216 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include <array>+#include <cassert>+#include <type_traits>+#include <immintrin.h>++#include "bestla.h"+#include "bestla_utils.h"+#include "kernel_ref.h"+#include "kernel_jit.h"+#include "kernel_avx2.h"+#include "kernel_avx_vnni.h"+#include "kernel_avx512f.h"+#include "kernel_avx512_vnni.h"+#include "kernel_avx512_bf16.h"+#include "kernel_avx512_fp16.h"++namespace bestla {+namespace kernel {+namespace wrapper {++class ZeroReg {+ public:+ static void forward() {+#if CompileAVX2()+ kernel::avx2::zero_reg();+#endif+ }+};++template <int NTILE, int RowPack, typename T_SRC, typename T_DST = T_SRC>+class PaddingInterleaveMN {+ // M x N ===> N/NTile x M/RowPack x NTile x RowPack (leading dim stride = NTile * dststride)+ public:+ TLACALL BTLA_CODE forward(const T_SRC* src, T_DST* dst, int row, int col, int row_pad, int col_pad, int src_step,+ int dst_step) {+#if CompileFP16()+ if constexpr (utils::isa_base<ISA_T>::avx512_fp16 && NTILE % 16 == 0) {+ const auto kern_ret = kernel::avx512f::avx512_fp16::padding_interleave_cvt<T_SRC, T_DST, RowPack>::forward(+ src, dst, NTILE, row, col, row_pad, col_pad, src_step, dst_step);+ if (kern_ret != BTLA_CODE::NotSupport) return kern_ret;+ }+#endif+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && NTILE % 16 == 0) {+ const auto kern_ret = kernel::avx512f::padding_interleave_cvt<T_SRC, T_DST, RowPack>::forward(+ src, dst, NTILE, row, col, row_pad, col_pad, src_step, dst_step);+ if (kern_ret != BTLA_CODE::NotSupport) return kern_ret;+ }+#endif+ return ref::padding_interleave(src, dst, row, col, row_pad, col_pad, src_step, dst_step, NTILE, RowPack);+ }++ AUTOCALL BTLA_CODE forward_auto(const T_SRC* src, T_DST* dst, int row, int col, int row_pad, int col_pad,+ int src_step, int dst_step) {+ GetCPUDevice();+ if (_cd->AVX512_FP16()) {+ return forward<BTLA_ISA::AVX512_FP16>(src, dst, row, col, row_pad, col_pad, src_step, dst_step);+ }+ if (_cd->AVX512F()) {+ return forward<BTLA_ISA::AVX512F>(src, dst, row, col, row_pad, col_pad, src_step, dst_step);+ }+ if (_cd->AVX2()) {+ return forward<BTLA_ISA::AVX2>(src, dst, row, col, row_pad, col_pad, src_step, dst_step);+ }+ return forward<BTLA_ISA::NoSIMD>(src, dst, row, col, row_pad, col_pad, src_step, dst_step);+ }+};++template <int NTile, int RowPack, typename T_SRC, typename T_DST = T_SRC>+class RevertPaddingInterleaveMN {+ // M x N ===> N/NTile x M/RowPack x NTile x RowPack (leading dim stride = NTile * dststride)+ public:+ TLACALL BTLA_CODE forward(const T_SRC* src, T_DST* dst, int row, int col, int row_pad, int col_pad, int src_step,+ int dst_step) {+ return ref::revert_padding_interleave(src, dst, row, col, row_pad, col_pad, src_step, dst_step, NTile, RowPack);+ }++ AUTOCALL BTLA_CODE forward_auto(const T_SRC* src, T_DST* dst, int row, int col, int row_pad, int col_pad,+ int src_step, int dst_step) {+ return forward<BTLA_ISA::NoSIMD>(src, dst, row, col, row_pad, col_pad, src_step, dst_step);+ }+};++template <int MTile, int ColPack, typename T_SRC, typename T_DST = T_SRC>+class PaddingTransInterleaveMN {+ // row and cols are in terms of src+ // M x N ===> M/MTile x N/ColPack x MTile x ColPack (leading dim stride = MTile * dststride)+ public:+ TLACALL BTLA_CODE forward(const T_SRC* src, T_DST* dst, int row, int col, int row_pad, int col_pad, int src_step,+ int dst_step) {+#if CompileFP16()+ if constexpr (utils::isa_base<ISA_T>::avx512_fp16) {+ const auto kern_ret = kernel::avx512f::avx512_fp16::padding_trans_interleave_cvt<T_SRC, T_DST, ColPack>::forward(+ src, dst, MTile, row, col, row_pad, col_pad, src_step, dst_step);+ if (kern_ret != BTLA_CODE::NotSupport) return kern_ret;+ }+#endif+#if CompileAVX512F()+ // Note: rows/cols and i/j are in terms of src+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ const auto kern_ret = kernel::avx512f::padding_trans_interleave_cvt<T_SRC, T_DST, ColPack>::forward(+ src, dst, MTile, row, col, row_pad, col_pad, src_step, dst_step);+ if (kern_ret != BTLA_CODE::NotSupport) return kern_ret;+ }+#endif+ return ref::padding_trans_interleave(src, dst, row, col, row_pad, col_pad, src_step, dst_step, MTile, ColPack);+ }++ AUTOCALL BTLA_CODE forward_auto(const T_SRC* src, T_DST* dst, int row, int col, int row_pad, int col_pad,+ int src_step, int dst_step) {+ GetCPUDevice();+ if (_cd->AVX512_FP16()) {+ return forward<BTLA_ISA::AVX512_FP16>(src, dst, row, col, row_pad, col_pad, src_step, dst_step);+ }+ if (_cd->AVX512F()) {+ return forward<BTLA_ISA::AVX512F>(src, dst, row, col, row_pad, col_pad, src_step, dst_step);+ }+ return forward<BTLA_ISA::NoSIMD>(src, dst, row, col, row_pad, col_pad, src_step, dst_step);+ }+};++class Memcpy2D {+ public:+ template <BTLA_ISA ISA_T, typename _SRC_T, typename _DST_T>+ static BTLA_CODE forward(const _SRC_T* srcptr, _DST_T* dstptr, int row, int col, int srcstep, int dststep,+ void* const_elt_v = nullptr) {+ auto ret = BTLA_CODE::NotSupport;+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ ret = kernel::jit::JitMemcpy2DAvx512f::forward<_SRC_T, _DST_T>(srcptr, dstptr, row, col, srcstep, dststep,+ const_elt_v);+ if (ret == BTLA_CODE::Success) {+ return ret;+ }+ }+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ auto align_col = static_cast<int>(col * sizeof(_SRC_T) / 32 * 32 / sizeof(_SRC_T));+ ret = kernel::jit::JitMemcpy2DAvx2::forward<_SRC_T, _DST_T>(srcptr, dstptr, row, align_col, srcstep, dststep,+ const_elt_v);+ if (col - align_col > 0)+ ret = kernel::ref::memcpy2d(+ srcptr + align_col, dstptr + align_col, row, static_cast<int>((col - align_col) * sizeof(_SRC_T)),+ static_cast<int>(srcstep * sizeof(_SRC_T)), static_cast<int>(dststep * sizeof(_DST_T)));+ if (ret == BTLA_CODE::Success) {+ return ret;+ }+ }+ return kernel::ref::memcpy2d(srcptr, dstptr, row, static_cast<int>(col * sizeof(_SRC_T)),+ static_cast<int>(srcstep * sizeof(_SRC_T)),+ static_cast<int>(dststep * sizeof(_DST_T)));+ }++ template <BTLA_ISA ISA_T, typename _SRC_T, typename _DST_T, BTLA_ELTWISEOP OP_T>+ static BTLA_CODE forward1(const _SRC_T* srcptr, _DST_T* dstptr, int row, int col, int srcstep, int dststep,+ void* const_elt_v = nullptr) {+ auto ret = BTLA_CODE::NotSupport;+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ ret = kernel::jit::JitMemcpy2DAvx512f::forward1<_SRC_T, _DST_T, OP_T>(srcptr, dstptr, row, col, srcstep, dststep,+ const_elt_v);+ if (ret == BTLA_CODE::Success) {+ return ret;+ }+ }+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ auto align_col = static_cast<int>(col * sizeof(_SRC_T) / 32 * 32 / sizeof(_SRC_T));+ ret = kernel::jit::JitMemcpy2DAvx2::forward1<_SRC_T, _DST_T, OP_T>(srcptr, dstptr, row, align_col, srcstep,+ dststep, const_elt_v);+ if (col - align_col > 0)+ ret = kernel::ref::memcpy2d_withop<_SRC_T, _DST_T, OP_T>(srcptr + align_col, dstptr + align_col, row,+ col - align_col, srcstep, dststep, const_elt_v);+ if (ret == BTLA_CODE::Success) {+ return ret;+ }+ }+ return ref::memcpy2d_withop<_SRC_T, _DST_T, OP_T>(srcptr, dstptr, row, col, srcstep, dststep, const_elt_v);+ }++ template <typename _SRC_T, typename _DST_T, BTLA_ELTWISEOP OP_T>+ static BTLA_CODE forward1_auto(const _SRC_T* srcptr, _DST_T* dstptr, int row, int col, int srcstep, int dststep,+ void* const_elt_v = nullptr) {+ GetCPUDevice();+ if (_cd->AVX512F()) {+ return forward1<BTLA_ISA::AVX512F, _SRC_T, _DST_T, OP_T>(srcptr, dstptr, row, col, srcstep, dststep, const_elt_v);+ }+ if (_cd->AVX2()) {+ return forward1<BTLA_ISA::AVX2, _SRC_T, _DST_T, OP_T>(srcptr, dstptr, row, col, srcstep, dststep, const_elt_v);+ }+ return forward1<BTLA_ISA::NoSIMD, _SRC_T, _DST_T, OP_T>(srcptr, dstptr, row, col, srcstep, dststep, const_elt_v);+ }++ template <typename _SRC_T, typename _DST_T>+ static BTLA_CODE forward_auto(const _SRC_T* srcptr, _DST_T* dstptr, int row, int col, int srcstep, int dststep) {+ GetCPUDevice();+ if (_cd->AVX512F()) {+ return forward<BTLA_ISA::AVX512F, _SRC_T, _DST_T>(srcptr, dstptr, row, col, srcstep, dststep);+ }+ if (_cd->AVX2()) {+ return forward<BTLA_ISA::AVX2, _SRC_T, _DST_T>(srcptr, dstptr, row, col, srcstep, dststep);+ }+ return forward<BTLA_ISA::NoSIMD, _SRC_T, _DST_T>(srcptr, dstptr, row, col, srcstep, dststep);+ }+};++class Memcpy2DFp32CvtBf16 {+ public:+ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(const void* srcptr, void* dstptr, int row, int col, int srcstride, int dststride,+ bool zeropadding) {+#if CompileBF16()+ if constexpr (utils::isa_base<ISA_T>::avx512_bf16) {+ return kernel::avx512f::avx512_bf16::fp32_cvt_bf16_2D_write_back(srcptr, dstptr, row, col, srcstride, dststride,+ zeropadding);+ }+#endif+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return kernel::avx512f::cvt_fp32_T_2D((const float*)srcptr, (utils::bf16*)dstptr, row, col,+ srcstride / sizeof(float), dststride / sizeof(utils::bf16), zeropadding);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return kernel::avx2::cvt_fp32_T_2D((const float*)srcptr, (utils::bf16*)dstptr, row, col,+ srcstride / sizeof(float), dststride / sizeof(utils::bf16), zeropadding);+ }+#endif+ return kernel::ref::dt_cvt_2D_write_back<float, utils::bf16>(srcptr, dstptr, row, col, srcstride, dststride,+ zeropadding);+ }+};++class Memcpy2DFp32CvtFp16 {+ public:+ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(const void* srcptr, void* dstptr, int row, int col, int srcstride, int dststride,+ bool zeropadding) {+#if CompileFP16()+ if constexpr (utils::isa_base<ISA_T>::avx512_fp16) {+ return kernel::avx512f::avx512_fp16::fp32_cvt_fp16_2D_write_back(+ reinterpret_cast<const float*>(srcptr), reinterpret_cast<utils::fp16*>(dstptr), row, col,+ srcstride / sizeof(float), dststride / sizeof(utils::fp16), zeropadding);+ }+#endif+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return kernel::avx512f::cvt_fp32_T_2D(reinterpret_cast<const float*>(srcptr),+ reinterpret_cast<utils::fp16*>(dstptr), row, col, srcstride / sizeof(float),+ dststride / sizeof(utils::fp16), zeropadding);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return kernel::avx2::cvt_fp32_T_2D(reinterpret_cast<const float*>(srcptr), reinterpret_cast<utils::fp16*>(dstptr),+ row, col, srcstride / sizeof(float), dststride / sizeof(utils::fp16),+ zeropadding);+ }+#endif+ return kernel::ref::dt_cvt_2D_write_back<float, utils::fp16>(srcptr, dstptr, row, col, srcstride, dststride,+ zeropadding);+ }+};++template <typename T>+class Memcpy2DFp32TPadding {+ public:+ TLACALL BTLA_CODE forward(const float* srcptr, T* dstptr, int row, int col, int srcstride, int dststride,+ bool zeropadding) {+ if constexpr (std::is_same_v<T, utils::fp16>) {+ return Memcpy2DFp32CvtFp16::forward<ISA_T>(srcptr, dstptr, row, col, srcstride, dststride, zeropadding);+ }+ if constexpr (std::is_same_v<T, utils::bf16>) {+ return Memcpy2DFp32CvtBf16::forward<ISA_T>(srcptr, dstptr, row, col, srcstride, dststride, zeropadding);+ }+ return Memcpy2D::forward<ISA_T, float, T>(srcptr, dstptr, row, col, srcstride / sizeof(float),+ dststride / sizeof(T), nullptr);+ }++ AUTOCALL BTLA_CODE forward_auto(const float* srcptr, T* dstptr, int row, int col, int srcstride, int dststride,+ bool zeropadding) {+ GetCPUDevice();+ if (_cd->AVX512_FP16()) {+ return forward<BTLA_ISA::AVX512_FP16>(srcptr, dstptr, row, col, srcstride, dststride, zeropadding);+ }+ if (_cd->AVX512_BF16()) {+ return forward<BTLA_ISA::AVX512_BF16>(srcptr, dstptr, row, col, srcstride, dststride, zeropadding);+ }+ if (_cd->AVX512F()) {+ return forward<BTLA_ISA::AVX512F>(srcptr, dstptr, row, col, srcstride, dststride, zeropadding);+ }+ if (_cd->AVX2()) {+ return forward<BTLA_ISA::AVX2>(srcptr, dstptr, row, col, srcstride, dststride, zeropadding);+ }+ return forward<BTLA_ISA::NoSIMD>(srcptr, dstptr, row, col, srcstride, dststride, zeropadding);+ }+};++class Memcpy2DFp16CvtFp32 {+ public:+ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(const void* srcptr, void* dstptr, int row, int col, int srcstride, int dststride,+ bool zeropadding) {+#if CompileFP16()+ if constexpr (utils::isa_base<ISA_T>::avx512_fp16) {+ return kernel::avx512f::avx512_fp16::fp16_cvt_fp32_2D_write_back( //+ reinterpret_cast<const utils::fp16*>(srcptr), reinterpret_cast<float*>(dstptr), row, col,+ srcstride / sizeof(utils::fp16), dststride / sizeof(float), zeropadding);+ }+#endif+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return kernel::avx512f::cvt_T_fp32_2D( //+ reinterpret_cast<const utils::fp16*>(srcptr), reinterpret_cast<float*>(dstptr), row, col,+ srcstride / sizeof(utils::fp16), dststride / sizeof(float), zeropadding);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return kernel::avx2::cvt_T_fp32_2D( //+ reinterpret_cast<const utils::fp16*>(srcptr), reinterpret_cast<float*>(dstptr), row, col,+ srcstride / sizeof(utils::fp16), dststride / sizeof(float), zeropadding);+ }+#endif+ return kernel::ref::dt_cvt_2D_write_back<utils::fp16, float>(srcptr, dstptr, row, col, srcstride, dststride,+ zeropadding);+ }+};++class Memcpy2DBf16CvtFp32 {+ public:+ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(const void* srcptr, void* dstptr, int row, int col, int srcstride, int dststride,+ bool zeropadding) {+#if CompileBF16()+ if constexpr (utils::isa_base<ISA_T>::avx512_bf16) {+ return kernel::avx512f::avx512_bf16::bf16_cvt_fp32_2D_write_back( //+ reinterpret_cast<const utils::bf16*>(srcptr), reinterpret_cast<float*>(dstptr), row, col,+ srcstride / sizeof(utils::bf16), dststride / sizeof(float), zeropadding);+ }+#endif+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return kernel::avx512f::cvt_T_fp32_2D(reinterpret_cast<const utils::bf16*>(srcptr),+ reinterpret_cast<float*>(dstptr), row, col, srcstride / sizeof(utils::bf16),+ dststride / sizeof(float), zeropadding);+ }+#endif+#if CompileAVX2()+ if constexpr (ISA_T >= BTLA_ISA::AVX2) {+ return kernel::avx2::cvt_T_fp32_2D(reinterpret_cast<const utils::bf16*>(srcptr), reinterpret_cast<float*>(dstptr),+ row, col, srcstride / sizeof(utils::bf16), dststride / sizeof(float),+ zeropadding);+ }+#endif+ return kernel::ref::dt_cvt_2D_write_back<utils::bf16, float>(srcptr, dstptr, row, col, srcstride, dststride,+ zeropadding);+ }+};++class Memcpy2DFp8e4m3CvtFp32 {+ public:+ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(const void* srcptr, void* dstptr, int row, int col, int srcstride, int dststride,+ bool zeropadding) {+ return kernel::ref::dt_cvt_2D_write_back<utils::f8e4m3, float>(srcptr, dstptr, row, col, srcstride, dststride,+ zeropadding);+ }+};++class Memcpy2DPadding {+ public:+ static BTLA_CODE forward(const void* srcptr, void* dstptr, int row, int colsize, int srcstride, int dststride,+ bool zeropadding) {+ auto srcp = (char*)srcptr;+ if (zeropadding && dststride != colsize) {+ for (int i = 0; i < row; i++) {+ auto dstp = (char*)dstptr + i * dststride;+ std::memcpy(dstp, srcp + i * srcstride, colsize);+ std::memset(dstp + colsize, 0, (dststride - colsize));+ }+ } else {+ for (int i = 0; i < row; i++) {+ auto dstp = (char*)dstptr + i * dststride;+ std::memcpy(dstp, srcp + i * srcstride, colsize);+ }+ }+ return BTLA_CODE::Success;+ }+};++class Memcpy2DCvtPadding {+ public:+ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(const void* srcptr, BTLA_DTYPE srct, void* dstptr, BTLA_DTYPE dstt, int row, int col,+ int srcstep, int dststep, bool zeropadding, void* tmpcache, size_t cachesize) {+ bool needf32 = false;+ int rowstep = utils::downdiv(cachesize, col * 4);+ rowstep = std::min(row, rowstep);+ if (dstt != BTLA_DTYPE::F32 && srct != BTLA_DTYPE::F32) {+ needf32 = true;+ assert(col <= cachesize * 4);+ }+ auto srcbytes = utils::bestla_dtype_bytes(srct);+ auto dstbytes = utils::bestla_dtype_bytes(dstt);+ if (srct == dstt) {+ return Memcpy2D::forward_auto<int8_t, int8_t>((const int8_t*)srcptr, (int8_t*)dstptr, row, col * srcbytes,+ srcstep * srcbytes, dststep * dstbytes);+ }+ if (needf32) {+ auto tmpf32 = (float*)tmpcache;+ int cachestride = col * 4;+ for (int irow = 0; irow < row; irow += rowstep) {+ auto row_re = utils::remainsize(irow, row, rowstep);+ if (srct == BTLA_DTYPE::F16) {+ Memcpy2DFp16CvtFp32::forward<ISA_T>((utils::fp16*)srcptr + irow * srcstep, tmpf32, row_re, col, srcstep * 2,+ cachestride, false);+ }+ if (srct == BTLA_DTYPE::BF16) {+ Memcpy2DBf16CvtFp32::forward<ISA_T>((utils::bf16*)srcptr + irow * srcstep, tmpf32, row_re, col, srcstep * 2,+ cachestride, false);+ }+ if (dstt == BTLA_DTYPE::F16) {+ Memcpy2DFp32CvtFp16::forward<ISA_T>(tmpf32, (utils::fp16*)dstptr + irow * dststep, row_re, col, cachestride,+ dststep * 2, zeropadding);+ }+ if (dstt == BTLA_DTYPE::BF16) {+ Memcpy2DFp32CvtBf16::forward<ISA_T>(tmpf32, (utils::bf16*)dstptr + irow * dststep, row_re, col, cachestride,+ dststep * 2, zeropadding);+ }+ }+ return BTLA_CODE::Success;+ } else {+ if (dstt == BTLA_DTYPE::F32) {+ if (srct == BTLA_DTYPE::F16) {+ return Memcpy2DFp16CvtFp32::forward<ISA_T>(srcptr, dstptr, row, col, srcstep * 2, dststep * 4, zeropadding);+ }+ if (srct == BTLA_DTYPE::BF16) {+ return Memcpy2DBf16CvtFp32::forward<ISA_T>(srcptr, dstptr, row, col, srcstep * 2, dststep * 4, zeropadding);+ }+ if (srct == BTLA_DTYPE::F8_E4M3) {+ return Memcpy2DFp8e4m3CvtFp32::forward<ISA_T>(srcptr, dstptr, row, col, srcstep, dststep * 4, zeropadding);+ }+ }+ if (dstt == BTLA_DTYPE::BF16) {+ assert(srct == BTLA_DTYPE::F32);+ return Memcpy2DFp32CvtBf16::forward<ISA_T>(srcptr, dstptr, row, col, srcstep * 4, dststep * 2, zeropadding);+ }+ if (dstt == BTLA_DTYPE::F16) {+ assert(srct == BTLA_DTYPE::F32);+ return Memcpy2DFp32CvtFp16::forward<ISA_T>(srcptr, dstptr, row, col, srcstep * 4, dststep * 2, zeropadding);+ }+ }+ return BTLA_CODE::NotSupport;+ }++ static BTLA_CODE forward_auto(const void* srcptr, BTLA_DTYPE srct, void* dstptr, BTLA_DTYPE dstt, int row, int col,+ int srcstep, int dststep, bool zeropadding, void* tmpcache, size_t cachesize) {+ GetCPUDevice();+ if (_cd->AVX512_FP16()) {+ return forward<BTLA_ISA::AVX512_FP16>(srcptr, srct, dstptr, dstt, row, col, srcstep, dststep, zeropadding,+ tmpcache, cachesize);+ }+ if (_cd->AVX512_BF16()) {+ return forward<BTLA_ISA::AVX512_BF16>(srcptr, srct, dstptr, dstt, row, col, srcstep, dststep, zeropadding,+ tmpcache, cachesize);+ }+ if (_cd->AVX512F()) {+ return forward<BTLA_ISA::AVX512F>(srcptr, srct, dstptr, dstt, row, col, srcstep, dststep, zeropadding, tmpcache,+ cachesize);+ }+ if (_cd->AVX2()) {+ return forward<BTLA_ISA::AVX2>(srcptr, srct, dstptr, dstt, row, col, srcstep, dststep, zeropadding, tmpcache,+ cachesize);+ }+ return forward<BTLA_ISA::NoSIMD>(srcptr, srct, dstptr, dstt, row, col, srcstep, dststep, zeropadding, tmpcache,+ cachesize);+ }+};+template <typename _DST_T, int _PACK_ROW>+class DecompressDQKBlockS4Fp {+ public:+ template <BTLA_ISA ISA_T, BTLA_DTYPE S4_T>+ static inline BTLA_CODE forward(utils::int4x2* srcptr, _DST_T* dstptr, int row, int col, int ld_src, int ld_dst,+ uint8_t* scales, float* dq_scale, int k_offset, int n_offset, int kblock, int NPad,+ int N, int dq_blk, int dq_offset_idx, void* tmp, size_t tmpsize) {+ BTLA_CODE ret = BTLA_CODE::NotSupport;+ ret = ref::decompress_dq_kblock_s4_fp<S4_T, _DST_T, _PACK_ROW>(+ srcptr, dstptr, row, col, ld_src, ld_dst, scales, dq_scale, k_offset, n_offset, kblock, dq_blk, dq_offset_idx,+ NPad, N, reinterpret_cast<int8_t*>(tmp), tmpsize);+ return ret;+ }+};++class Dq8GetScale {+ public:+ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(uint8_t* src, float* dst, int row, int col, int scale_offset, int dq_blk, int dq_offset_idx,+ float* dq_scale, int src_stride, int dst_stride, bool zeropadding, int mN) {+#if CompileAVX512F()+ if (ISA_T >= BTLA_ISA::AVX512F) {+ return kernel::avx512f::dq8_get_fp_scale(src, dst, row, col, scale_offset, dq_blk, dq_offset_idx, dq_scale,+ src_stride, dst_stride, zeropadding, mN);+ }+#endif+#if CompileAVX2()+ if (ISA_T >= BTLA_ISA::AVX2) {+ return kernel::avx2::dq8_get_fp_scale(src, dst, row, col, scale_offset, dq_blk, dq_offset_idx, dq_scale,+ src_stride, dst_stride, zeropadding, mN);+ }+#endif+ return kernel::ref::dq8_get_fp_scale(src, dst, row, col, scale_offset, dq_blk, dq_offset_idx, dq_scale, src_stride,+ dst_stride, zeropadding, mN);+ }+};++class CompressS8S4 {+ public:+ TLACALL BTLA_CODE forward(const int8_t* srcptr, bestla::utils::int4x2* dstptr, size_t size) {+ return ref::compress_s8_s4(srcptr, dstptr, size);+ }++ AUTOCALL BTLA_CODE forward_auto(const int8_t* srcptr, bestla::utils::int4x2* dstptr, size_t size) {+ return forward<BTLA_ISA::NoSIMD>(srcptr, dstptr, size);+ }+};++class CompressFp4 {+ public:+ TLACALL BTLA_CODE forward(const int8_t* srcptr, bestla::utils::f4x2* dstptr, size_t size) {+ return ref::compress_f4(srcptr, dstptr, size);+ }++ AUTOCALL BTLA_CODE forward_auto(const int8_t* srcptr, bestla::utils::f4x2* dstptr, size_t size) {+ return forward<BTLA_ISA::NoSIMD>(srcptr, dstptr, size);+ }+};++class CompressBit7 {+ public:+ TLACALL BTLA_CODE forward(const int8_t* srcptr, bestla::utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr,+ utils::bit1x8* bit1ptr, size_t size) {+ return ref::compress_7bit(srcptr, bit4ptr, bit2ptr, bit1ptr, size);+ }++ AUTOCALL BTLA_CODE forward_auto(const int8_t* srcptr, bestla::utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr,+ utils::bit1x8* bit1ptr, size_t size) {+ return forward<BTLA_ISA::NoSIMD>(srcptr, bit4ptr, bit2ptr, bit1ptr, size);+ }+};++class CompressBit6 {+ public:+ TLACALL BTLA_CODE forward(const int8_t* srcptr, bestla::utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr, size_t size) {+ return ref::compress_6bit(srcptr, bit4ptr, bit2ptr, size);+ }++ AUTOCALL BTLA_CODE forward_auto(const int8_t* srcptr, bestla::utils::bit4x2* bit4ptr, utils::bit2x4* bit2ptr,+ size_t size) {+ return forward<BTLA_ISA::NoSIMD>(srcptr, bit4ptr, bit2ptr, size);+ }+};++class CompressBit5 {+ public:+ TLACALL BTLA_CODE forward(const int8_t* srcptr, bestla::utils::bit4x2* bit4ptr, utils::bit1x8* bit1ptr, size_t size) {+ return ref::compress_5bit(srcptr, bit4ptr, bit1ptr, size);+ }++ AUTOCALL BTLA_CODE forward_auto(const int8_t* srcptr, bestla::utils::bit4x2* bit4ptr, utils::bit1x8* bit1ptr,+ size_t size) {+ return forward<BTLA_ISA::NoSIMD>(srcptr, bit4ptr, bit1ptr, size);+ }+};++class CompressBit3 {+ public:+ TLACALL BTLA_CODE forward(const int8_t* srcptr, bestla::utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr, size_t size) {+ return ref::compress_3bit(srcptr, bit2ptr, bit1ptr, size);+ }++ AUTOCALL BTLA_CODE forward_auto(const int8_t* srcptr, bestla::utils::bit2x4* bit2ptr, utils::bit1x8* bit1ptr,+ size_t size) {+ return forward<BTLA_ISA::NoSIMD>(srcptr, bit2ptr, bit1ptr, size);+ }+};++class CompressBit2 {+ public:+ TLACALL BTLA_CODE forward(const int8_t* srcptr, bestla::utils::bit2x4* bit2ptr, size_t size) {+ return ref::compress_2bit(srcptr, bit2ptr, size);+ }++ AUTOCALL BTLA_CODE forward_auto(const int8_t* srcptr, bestla::utils::bit2x4* bit2ptr, size_t size) {+ return forward<BTLA_ISA::NoSIMD>(srcptr, bit2ptr, size);+ }+};++class CompressBit1 {+ public:+ TLACALL BTLA_CODE forward(const int8_t* srcptr, bestla::utils::bit1x8* bit1ptr, size_t size) {+ return ref::compress_1bit(srcptr, bit1ptr, size);+ }++ AUTOCALL BTLA_CODE forward_auto(const int8_t* srcptr, bestla::utils::bit1x8* bit1ptr, size_t size) {+ return forward<BTLA_ISA::NoSIMD>(srcptr, bit1ptr, size);+ }+};++template <typename _T>+class Transpose2D {+ public:+ TLACALL BTLA_CODE forward(const _T* srcptr, _T* dstptr, int row, int col, int ld_src, int ld_dst) {+ return ref::transpose2d(srcptr, dstptr, row, col, ld_src, ld_dst);+ }++ AUTOCALL BTLA_CODE forward_auto(const _T* srcptr, _T* dstptr, int row, int col, int ld_src, int ld_dst) {+ return forward<BTLA_ISA::NoSIMD>(srcptr, dstptr, row, col, ld_src, ld_dst);+ }+};++class QuantizeSignIntRowBlock {+ public:+ TLACALL BTLA_CODE forward(const float* srcptr, int8_t* dstptr, int row, int col, int ld_src, int ld_dst,+ float* scales, int8_t* zero_points, int blocksize, BTLA_DTYPE qtype) {+ // TODO(Yu) simd version for quick quant+ // #if CompileAVX512F()+ // if constexpr (utils::isa_base<ISA_T>::avx512f) {+ // return avx512f::quantize_f32_sign_int_rowblock<QDT_T>(srcptr, dstptr, row, col, ld_src, ld_dst, scales,+ // zero_points, blocksize);+ // }+ // #endif+ return ref::quantize_f32_sign_int_rowblock(srcptr, dstptr, row, col, ld_src, ld_dst, scales, zero_points, blocksize,+ qtype);+ }++ AUTOCALL BTLA_CODE forward_auto(const float* srcptr, int8_t* dstptr, int row, int col, int ld_src, int ld_dst,+ float* scales, int8_t* zero_points, int blocksize, BTLA_DTYPE qtype) {+ return forward<BTLA_ISA::NoSIMD>(srcptr, dstptr, row, col, ld_src, ld_dst, scales, zero_points, blocksize, qtype);+ }+};++template <BTLA_DTYPE F8_T>+class QuantizeF8RowBlock {+ public:+ TLACALL BTLA_CODE forward(const float* srcptr, int8_t* dstptr, int row, int col, int ld_src, int ld_dst,+ float* scales, int blocksize, BTLA_DTYPE scale_dtype) {+ return ref::quantize_f32_f8_rowblock_mxscale<F8_T>(srcptr, dstptr, row, col, ld_src, ld_dst, scales, blocksize,+ scale_dtype);+ }++ AUTOCALL BTLA_CODE forward_auto(const float* srcptr, int8_t* dstptr, int row, int col, int ld_src, int ld_dst,+ float* scales, int blocksize, BTLA_DTYPE scale_dtype) {+ return forward<BTLA_ISA::NoSIMD>(srcptr, dstptr, row, col, ld_src, ld_dst, scales, blocksize, scale_dtype);+ }+};++template <BTLA_DTYPE F4_T>+class QuantizeF4RowBlock {+ public:+ TLACALL BTLA_CODE forward(const float* srcptr, int8_t* dstptr, int row, int col, int ld_src, int ld_dst,+ float* scales, int blocksize) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::quantize_f32_f4_rowblock<F4_T>(srcptr, dstptr, row, col, ld_src, ld_dst, scales, blocksize);+ }+#endif+ return ref::quantize_f32_f4_rowblock<F4_T>(srcptr, dstptr, row, col, ld_src, ld_dst, scales, blocksize);+ }++ AUTOCALL BTLA_CODE forward_auto(const float* srcptr, int8_t* dstptr, int row, int col, int ld_src, int ld_dst,+ float* scales, int blocksize) {+ GetCPUDevice();+ if (_cd->AVX512F()) {+ return forward<BTLA_ISA::AVX512F>(srcptr, dstptr, row, col, ld_src, ld_dst, scales, blocksize);+ }+ return forward<BTLA_ISA::NoSIMD>(srcptr, dstptr, row, col, ld_src, ld_dst, scales, blocksize);+ }+};++template <typename SRC_T>+class QuantizeU8ColBlock {+ public:+ TLACALL BTLA_CODE forward(int row, int col, const SRC_T* srcptr, int ld_src, uint8_t* dstptr, int ld_dst,+ float* scales, int ld_scale, uint8_t* zps, int blocksize, float* blkreduce) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::quantize_fp_u8_colblock<SRC_T>(row, col, srcptr, ld_src, dstptr, ld_dst, scales, ld_scale, zps,+ blocksize, blkreduce);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::quantize_fp_u8_colblock<SRC_T>(row, col, srcptr, ld_src, dstptr, ld_dst, scales, ld_scale, zps,+ blocksize, blkreduce);+ }+#endif+ return ref::quantize_fp_u8_colblock(row, col, srcptr, ld_src, dstptr, ld_dst, scales, ld_scale, zps, blocksize,+ blkreduce);+ }++ AUTOCALL BTLA_CODE forward_auto(int row, int col, const SRC_T* srcptr, int ld_src, uint8_t* dstptr, int ld_dst,+ float* scales, int ld_scale, uint8_t* zps, int blocksize, float* blkreduce) {+ GetCPUDevice();+ if (_cd->AVX512F()) {+ return forward<BTLA_ISA::AVX512F>(row, col, srcptr, ld_src, dstptr, ld_dst, scales, ld_scale, zps, blocksize,+ blkreduce);+ }+ if (_cd->AVX2()) {+ return forward<BTLA_ISA::AVX2>(row, col, srcptr, ld_src, dstptr, ld_dst, scales, ld_scale, zps, blocksize,+ blkreduce);+ }+ return forward<BTLA_ISA::NoSIMD>(row, col, srcptr, ld_src, dstptr, ld_dst, scales, ld_scale, zps, blocksize,+ blkreduce);+ }+};++template <typename SRC_T>+class QuantizeS8ColBlock {+ public:+ TLACALL BTLA_CODE forward(int row, int col, const SRC_T* srcptr, int ld_src, int8_t* dstptr, int ld_dst,+ float* scales, int ld_scale, int blocksize, float* reduce) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::quantize_fp_s8_colblock<SRC_T>(row, col, srcptr, ld_src, dstptr, ld_dst, scales, ld_scale,+ blocksize, reduce);+ }+#endif+ return ref::quantize_fp_s8_colblock(row, col, srcptr, ld_src, dstptr, ld_dst, scales, ld_scale, blocksize, reduce);+ }++ AUTOCALL BTLA_CODE forward_auto(int row, int col, const SRC_T* srcptr, int ld_src, int8_t* dstptr, int ld_dst,+ float* scales, int ld_scale, int blocksize, float* reduce) {+ GetCPUDevice();+ if (_cd->AVX512F()) {+ return forward<BTLA_ISA::AVX512F>(row, col, srcptr, ld_src, dstptr, ld_dst, scales, ld_scale, blocksize, reduce);+ }+ if (_cd->AVX2()) {+ return forward<BTLA_ISA::AVX2>(row, col, srcptr, ld_src, dstptr, ld_dst, scales, ld_scale, blocksize, reduce);+ }+ return forward<BTLA_ISA::NoSIMD>(row, col, srcptr, ld_src, dstptr, ld_dst, scales, ld_scale, blocksize, reduce);+ }+};++class Broadcast {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(int num, const uint8_t& srcval, uint8_t* dstptr) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::broadcast_u8(num, srcval, dstptr);+ }+#endif+ return ref::broadcast_u8(num, srcval, dstptr);+ }+};++class AccumulateDequantizeS32F32 {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(const int32_t* srcptr, float* dstptr, float alpha, float beta, int row, int col,+ int ld_src, int ld_dst, float* ascales, int ldas, float* wscales) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::accumulate_dequantize_s32_f32(srcptr, dstptr, alpha, beta, row, col, ld_src, ld_dst, ascales,+ ldas, wscales);+ }+#endif+ return ref::accumulate_dequantize_s32_f32(srcptr, dstptr, alpha, beta, row, col, ld_src, ld_dst, ascales, ldas,+ wscales);+ }+};++template <int PackRow, int NTILE>+class DecompressKBlockS4S8 {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(utils::int4x2* srcptr, int8_t* zpptr, int8_t* dstptr, int blocksize, int ldzp,+ int n_offset, int k_offset, int row, int col, void* tmp, size_t tmpsize) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && NTILE % 16 == 0) {+ return avx512f::decompress_kblock_s4_s8<PackRow, NTILE>(srcptr, zpptr, dstptr, blocksize, ldzp, n_offset,+ k_offset, row, col, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2 && NTILE % 8 == 0) {+ return avx2::decompress_kblock_s4_s8<PackRow, NTILE>(srcptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset,+ row, col, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::decompress_kblock_s4_s8<PackRow, NTILE>(srcptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, row,+ col, (int8_t*)tmp, tmpsize);+ }+};++template <int PackRow, int NTILE>+class DecompressKBlockS7S8 {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(utils::bit4x2* b4ptr, utils::bit2x4* b2ptr, utils::bit1x8* b1ptr, int8_t* zpptr,+ int8_t* dstptr, int blocksize, int ldzp, int n_offset, int k_offset, int row, int col,+ void* tmp, size_t tmpsize) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && NTILE % 16 == 0) {+ return avx512f::decompress_kblock_s7_s8<PackRow, NTILE>(b4ptr, b2ptr, b1ptr, zpptr, dstptr, blocksize, ldzp,+ n_offset, k_offset, row, col, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2 && NTILE % 8 == 0) {+ return avx2::decompress_kblock_s7_s8<PackRow, NTILE>(b4ptr, b2ptr, b1ptr, zpptr, dstptr, blocksize, ldzp,+ n_offset, k_offset, row, col, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::decompress_kblock_s7_s8<PackRow, NTILE>(b4ptr, b2ptr, b1ptr, zpptr, dstptr, blocksize, ldzp, n_offset,+ k_offset, row, col, (int8_t*)tmp, tmpsize);+ }+};++template <int PackRow, int NTILE>+class DecompressKBlockS6S8 {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(utils::bit4x2* b4ptr, utils::bit2x4* b2ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row, int col, void* tmp,+ size_t tmpsize) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && NTILE % 16 == 0) {+ return avx512f::decompress_kblock_s6_s8<PackRow, NTILE>(b4ptr, b2ptr, zpptr, dstptr, blocksize, ldzp, n_offset,+ k_offset, row, col, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::decompress_kblock_s6_s8<PackRow, NTILE>(b4ptr, b2ptr, zpptr, dstptr, blocksize, ldzp, n_offset,+ k_offset, row, col, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::decompress_kblock_s6_s8<PackRow, NTILE>(b4ptr, b2ptr, zpptr, dstptr, blocksize, ldzp, n_offset,+ k_offset, row, col, (int8_t*)tmp, tmpsize);+ }+};++template <int PackRow, int NTILE>+class DecompressKBlockS5S8 {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(utils::bit4x2* b4ptr, utils::bit1x8* b1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row, int col, void* tmp,+ size_t tmpsize) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && NTILE % 16 == 0) {+ return avx512f::decompress_kblock_s5_s8<PackRow, NTILE>(b4ptr, b1ptr, zpptr, dstptr, blocksize, ldzp, n_offset,+ k_offset, row, col, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::decompress_kblock_s5_s8<PackRow, NTILE>(b4ptr, b1ptr, zpptr, dstptr, blocksize, ldzp, n_offset,+ k_offset, row, col, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::decompress_kblock_s5_s8<PackRow, NTILE>(b4ptr, b1ptr, zpptr, dstptr, blocksize, ldzp, n_offset,+ k_offset, row, col, (int8_t*)tmp, tmpsize);+ }+};++template <int PackRow, int NTILE>+class DecompressKBlockS3S8 {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(utils::bit2x4* b2ptr, utils::bit1x8* b1ptr, int8_t* zpptr, int8_t* dstptr,+ int blocksize, int ldzp, int n_offset, int k_offset, int row, int col, void* tmp,+ size_t tmpsize) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && NTILE % 16 == 0) {+ return avx512f::decompress_kblock_s3_s8<PackRow, NTILE>(b2ptr, b1ptr, zpptr, dstptr, blocksize, ldzp, n_offset,+ k_offset, row, col, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::decompress_kblock_s3_s8<PackRow, NTILE>(b2ptr, b1ptr, zpptr, dstptr, blocksize, ldzp, n_offset,+ k_offset, row, col, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::decompress_kblock_s3_s8<PackRow, NTILE>(b2ptr, b1ptr, zpptr, dstptr, blocksize, ldzp, n_offset,+ k_offset, row, col, (int8_t*)tmp, tmpsize);+ }+};++template <int PackRow, int NTILE>+class DecompressKBlockS2S8 {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(utils::bit2x4* b2ptr, int8_t* zpptr, int8_t* dstptr, int blocksize, int ldzp,+ int n_offset, int k_offset, int row, int col, void* tmp, size_t tmpsize) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && NTILE % 16 == 0) {+ return avx512f::decompress_kblock_s2_s8<PackRow, NTILE>(b2ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset,+ row, col, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::decompress_kblock_s2_s8<PackRow, NTILE>(b2ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset,+ row, col, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::decompress_kblock_s2_s8<PackRow, NTILE>(b2ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, row,+ col, (int8_t*)tmp, tmpsize);+ }+};++template <int PackRow, int NTILE>+class DecompressKBlockS1S8 {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(utils::bit1x8* b1ptr, int8_t* zpptr, int8_t* dstptr, int blocksize, int ldzp,+ int n_offset, int k_offset, int row, int col, void* tmp, size_t tmpsize) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && NTILE % 16 == 0) {+ return avx512f::decompress_kblock_s1_s8<PackRow, NTILE>(b1ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset,+ row, col, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::decompress_kblock_s1_s8<PackRow, NTILE>(b1ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset,+ row, col, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::decompress_kblock_s1_s8<PackRow, NTILE>(b1ptr, zpptr, dstptr, blocksize, ldzp, n_offset, k_offset, row,+ col, (int8_t*)tmp, tmpsize);+ }+};++template <int PackRow, int NTILE, typename DstT>+class DecompressKBlockS8Fp {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(int8_t* srcptr, DstT* dstptr, int row, int col, void* scales, BTLA_DTYPE sdtype,+ int8_t* zero_points, int k_offset, int n_offset, int kblock, int NPad, void* tmp,+ size_t tmpsize) {+ BTLA_CODE ret = BTLA_CODE::NotSupport;+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && NTILE % 16 == 0) {+ ret = avx512f::decompress_kblock_s8_fp<PackRow, NTILE, DstT>(srcptr, dstptr, row, col, scales, sdtype,+ zero_points, k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ if (ret == BTLA_CODE::Success) return ret;+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ ret = avx2::decompress_kblock_s8_fp<PackRow, NTILE, DstT>(srcptr, dstptr, row, col, scales, sdtype, zero_points,+ k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ if (ret == BTLA_CODE::Success) return ret;+ }+#endif+ ret = ref::decompress_kblock_s8_fp<PackRow, NTILE, DstT>(srcptr, dstptr, row, col, scales, sdtype, zero_points,+ k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ return ret;+ }+};++template <int PackRow, int NTILE, typename DstT>+class DecompressKBlockS7Fp {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(utils::bit4x2* b4ptr, utils::bit2x4* b2ptr, utils::bit1x8* b1ptr, DstT* dstptr,+ int row, int col, void* scales, BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset,+ int n_offset, int kblock, int NPad, void* tmp, size_t tmpsize) {+ BTLA_CODE ret = BTLA_CODE::NotSupport;+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && NTILE % 16 == 0) {+ return avx512f::decompress_kblock_s7_fp<PackRow, NTILE, DstT>(b4ptr, b2ptr, b1ptr, dstptr, row, col, scales,+ sdtype, zero_points, k_offset, n_offset, kblock,+ NPad, reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::decompress_kblock_s7_fp<PackRow, NTILE, DstT>(b4ptr, b2ptr, b1ptr, dstptr, row, col, scales, sdtype,+ zero_points, k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+#endif+ ret = ref::decompress_kblock_s7_fp<PackRow, NTILE, DstT>(b4ptr, b2ptr, b1ptr, dstptr, row, col, scales, sdtype,+ zero_points, k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ return ret;+ }+};++template <int PackRow, int NTILE, typename DstT>+class DecompressKBlockS6Fp {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(utils::bit4x2* b4ptr, utils::bit2x4* b2ptr, DstT* dstptr, int row, int col,+ void* scales, BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int kblock, int NPad, void* tmp, size_t tmpsize) {+ BTLA_CODE ret = BTLA_CODE::NotSupport;+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && NTILE % 16 == 0) {+ return avx512f::decompress_kblock_s6_fp<PackRow, NTILE, DstT>(b4ptr, b2ptr, dstptr, row, col, scales, sdtype,+ zero_points, k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::decompress_kblock_s6_fp<PackRow, NTILE, DstT>(b4ptr, b2ptr, dstptr, row, col, scales, sdtype,+ zero_points, k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+#endif+ ret = ref::decompress_kblock_s6_fp<PackRow, NTILE, DstT>(b4ptr, b2ptr, dstptr, row, col, scales, sdtype,+ zero_points, k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ return ret;+ }+};++template <int PackRow, int NTILE, typename DstT>+class DecompressKBlockS5Fp {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(utils::bit4x2* b4ptr, utils::bit1x8* b1ptr, DstT* dstptr, int row, int col,+ void* scales, BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int kblock, int NPad, void* tmp, size_t tmpsize) {+ BTLA_CODE ret = BTLA_CODE::NotSupport;+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && NTILE % 16 == 0) {+ return avx512f::decompress_kblock_s5_fp<PackRow, NTILE, DstT>(b4ptr, b1ptr, dstptr, row, col, scales, sdtype,+ zero_points, k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::decompress_kblock_s5_fp<PackRow, NTILE, DstT>(b4ptr, b1ptr, dstptr, row, col, scales, sdtype,+ zero_points, k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+#endif+ ret = ref::decompress_kblock_s5_fp<PackRow, NTILE, DstT>(b4ptr, b1ptr, dstptr, row, col, scales, sdtype,+ zero_points, k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ return ret;+ }+};++template <int PackRow, int NTILE, typename DstT>+class DecompressKBlockS4Fp {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(utils::int4x2* srcptr, DstT* dstptr, int row, int col, void* scales,+ BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset, int kblock,+ int NPad, void* tmp, size_t tmpsize) {+ BTLA_CODE ret = BTLA_CODE::NotSupport;+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && NTILE % 16 == 0) {+ return avx512f::decompress_kblock_s4_fp<PackRow, NTILE, DstT>(srcptr, dstptr, row, col, scales, sdtype,+ zero_points, k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::decompress_kblock_s4_fp<PackRow, NTILE, DstT>(srcptr, dstptr, row, col, scales, sdtype, zero_points,+ k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+#endif+ ret = ref::decompress_kblock_s4_fp<PackRow, NTILE, DstT>(srcptr, dstptr, row, col, scales, sdtype, zero_points,+ k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ return ret;+ }+};++template <int PackRow, int NTILE, typename DstT>+class DecompressKBlockS3Fp {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(utils::bit2x4* b2ptr, utils::bit1x8* b1ptr, DstT* dstptr, int row, int col,+ void* scales, BTLA_DTYPE sdtype, int8_t* zero_points, int k_offset, int n_offset,+ int kblock, int NPad, void* tmp, size_t tmpsize) {+ BTLA_CODE ret = BTLA_CODE::NotSupport;+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && NTILE % 16 == 0) {+ return avx512f::decompress_kblock_s3_fp<PackRow, NTILE, DstT>(b2ptr, b1ptr, dstptr, row, col, scales, sdtype,+ zero_points, k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::decompress_kblock_s3_fp<PackRow, NTILE, DstT>(b2ptr, b1ptr, dstptr, row, col, scales, sdtype,+ zero_points, k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+#endif+ ret = ref::decompress_kblock_s3_fp<PackRow, NTILE, DstT>(b2ptr, b1ptr, dstptr, row, col, scales, sdtype,+ zero_points, k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ return ret;+ }+};++template <int PackRow, int NTILE, typename DstT>+class DecompressKBlockS2Fp {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(utils::bit2x4* b2ptr, DstT* dstptr, int row, int col, void* scales, BTLA_DTYPE sdtype,+ int8_t* zero_points, int k_offset, int n_offset, int kblock, int NPad, void* tmp,+ size_t tmpsize) {+ BTLA_CODE ret = BTLA_CODE::NotSupport;+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && NTILE % 16 == 0) {+ return avx512f::decompress_kblock_s2_fp<PackRow, NTILE, DstT>(b2ptr, dstptr, row, col, scales, sdtype,+ zero_points, k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::decompress_kblock_s2_fp<PackRow, NTILE, DstT>(b2ptr, dstptr, row, col, scales, sdtype, zero_points,+ k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+#endif+ ret = ref::decompress_kblock_s2_fp<PackRow, NTILE, DstT>(b2ptr, dstptr, row, col, scales, sdtype, zero_points,+ k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ return ret;+ }+};++template <int PackRow, int NTILE, typename DstT>+class DecompressKBlockS1Fp {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(utils::bit1x8* b1ptr, DstT* dstptr, int row, int col, void* scales, BTLA_DTYPE sdtype,+ int8_t* zero_points, int k_offset, int n_offset, int kblock, int NPad, void* tmp,+ size_t tmpsize) {+ BTLA_CODE ret = BTLA_CODE::NotSupport;+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && NTILE % 16 == 0) {+ return avx512f::decompress_kblock_s1_fp<PackRow, NTILE, DstT>(b1ptr, dstptr, row, col, scales, sdtype,+ zero_points, k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::decompress_kblock_s1_fp<PackRow, NTILE, DstT>(b1ptr, dstptr, row, col, scales, sdtype, zero_points,+ k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+#endif+ ret = ref::decompress_kblock_s1_fp<PackRow, NTILE, DstT>(b1ptr, dstptr, row, col, scales, sdtype, zero_points,+ k_offset, n_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ return ret;+ }+};++template <typename _DST_T, int _PACK_ROW>+class DecompressKBlockF4Fp {+ public:+ template <BTLA_ISA ISA_T, typename SCA_T, BTLA_DTYPE F4_T>+ static inline BTLA_CODE forward(utils::f4x2* srcptr, _DST_T* dstptr, int row, int col, int ld_src, int ld_dst,+ SCA_T* scales, int k_offset, int kblock, int NPad, void* tmp, size_t tmpsize) {+ BTLA_CODE ret = BTLA_CODE::NotSupport;+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ ret = avx512f::decompress_kblock_f4_fp<F4_T, _DST_T, _PACK_ROW, SCA_T>(srcptr, dstptr, row, col, ld_src, ld_dst,+ scales, k_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ if (ret == BTLA_CODE::Success) return ret;+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2 && std::is_same_v<SCA_T, float>) {+ ret = avx2::decompress_kblock_f4_fp<F4_T, _DST_T, _PACK_ROW, SCA_T>(srcptr, dstptr, row, col, ld_src, ld_dst,+ scales, k_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ if (ret == BTLA_CODE::Success) return ret;+ }+#endif+ return ref::decompress_kblock_f4_fp<F4_T, _DST_T, _PACK_ROW, SCA_T>(srcptr, dstptr, row, col, ld_src, ld_dst,+ scales, k_offset, kblock, NPad,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+};++template <typename _DST_T, int _PACK_ROW>+class DecompressDqKBlockF4Fp {+ public:+ template <BTLA_ISA ISA_T, BTLA_DTYPE F4_T, typename SCA_T>+ static inline BTLA_CODE forward(utils::f4x2* srcptr, _DST_T* dstptr, int row, int col, int ld_src, int ld_dst,+ SCA_T* scales, float* dq_scale, int k_offset, int n_offset, int kblock, int dq_blk,+ int dq_offset_idx, int NPad, int N, void* tmp, size_t tmpsize) {+ return ref::decompress_dq_kblock_f4_fp<F4_T, _PACK_ROW>(srcptr, dstptr, row, col, ld_src, ld_dst, scales, dq_scale,+ k_offset, n_offset, kblock, dq_blk, dq_offset_idx, NPad, N,+ tmp, tmpsize);+ }+};++template <typename _DST_T>+class DecompressKBlockF4FpNoscale {+ public:+ template <BTLA_ISA ISA_T, BTLA_DTYPE F4_T>+ static inline BTLA_CODE forward(utils::f4x2* srcptr, _DST_T* dstptr, int row, int col, int ld_src, int ld_dst,+ void* tmp, size_t tmpsize) {+ BTLA_CODE ret = BTLA_CODE::NotSupport;+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::decompress_kblock_f4_fp_noscale<F4_T, _DST_T>(srcptr, dstptr, row, col, ld_src, ld_dst,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::decompress_kblock_f4_fp_noscale<F4_T, _DST_T>(srcptr, dstptr, row, col, ld_src, ld_dst,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+#endif+ return ref::decompress_kblock_f4_fp_noscale<F4_T, _DST_T>(srcptr, dstptr, row, col, ld_src, ld_dst,+ reinterpret_cast<int8_t*>(tmp), tmpsize);+ }+};++template <int PACK_ROW>+class DecompressKBlockF8FP {+ public:+ template <BTLA_ISA ISA_T, typename SCA_T, typename DST_T>+ static inline BTLA_CODE forward(utils::f8* srcptr, DST_T* dstptr, int row, int col, int ld_src, int ld_dst,+ SCA_T* scales, int k_offset, int kblock, int NPad, BTLA_DTYPE src_f8_type) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::decompress_kblock_f8_fp<true, DST_T, PACK_ROW, SCA_T>(+ srcptr, dstptr, row, col, ld_src, ld_dst, scales, k_offset, kblock, NPad, src_f8_type);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::decompress_kblock_f8_fp<true, DST_T, PACK_ROW, SCA_T>(srcptr, dstptr, row, col, ld_src, ld_dst,+ scales, k_offset, kblock, NPad, src_f8_type);+ }+#endif+ return ref::decompress_kblock_f8_fp<DST_T, PACK_ROW, SCA_T>(srcptr, dstptr, row, col, ld_src, ld_dst, scales,+ k_offset, kblock, NPad, src_f8_type);+ }++ template <typename SCA_T, typename DST_T>+ static inline BTLA_CODE forward_auto(utils::f8* srcptr, DST_T* dstptr, int row, int col, int ld_src, int ld_dst,+ SCA_T* scales, int k_offset, int kblock, int NPad, BTLA_DTYPE src_f8_type) {+ GetCPUDevice();+ if (_cd->AVX512F()) {+ return forward<BTLA_ISA::AVX512F>(srcptr, dstptr, row, col, ld_src, ld_dst, scales, k_offset, kblock, NPad,+ src_f8_type);+ }+ if (_cd->AVX2()) {+ return forward<BTLA_ISA::AVX2>(srcptr, dstptr, row, col, ld_src, ld_dst, scales, k_offset, kblock, NPad,+ src_f8_type);+ }+ return forward<BTLA_ISA::NoSIMD>(srcptr, dstptr, row, col, ld_src, ld_dst, scales, k_offset, kblock, NPad,+ src_f8_type);+ }+};++template <typename _DST_T>+class DecompressKBlockF8FpNoScale {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(utils::f8* srcptr, _DST_T* dstptr, int row, int col, int ld_src, int ld_dst,+ void* tmp, size_t tmpsize, BTLA_DTYPE src_f8_t) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::decompress_kblock_f8_fp<false, _DST_T, 1>(+ srcptr, dstptr, row, col, ld_src, ld_dst, reinterpret_cast<utils::f8*>(tmp), -1, -1, -1, src_f8_t);+ }+#endif+#if CompileAVX2()+ if (utils::isa_base<ISA_T>::avx2) {+ return avx2::decompress_kblock_f8_fp<false, _DST_T, 1>(srcptr, dstptr, row, col, ld_src, ld_dst,+ reinterpret_cast<utils::f8*>(tmp), -1, -1, -1, src_f8_t);+ }+#endif+ return ref::decompress_kblock_f8_fp_noscale<_DST_T>(srcptr, dstptr, row, col, ld_src, ld_dst, src_f8_t);+ }+};++class AlphaBetaF32F32 {+ public:+ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(const float alpha, const float* srcptr, const int srcstep, const float beta,+ const float* src1ptr, const int src1step, float* dstptr, const int dststep, const int M,+ const int N) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::alphabeta_f32_f32(alpha, srcptr, srcstep, beta, src1ptr, src1step, dstptr, dststep, M, N);+ }+#endif+#if CompileAVX2()+ if (utils::isa_base<ISA_T>::avx2) {+ return avx2::alphabeta_f32_f32(alpha, srcptr, srcstep, beta, src1ptr, src1step, dstptr, dststep, M, N);+ }+#endif+ return ref::alphabeta_f32_f32(alpha, srcptr, srcstep, beta, src1ptr, src1step, dstptr, dststep, M, N);+ }+};++class CompFp32BlockScale {+ public:+ template <BTLA_ISA ISA_T, typename SCA_T>+ static BTLA_CODE forward(const SCA_T* alpha, const float* srcptr, const int srcstep, float* dstptr, const int dststep,+ const int M, const int N) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::accum_alphaN_f32_f32(alpha, srcptr, srcstep, dstptr, dststep, M, N);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::accum_alphaN_f32_f32(alpha, srcptr, srcstep, dstptr, dststep, M, N);+ }+#endif+ return ref::accum_alphaN_f32_f32(alpha, srcptr, srcstep, dstptr, dststep, M, N);+ }+};++class AccumulateFp32 {+ public:+ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(const float* srcptr, const int srcstep, float* dstptr, const int dststep, const int M,+ const int N) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::accum_f32_f32(srcptr, srcstep, dstptr, dststep, M, N);+ }+#endif+ return ref::accum_f32_f32(srcptr, srcstep, dstptr, dststep, M, N);+ }+};++class QuanOutS32U32 {+ public:+ template <BTLA_ISA ISA_T>+ static BTLA_CODE forward(const float alpha, const int32_t* srcptr, const int srcstep, uint8_t* dstptr,+ const int dststep, const int M, const int N, float scaleSrc, float scaleDst, int zpDst) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::quanout_s32_u32(alpha, srcptr, srcstep, dstptr, dststep, M, N, scaleSrc, scaleDst, zpDst);+ }+#endif+ return ref::quanout_s32_u32(alpha, srcptr, srcstep, dstptr, dststep, M, N, scaleSrc, scaleDst, zpDst);+ }+};++// scaleA ldsa==0 per tensor, ldsa!=0 per M+// scaleB per channel(N)+class DequanS32Fp32 {+ public:+ template <BTLA_ISA ISA_T, typename SCAB_T>+ static BTLA_CODE forward(const int32_t* srcptr, const int srcstep, float* dstptr, const int dststep, const int M,+ const int N, const float* scaleA, const int ldsa, const SCAB_T* scaleB) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::dequant_s32_fp32(srcptr, srcstep, dstptr, dststep, M, N, scaleA, ldsa, scaleB);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::dequant_s32_fp32(srcptr, srcstep, dstptr, dststep, M, N, scaleA, ldsa, scaleB);+ }+#endif+ return ref::dequant_s32_fp32(srcptr, srcstep, dstptr, dststep, M, N, scaleA, ldsa, scaleB);+ }+};++class MinMaxKBlock {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(const float* srcptr, int row, int col, int ld_src, float* minmaxptr, int ld_minmax,+ int fsize_minmax, int blocksize) {+ return ref::minmax_f32_kblock(srcptr, row, col, ld_src, minmaxptr, ld_minmax, fsize_minmax, blocksize);+ }+};++template <typename _RT>+class QuantS8RowReduceSum {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(const int8_t* srcptr, int ldsrc, const float* scales, const int8_t* zero_points,+ int row, int col, _RT* reduce) {+ return ref::quant_s8_row_reduce_sum(srcptr, ldsrc, scales, zero_points, row, col, reduce);+ }+};++template <typename _SRCT, typename _DSTT>+class RowReduceSum {+ public:+ TLACALL BTLA_CODE forward(const _SRCT* srcptr, int ldsrc, int row, int col, _DSTT* reduce) {+ return ref::row_reduce_sum(srcptr, ldsrc, row, col, reduce);+ }+ AUTOCALL BTLA_CODE forward_auto(const _SRCT* srcptr, int ldsrc, int row, int col, _DSTT* reduce) {+ return forward<BTLA_ISA::NoSIMD>(srcptr, ldsrc, row, col, reduce);+ }+};++template <typename SRC_T>+class ColBlockReduceSum {+ public:+ TLACALL BTLA_CODE forward(const SRC_T* srcptr, int ldsrc, int row, int col, int blocksize, float* reduce, int ldr) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && std::is_same_v<SRC_T, float>) {+ return avx512f::col_block_reduce_sum<SRC_T>(srcptr, ldsrc, row, col, blocksize, reduce, ldr);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2 && std::is_same_v<SRC_T, float>) {+ return avx2::col_block_reduce_sum<SRC_T>(srcptr, ldsrc, row, col, blocksize, reduce, ldr);+ }+#endif+ return ref::col_block_reduce_sum<SRC_T>(srcptr, ldsrc, row, col, blocksize, reduce, ldr);+ }++ AUTOCALL BTLA_CODE forward_auto(const SRC_T* srcptr, int ldsrc, int row, int col, int blocksize, float* reduce,+ int ldr) {+ GetCPUDevice();+ if (_cd->AVX512F()) {+ return forward<BTLA_ISA::AVX512F>(srcptr, ldsrc, row, col, blocksize, reduce, ldr);+ }+ if (_cd->AVX2()) {+ return forward<BTLA_ISA::AVX2>(srcptr, ldsrc, row, col, blocksize, reduce, ldr);+ }+ return forward<BTLA_ISA::NoSIMD>(srcptr, ldsrc, row, col, blocksize, reduce, ldr);+ }+};+class RemoveZeroPointBias {+ public:+ template <BTLA_ISA ISA_T, bool ZPA, bool ZPB>+ static inline BTLA_CODE forward_inner(int32_t* accptr, int ldacc, int row, int col, uint8_t* zpa, int8_t* zpb,+ int lda, int k, const int32_t* reducea, const int32_t* reduceb) {+ if constexpr (ZPA) {+ assert(zpa);+ }+ if constexpr (ZPB) {+ assert(zpb);+ }+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::remove_zeropoint_bias<ZPA, ZPB>(accptr, ldacc, row, col, zpa, zpb, lda, k, reducea, reduceb);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::remove_zeropoint_bias<ZPA, ZPB>(accptr, ldacc, row, col, zpa, zpb, lda, k, reducea, reduceb);+ }+#endif+ return ref::remove_zeropoint_bias<ZPA, ZPB>(accptr, ldacc, row, col, zpa, zpb, lda, k, reducea, reduceb);+ }++ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(int32_t* accptr, int ldacc, int row, int col, uint8_t* zpa, int8_t* zpb, int lda,+ int k, const int32_t* reducea, const int32_t* reduceb) {+ if (zpa) {+ if (zpb) {+ return forward_inner<ISA_T, true, true>(accptr, ldacc, row, col, zpa, zpb, lda, k, reducea, reduceb);+ } else {+ return forward_inner<ISA_T, true, false>(accptr, ldacc, row, col, zpa, zpb, lda, k, reducea, reduceb);+ }+ } else {+ if (zpb) {+ return forward_inner<ISA_T, false, true>(accptr, ldacc, row, col, zpa, zpb, lda, k, reducea, reduceb);+ } else {+ assert(0);+ }+ }+ assert(0);+ return BTLA_CODE::NotSupport;+ }+};++class LayerNormalization {+ public:+ template <BTLA_ISA ISA_T, typename T>+ static inline BTLA_CODE forward(const T* srcptr, const T* scaleptr, const T* biasptr, T epsilon, int norm_size,+ T* dstptr, T* mean, T* mean_square, bool simplified) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && std::is_same_v<T, float>) {+ return avx512f::layernorm(srcptr, scaleptr, biasptr, epsilon, norm_size, dstptr, mean, mean_square, simplified);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2 && std::is_same_v<T, float>) {+ return avx2::layernorm(srcptr, scaleptr, biasptr, epsilon, norm_size, dstptr, mean, mean_square, simplified);+ }+#endif+ return ref::layernorm(srcptr, scaleptr, biasptr, epsilon, norm_size, dstptr, mean, mean_square, simplified);+ }+ template <typename T>+ static inline BTLA_CODE forward_auto(const T* srcptr, const T* scaleptr, const T* biasptr, T epsilon, int norm_size,+ T* dstptr, T* mean, T* mean_square, bool simplified) {+ GetCPUDevice();+ if (_cd->AVX512F()) {+ return forward<BTLA_ISA::AVX512F, T>(srcptr, scaleptr, biasptr, epsilon, norm_size, dstptr, mean, mean_square,+ simplified);+ }+ if (_cd->AVX2()) {+ return forward<BTLA_ISA::AVX2, T>(srcptr, scaleptr, biasptr, epsilon, norm_size, dstptr, mean, mean_square,+ simplified);+ }+ return forward<BTLA_ISA::NoSIMD, T>(srcptr, scaleptr, biasptr, epsilon, norm_size, dstptr, mean, mean_square,+ simplified);+ }+};++class GEMVWoqNBits {+ public:+ template <BTLA_ISA ISA_T, typename ScaleT, int NTILE, int MTILE>+ static inline BTLA_CODE forward_u8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, void* tmp, size_t tmpsize) {+ if (B.nbits == 6) {+#if CompileAVX512VNNI()+ if (ISA_T >= BTLA_ISA::AVX512_VNNI && NTILE % 16 == 0) {+ return avx512f::vnni::gemv_6bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVX512F()+ if (ISA_T >= BTLA_ISA::AVX512BW && NTILE % 16 == 0) {+ return avx512f::gemv_6bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVXVNNI()+ if (ISA_T >= BTLA_ISA::AVX_VNNI) {+ return avx2::vnni::gemv_6bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVX2()+ if (ISA_T >= BTLA_ISA::AVX2) {+ return avx2::gemv_6bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_6bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 5) {+#if CompileAVX512VNNI()+ if (ISA_T >= BTLA_ISA::AVX512_VNNI && NTILE % 16 == 0) {+ return avx512f::vnni::gemv_5bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVX512F()+ if (ISA_T >= BTLA_ISA::AVX512BW && NTILE % 16 == 0) {+ return avx512f::gemv_5bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVXVNNI()+ if (ISA_T >= BTLA_ISA::AVX_VNNI) {+ return avx2::vnni::gemv_5bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVX2()+ if (ISA_T >= BTLA_ISA::AVX2) {+ return avx2::gemv_5bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_5bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 4) {+#if CompileAVX512VNNI()+ if (ISA_T >= BTLA_ISA::AVX512_VNNI && NTILE % 16 == 0) {+ return avx512f::vnni::gemv_4bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVX512F()+ if (ISA_T >= BTLA_ISA::AVX512BW && NTILE % 16 == 0) {+ return avx512f::gemv_4bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVXVNNI()+ if (ISA_T >= BTLA_ISA::AVX_VNNI) {+ return avx2::vnni::gemv_4bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVX2()+ if (ISA_T >= BTLA_ISA::AVX2) {+ return avx2::gemv_4bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_4bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 3) {+#if CompileAVX512VNNI()+ if (ISA_T >= BTLA_ISA::AVX512_VNNI && NTILE % 16 == 0) {+ return avx512f::vnni::gemv_3bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVX512F()+ if (ISA_T >= BTLA_ISA::AVX512BW && NTILE % 16 == 0) {+ return avx512f::gemv_3bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVXVNNI()+ if (ISA_T >= BTLA_ISA::AVX_VNNI) {+ return avx2::vnni::gemv_3bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVX2()+ if (ISA_T >= BTLA_ISA::AVX2) {+ return avx2::gemv_3bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_3bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 2) {+#if CompileAVX512VNNI()+ if (ISA_T >= BTLA_ISA::AVX512_VNNI && NTILE % 16 == 0) {+ return avx512f::vnni::gemv_2bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVX512F()+ if (ISA_T >= BTLA_ISA::AVX512BW && NTILE % 16 == 0) {+ return avx512f::gemv_2bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVXVNNI()+ if (ISA_T >= BTLA_ISA::AVX_VNNI) {+ return avx2::vnni::gemv_2bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVX2()+ if (ISA_T >= BTLA_ISA::AVX2) {+ return avx2::gemv_2bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_2bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 7) {+#if CompileAVX512VNNI()+ if (ISA_T >= BTLA_ISA::AVX512_VNNI && NTILE % 16 == 0) {+ return avx512f::vnni::gemv_7bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVX512F()+ if (ISA_T >= BTLA_ISA::AVX512BW && NTILE % 16 == 0) {+ return avx512f::gemv_7bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVXVNNI()+ if (ISA_T >= BTLA_ISA::AVX_VNNI) {+ return avx2::vnni::gemv_7bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVX2()+ if (ISA_T >= BTLA_ISA::AVX2) {+ return avx2::gemv_7bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_7bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 1) {+#if CompileAVX512VNNI()+ if (ISA_T >= BTLA_ISA::AVX512_VNNI && NTILE % 16 == 0) {+ return avx512f::vnni::gemv_1bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVX512F()+ if (ISA_T >= BTLA_ISA::AVX512BW && NTILE % 16 == 0) {+ return avx512f::gemv_1bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVXVNNI()+ if (ISA_T >= BTLA_ISA::AVX_VNNI) {+ return avx2::vnni::gemv_1bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+#if CompileAVX2()+ if (ISA_T >= BTLA_ISA::AVX2) {+ return avx2::gemv_1bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_1bit_u8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ return BTLA_CODE::NotSupport;+ }++ template <BTLA_ISA ISA_T, typename ScaleT, int NTILE, int MTILE>+ static inline BTLA_CODE forward_s8s8_fp32(const utils::GemvParamA& A, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, void* tmp, size_t tmpsize) {+ if (B.nbits == 5) {+#if CompileAVX512VNNI()+ if (ISA_T >= BTLA_ISA::AVX512_VNNI && NTILE % 16 == 0) {+ return avx512f::vnni::gemv_5bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVXVNNI()+ if (ISA_T >= BTLA_ISA::AVX_VNNI) {+ return avx2::vnni::gemv_5bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_5bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 4) {+#if CompileAVX512VNNI()+ if (ISA_T >= BTLA_ISA::AVX512_VNNI && NTILE % 16 == 0) {+ return avx512f::vnni::gemv_4bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVXVNNI()+ if (ISA_T >= BTLA_ISA::AVX_VNNI) {+ return avx2::vnni::gemv_4bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_4bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 3) {+#if CompileAVX512VNNI()+ if (ISA_T >= BTLA_ISA::AVX512_VNNI && NTILE % 16 == 0) {+ return avx512f::vnni::gemv_3bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVXVNNI()+ if (ISA_T >= BTLA_ISA::AVX_VNNI) {+ return avx2::vnni::gemv_3bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_3bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 2) {+#if CompileAVX512VNNI()+ if (ISA_T >= BTLA_ISA::AVX512_VNNI && NTILE % 16 == 0) {+ return avx512f::vnni::gemv_2bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVXVNNI()+ if (ISA_T >= BTLA_ISA::AVX_VNNI) {+ return avx2::vnni::gemv_2bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_2bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 6) {+#if CompileAVX512VNNI()+ if (ISA_T >= BTLA_ISA::AVX512_VNNI && NTILE % 16 == 0) {+ return avx512f::vnni::gemv_6bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVXVNNI()+ if (ISA_T >= BTLA_ISA::AVX_VNNI) {+ return avx2::vnni::gemv_6bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_6bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 7) {+#if CompileAVX512VNNI()+ if (ISA_T >= BTLA_ISA::AVX512_VNNI && NTILE % 16 == 0) {+ return avx512f::vnni::gemv_7bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVXVNNI()+ if (ISA_T >= BTLA_ISA::AVX_VNNI) {+ return avx2::vnni::gemv_7bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_7bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 1) {+#if CompileAVX512VNNI()+ if (ISA_T >= BTLA_ISA::AVX512_VNNI && NTILE % 16 == 0) {+ return avx512f::vnni::gemv_1bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVXVNNI()+ if (ISA_T >= BTLA_ISA::AVX_VNNI) {+ return avx2::vnni::gemv_1bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_1bit_s8s8_fp32<ScaleT, NTILE, MTILE>(A, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ return BTLA_CODE::NotSupport;+ }++ template <BTLA_ISA ISA_T, typename ScaleT, int NTILE, int MTILE>+ static inline BTLA_CODE forward_fp32_fp32(const float* A, int lda, const utils::GemvParamB<ScaleT>& B, float* C,+ int ldc, int k, int blocksize, void* tmp, size_t tmpsize) {+ if (B.nbits == 6) {+#if CompileAVX512F()+ if (ISA_T >= BTLA_ISA::AVX512F && NTILE % 16 == 0) {+ return avx512f::gemv_6bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVX2()+ if (ISA_T >= BTLA_ISA::AVX2) {+ return avx2::gemv_6bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_6bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 5) {+#if CompileAVX512F()+ if (ISA_T >= BTLA_ISA::AVX512F && NTILE % 16 == 0) {+ return avx512f::gemv_5bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVX2()+ if (ISA_T >= BTLA_ISA::AVX2) {+ return avx2::gemv_5bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_5bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 4) {+#if CompileAVX512F()+ if (ISA_T >= BTLA_ISA::AVX512F && NTILE % 16 == 0) {+ return avx512f::gemv_4bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVX2()+ if (ISA_T >= BTLA_ISA::AVX2) {+ return avx2::gemv_4bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_4bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 3) {+#if CompileAVX512F()+ if (ISA_T >= BTLA_ISA::AVX512F && NTILE % 16 == 0) {+ return avx512f::gemv_3bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVX2()+ if (ISA_T >= BTLA_ISA::AVX2) {+ return avx2::gemv_3bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_3bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 2) {+#if CompileAVX512F()+ if (ISA_T >= BTLA_ISA::AVX512F && NTILE % 16 == 0) {+ return avx512f::gemv_2bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVX2()+ if (ISA_T >= BTLA_ISA::AVX2) {+ return avx2::gemv_2bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_2bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 7) {+#if CompileAVX512F()+ if (ISA_T >= BTLA_ISA::AVX512F && NTILE % 16 == 0) {+ return avx512f::gemv_7bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVX2()+ if (ISA_T >= BTLA_ISA::AVX2) {+ return avx2::gemv_7bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_7bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ if (B.nbits == 1) {+#if CompileAVX512F()+ if (ISA_T >= BTLA_ISA::AVX512F && NTILE % 16 == 0) {+ return avx512f::gemv_1bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp,+ tmpsize);+ }+#endif+#if CompileAVX2()+ if (ISA_T >= BTLA_ISA::AVX2) {+ return avx2::gemv_1bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+#endif+ return ref::gemv_1bit_fp32_fp32<ScaleT, NTILE, MTILE>(A, lda, B, C, ldc, k, blocksize, (int8_t*)tmp, tmpsize);+ }+ return BTLA_CODE::NotSupport;+ }+};++template <typename T>+class Mul {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(const T* src0ptr, const T* src1ptr, T* dstptr, size_t size) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::mul(src0ptr, src1ptr, dstptr, size);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::mul(src0ptr, src1ptr, dstptr, size);+ }+#endif+ return ref::mul(src0ptr, src1ptr, dstptr, size);+ }++ static inline BTLA_CODE forward_auto(const T* src0ptr, const T* src1ptr, T* dstptr, size_t size) {+ GetCPUDevice();+ if (_cd->AVX512F()) {+ return forward<BTLA_ISA::AVX512F>(src0ptr, src1ptr, dstptr, size);+ }+ if (_cd->AVX2()) {+ return forward<BTLA_ISA::AVX2>(src0ptr, src1ptr, dstptr, size);+ }+ return forward<BTLA_ISA::NoSIMD>(src0ptr, src1ptr, dstptr, size);+ }+};++template <typename T>+class Inv {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(const T* src0ptr, T* dstptr, size_t size) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::inv(src0ptr, dstptr, size);+ }+#endif+ return ref::inv(src0ptr, dstptr, size);+ }++ static inline BTLA_CODE forward_auto(const T* src0ptr, T* dstptr, size_t size) {+ GetCPUDevice();+ if (_cd->AVX512F()) {+ return forward<BTLA_ISA::AVX512F>(src0ptr, dstptr, size);+ }+ return forward<BTLA_ISA::NoSIMD>(src0ptr, dstptr, size);+ }+};++template <typename T>+class Add {+ public:+ template <BTLA_ISA ISA_T>+ static inline BTLA_CODE forward(const T* src0ptr, const T* src1ptr, T* dstptr, size_t size) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::add(src0ptr, src1ptr, dstptr, size);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::add(src0ptr, src1ptr, dstptr, size);+ }+#endif+ return ref::add(src0ptr, src1ptr, dstptr, size);+ }++ static inline BTLA_CODE forward_auto(const T* src0ptr, const T* src1ptr, T* dstptr, size_t size) {+ GetCPUDevice();+ if (_cd->AVX512F()) {+ return forward<BTLA_ISA::AVX512F>(src0ptr, src1ptr, dstptr, size);+ }+ if (_cd->AVX2()) {+ return forward<BTLA_ISA::AVX2>(src0ptr, src1ptr, dstptr, size);+ }+ return forward<BTLA_ISA::NoSIMD>(src0ptr, src1ptr, dstptr, size);+ }+};++template <typename T_DST>+class ScaleExpAccSumFp32 {+ public:+ TLACALL BTLA_CODE forward(const float* src, const int src_step, T_DST* dst, int ld_dst, float* dst_sum,+ const int M_offset, const int N_offset, const int M, const int N, float scale,+ int causal_offset, void* tmpcache, size_t cachesize) {+ dst = dst + M_offset * ld_dst + N_offset;+ dst_sum = dst_sum + M_offset;+#if CompileBF16()+ if constexpr (utils::isa_base<ISA_T>::avx512_bf16 && std::is_same_v<T_DST, utils::bf16>) {+ return avx512f::avx512_bf16::scale_exp_acc_sum_fp32(src, src_step, dst, ld_dst, dst_sum, M_offset, N_offset, M, N,+ scale, causal_offset, tmpcache, cachesize);+ }+#endif+ return ref::scale_exp_acc_sum_fp32(src, src_step, dst, ld_dst, dst_sum, M_offset, N_offset, M, N, scale,+ causal_offset, tmpcache, cachesize);+ }+};++template <typename T_SRC, typename T_DST>+class ScaleTrackMax {+ public:+ using DType = T_DST;+ using SType = T_SRC;++ TLACALL BTLA_CODE forward(const SType* src, const int src_step, DType* dst, DType* dst_max, int ld_dst,+ const int M_offset, const int N_offset, const int M, const int N, float scale,+ int causal_offset, float alibi_slope, float tanh_scale, int padding_type, void* tmpcache,+ size_t cachesize) {+ dst = dst + M_offset * ld_dst + N_offset;+ dst_max = dst_max + M_offset;+ if constexpr (std::is_same_v<T_DST, float>) {+ if constexpr (std::is_same_v<T_SRC, utils::fp16>) {+ assert(padding_type != 2);+ assert(("alibi not supported!", alibi_slope == 0.f));+ assert(("tanh not supported!", tanh_scale == 0.f));+#if CompileFP16()+ if constexpr (utils::isa_base<ISA_T>::avx512_fp16) {+ return avx512f::avx512_fp16::scale_track_max_fp16_fp32(src, src_step, dst, dst_max, ld_dst, M_offset,+ N_offset, M, N, scale, causal_offset, alibi_slope,+ tanh_scale, tmpcache, cachesize);+ }+#endif+ }+ if constexpr (std::is_same_v<T_SRC, float>) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return forward_avx512(src, src_step, dst, dst_max, ld_dst, M_offset, N_offset, M, N, scale, causal_offset,+ alibi_slope, tanh_scale, padding_type, tmpcache, cachesize);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return forward_avx2(src, src_step, dst, dst_max, ld_dst, M_offset, N_offset, M, N, scale, causal_offset,+ alibi_slope, tanh_scale, padding_type, tmpcache, cachesize);+ }+#endif+ }+ if constexpr (std::is_same_v<T_SRC, int32_t>) {+ assert(padding_type != 2);+ assert(("alibi not supported!", alibi_slope == 0.f));+ assert(("tanh not supported!", tanh_scale == 0.f));+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::scale_track_max_int32_fp32(src, src_step, dst, dst_max, ld_dst, M_offset, N_offset, M, N,+ scale, causal_offset, alibi_slope, tanh_scale, tmpcache,+ cachesize);+ }+#endif+ }+ }++ return ref::scale_track_max<T_SRC, T_DST>(src, src_step, dst, dst_max, ld_dst, M_offset, N_offset, M, N, scale,+ causal_offset, alibi_slope, tanh_scale, padding_type, tmpcache,+ cachesize);+ }++ static BTLA_CODE forward_avx2(const SType* src, const int src_step, DType* dst, DType* dst_max, int ld_dst,+ const int M_offset, const int N_offset, const int M, const int N, float scale,+ int causal_offset, float alibi_slope, float tanh_scale, int padding_type,+ void* tmpcache, size_t cachesize) {+#if CompileAVX2()+ if (alibi_slope == 0 && tanh_scale == 0)+ return avx2::scale_track_max_fp32_fp32<false, false>(src, src_step, dst, dst_max, ld_dst, M_offset, N_offset, M,+ N, scale, causal_offset, alibi_slope, tanh_scale,+ padding_type, tmpcache, cachesize);+ else if (alibi_slope == 0 && tanh_scale != 0)+ return avx2::scale_track_max_fp32_fp32<false, true>(src, src_step, dst, dst_max, ld_dst, M_offset, N_offset, M, N,+ scale, causal_offset, alibi_slope, tanh_scale, padding_type,+ tmpcache, cachesize);+ else if (alibi_slope != 0 && tanh_scale == 0)+ return avx2::scale_track_max_fp32_fp32<true, false>(src, src_step, dst, dst_max, ld_dst, M_offset, N_offset, M, N,+ scale, causal_offset, alibi_slope, tanh_scale, padding_type,+ tmpcache, cachesize);++#endif+ return BTLA_CODE::NotSupport;+ }++ static BTLA_CODE forward_avx512(const SType* src, const int src_step, DType* dst, DType* dst_max, int ld_dst,+ const int M_offset, const int N_offset, const int M, const int N, float scale,+ int causal_offset, float alibi_slope, float tanh_scale, int padding_type,+ void* tmpcache, size_t cachesize) {+#if CompileAVX512F()+ if (alibi_slope == 0 && tanh_scale == 0)+ return avx512f::scale_track_max_fp32_fp32<false, false>(src, src_step, dst, dst_max, ld_dst, M_offset, N_offset,+ M, N, scale, causal_offset, alibi_slope, tanh_scale,+ padding_type, tmpcache, cachesize);+ else if (alibi_slope == 0 && tanh_scale != 0)+ return avx512f::scale_track_max_fp32_fp32<false, true>(src, src_step, dst, dst_max, ld_dst, M_offset, N_offset, M,+ N, scale, causal_offset, alibi_slope, tanh_scale,+ padding_type, tmpcache, cachesize);+ else if (alibi_slope != 0 && tanh_scale == 0)+ return avx512f::scale_track_max_fp32_fp32<true, false>(src, src_step, dst, dst_max, ld_dst, M_offset, N_offset, M,+ N, scale, causal_offset, alibi_slope, tanh_scale,+ padding_type, tmpcache, cachesize);+#endif+ return BTLA_CODE::NotSupport;+ }+};++template <typename T_DST>+class WeightCvtBf16Ntile48 {+ public:+ using BType = T_DST;+ using SType = utils::bf16;++ TLACALL BTLA_CODE forward(const SType* B, int ldb, bool is_padded, BType* dst_ptr, int dst_step, int k_size,+ int n_size, int k_offset, int n_offset, void* tmpcache, size_t cachesize) {+ assert(is_padded);+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f && std::is_same_v<BType, float>) {+ return avx512f::weight_cvt_bf16_fp32_n48(B, ldb, is_padded, dst_ptr, dst_step, k_size, n_size, k_offset, n_offset,+ tmpcache, cachesize);+ }+#endif+ return BTLA_CODE::NotSupport;+ }+};++template <typename T_DST>+class WeightCvtFp16Ntile24 {+ public:+ using BType = T_DST;+ using SType = utils::fp16;++ TLACALL BTLA_CODE forward(const SType* B, int ldb, bool is_padded, BType* dst_ptr, int dst_step, int k_size,+ int n_size, int k_offset, int n_offset, void* tmpcache, size_t cachesize) {+ assert(is_padded);+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2 && std::is_same_v<BType, float>) {+ return avx2::weight_cvt_fp16_fp32_n24(B, ldb, is_padded, dst_ptr, dst_step, k_size, n_size, k_offset, n_offset,+ tmpcache, cachesize);+ }+#endif+ return BTLA_CODE::NotSupport;+ }+};++template <class SRC_T, class DST_T>+struct InplacePrecomputeMaxSoftmax {+ // nsize is the staring n-size when causal mask enabled+ // src and dst cam be on the same address if sizeof(SRC_T) >= sizeof(DST_T) and ld is correctly set+ // s_max and expsum cam be on the same address+ TLACALL BTLA_CODE forward(int m_size, int n_size, int n_pad_size, bool is_causal, SRC_T* src, DST_T* dst,+ const SRC_T* s_max, float* expsum, int ld_src, int ld_dst) {+ if constexpr (std::is_same_v<SRC_T, float>) {+ if constexpr (std::is_same_v<DST_T, float>) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::inplace_precompute_max_softmax_fp32_fp32(m_size, n_size, n_pad_size, is_causal, src, dst,+ s_max, expsum, ld_src, ld_dst);+ }+#endif+#if CompileAVX2()+ if constexpr (utils::isa_base<ISA_T>::avx2) {+ return avx2::inplace_precompute_max_softmax_fp32_fp32(m_size, n_size, n_pad_size, is_causal, src, dst, s_max,+ expsum, ld_src, ld_dst);+ }+#endif+ }+ if constexpr (std::is_same_v<DST_T, utils::fp16>) {+#if CompileFP16()+ if constexpr (utils::isa_base<ISA_T>::avx512_fp16) {+ return avx512f::avx512_fp16::inplace_precompute_max_softmax_fp32_fp16(+ m_size, n_size, n_pad_size, is_causal, src, dst, s_max, expsum, ld_src, ld_dst);+ }+#endif+ }+ if constexpr (std::is_same_v<DST_T, utils::bf16>) {+#if CompileBF16()+ if constexpr (utils::isa_base<ISA_T>::avx512_bf16) {+ return avx512f::avx512_bf16::inplace_precompute_max_softmax_fp32_bf16(+ m_size, n_size, n_pad_size, is_causal, src, dst, s_max, expsum, ld_src, ld_dst);+ }+#endif+ }+ if constexpr (std::is_same_v<DST_T, uint8_t>) {+#if CompileAVX512F()+ if constexpr (utils::isa_base<ISA_T>::avx512f) {+ return avx512f::inplace_precompute_max_softmax_fp32_u8(m_size, n_size, n_pad_size, is_causal, src, dst, s_max,+ expsum, ld_src, ld_dst);+ }+#endif+ }+ }+ assert(false);+ return BTLA_CODE::NotSupport;+ }+};+} // namespace wrapper+} // namespace kernel+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/fp8_lut.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/fp8_lut.hnew file mode 100644index 000000000..c1d1a010e--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/fp8_lut.h@@ -0,0 +1,183 @@+#pragma once++#include <array>+#include <cmath>++namespace bestla {+namespace sycl_prologue_b {+namespace fp8_lut {++// NOTE:+// - Kernels index LUT with (u8 & 0x7F) and apply sign separately.+// - Therefore we only need entries [0..127] for the magnitude.+// - Special values are kept for correctness (E4M3: NaN at 0x7F; E5M2: Inf/NaN at 0x7C..0x7F).++inline constexpr std::array<float, 128> lut_e4m3_128 = {+ 0.0f, 0.001953125f, 0.00390625f, 0.005859375f,+ 0.0078125f, 0.009765625f, 0.01171875f, 0.013671875f,+ 0.015625f, 0.017578125f, 0.01953125f, 0.021484375f,+ 0.0234375f, 0.025390625f, 0.02734375f, 0.029296875f,+ 0.03125f, 0.03515625f, 0.0390625f, 0.04296875f,+ 0.046875f, 0.05078125f, 0.0546875f, 0.05859375f,+ 0.0625f, 0.0703125f, 0.078125f, 0.0859375f,+ 0.09375f, 0.1015625f, 0.109375f, 0.1171875f,+ 0.125f, 0.140625f, 0.15625f, 0.171875f,+ 0.1875f, 0.203125f, 0.21875f, 0.234375f,+ 0.25f, 0.28125f, 0.3125f, 0.34375f,+ 0.375f, 0.40625f, 0.4375f, 0.46875f,+ 0.5f, 0.5625f, 0.625f, 0.6875f,+ 0.75f, 0.8125f, 0.875f, 0.9375f,+ 1.0f, 1.125f, 1.25f, 1.375f,+ 1.5f, 1.625f, 1.75f, 1.875f,+ 2.0f, 2.25f, 2.5f, 2.75f,+ 3.0f, 3.25f, 3.5f, 3.75f,+ 4.0f, 4.5f, 5.0f, 5.5f,+ 6.0f, 6.5f, 7.0f, 7.5f,+ 8.0f, 9.0f, 10.0f, 11.0f,+ 12.0f, 13.0f, 14.0f, 15.0f,+ 16.0f, 18.0f, 20.0f, 22.0f,+ 24.0f, 26.0f, 28.0f, 30.0f,+ 32.0f, 36.0f, 40.0f, 44.0f,+ 48.0f, 52.0f, 56.0f, 60.0f,+ 64.0f, 72.0f, 80.0f, 88.0f,+ 96.0f, 104.0f, 112.0f, 120.0f,+ 128.0f, 144.0f, 160.0f, 176.0f,+ 192.0f, 208.0f, 224.0f, 240.0f,+ 256.0f, 288.0f, 320.0f, 352.0f,+ 384.0f, 416.0f, 448.0f, std::numeric_limits<float>::quiet_NaN(),+};++inline constexpr std::array<float, 128> lut_e5m2_128 = {+ 0.0f,+ 1.52587890625e-05f,+ 3.0517578125e-05f,+ 4.57763671875e-05f,+ 6.103515625e-05f,+ 7.62939453125e-05f,+ 9.1552734375e-05f,+ 0.0001068115234375f,+ 0.0001220703125f,+ 0.000152587890625f,+ 0.00018310546875f,+ 0.000213623046875f,+ 0.000244140625f,+ 0.00030517578125f,+ 0.0003662109375f,+ 0.00042724609375f,+ 0.00048828125f,+ 0.0006103515625f,+ 0.000732421875f,+ 0.0008544921875f,+ 0.0009765625f,+ 0.001220703125f,+ 0.00146484375f,+ 0.001708984375f,+ 0.001953125f,+ 0.00244140625f,+ 0.0029296875f,+ 0.00341796875f,+ 0.00390625f,+ 0.0048828125f,+ 0.005859375f,+ 0.0068359375f,+ 0.0078125f,+ 0.009765625f,+ 0.01171875f,+ 0.013671875f,+ 0.015625f,+ 0.01953125f,+ 0.0234375f,+ 0.02734375f,+ 0.03125f,+ 0.0390625f,+ 0.046875f,+ 0.0546875f,+ 0.0625f,+ 0.078125f,+ 0.09375f,+ 0.109375f,+ 0.125f,+ 0.15625f,+ 0.1875f,+ 0.21875f,+ 0.25f,+ 0.3125f,+ 0.375f,+ 0.4375f,+ 0.5f,+ 0.625f,+ 0.75f,+ 0.875f,+ 1.0f,+ 1.25f,+ 1.5f,+ 1.75f,+ 2.0f,+ 2.5f,+ 3.0f,+ 3.5f,+ 4.0f,+ 5.0f,+ 6.0f,+ 7.0f,+ 8.0f,+ 10.0f,+ 12.0f,+ 14.0f,+ 16.0f,+ 20.0f,+ 24.0f,+ 28.0f,+ 32.0f,+ 40.0f,+ 48.0f,+ 56.0f,+ 64.0f,+ 80.0f,+ 96.0f,+ 112.0f,+ 128.0f,+ 160.0f,+ 192.0f,+ 224.0f,+ 256.0f,+ 320.0f,+ 384.0f,+ 448.0f,+ 512.0f,+ 640.0f,+ 768.0f,+ 896.0f,+ 1024.0f,+ 1280.0f,+ 1536.0f,+ 1792.0f,+ 2048.0f,+ 2560.0f,+ 3072.0f,+ 3584.0f,+ 4096.0f,+ 5120.0f,+ 6144.0f,+ 7168.0f,+ 8192.0f,+ 10240.0f,+ 12288.0f,+ 14336.0f,+ 16384.0f,+ 20480.0f,+ 24576.0f,+ 28672.0f,+ 32768.0f,+ 40960.0f,+ 49152.0f,+ 57344.0f,+ std::numeric_limits<float>::infinity(),+ std::numeric_limits<float>::quiet_NaN(),+ std::numeric_limits<float>::quiet_NaN(),+ std::numeric_limits<float>::quiet_NaN(),+};++} // namespace fp8_lut+} // namespace sycl_prologue_b+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_device.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_device.hnew file mode 100644index 000000000..96f819fd7--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_device.h@@ -0,0 +1,123 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include <map>+#include <thread>+#include <vector>+#include <sycl/sycl.hpp>++namespace bestla {++namespace sycl_device {++class SyclDevice {+ public:+ SyclDevice(int gpu_idx, bool profile) {+ // Create an exception handler for asynchronous SYCL exceptions+ static auto exception_handler = [](sycl::exception_list e_list) {+ for (std::exception_ptr const& e : e_list) {+ try {+ std::rethrow_exception(e);+ } catch (std::exception const& e) {+#if _DEBUG+ std::cout << "Failure" << std::endl;+#endif+ std::terminate();+ }+ }+ };+ auto devices = sycl::device::get_devices(sycl::info::device_type::gpu);+ assert(gpu_idx < devices.size());++ if (profile) {+ sycl::property_list prop = {sycl::property::queue::enable_profiling(), sycl::property::queue::in_order()};+ mQueue = sycl::queue(devices[gpu_idx], exception_handler, prop);+ } else {+ sycl::property_list prop = {sycl::property::queue::in_order()};+ mQueue = sycl::queue(devices[gpu_idx], exception_handler);+ }+ }++ SyclDevice(bool profile) {+ // Create an exception handler for asynchronous SYCL exceptions+ static auto exception_handler = [](sycl::exception_list e_list) {+ for (std::exception_ptr const& e : e_list) {+ try {+ std::rethrow_exception(e);+ } catch (std::exception const& e) {+#if _DEBUG+ std::cout << "Failure" << std::endl;+#endif+ std::terminate();+ }+ }+ };+ auto d_selector{sycl::default_selector_v};+ if (profile) {+ sycl::property_list prop = {sycl::property::queue::enable_profiling(), sycl::property::queue::in_order()};+ mQueue = sycl::queue(d_selector, exception_handler, prop);+ } else {+ sycl::property_list prop = {sycl::property::queue::in_order()};+ mQueue = sycl::queue(d_selector, exception_handler);+ }+ }++ inline sycl::queue* getQueue() { return &mQueue; }++ inline std::string getName() { return mQueue.get_device().get_info<sycl::info::device::name>(); };++ size_t getGlobalMemSize() { return mQueue.get_device().get_info<sycl::info::device::global_mem_size>(); }+ size_t getMaxMemAllocSize() { return mQueue.get_device().get_info<sycl::info::device::max_mem_alloc_size>(); }++ double getGlobalMemSizeGB() { return double(getGlobalMemSize()) / 1e9; }+ double getMaxMemAllocSizeMB() { return double(getGlobalMemSize()) / 1e6; }++ static inline bool is_cpu(const sycl::device& dev) {+ return dev.get_info<sycl::info::device::device_type>() == sycl::info::device_type::cpu;+ }++ static inline bool is_gpu(const sycl::device& dev) {+ return dev.get_info<sycl::info::device::device_type>() == sycl::info::device_type::gpu;+ }++ static inline bool is_cpu(sycl::queue* q) {+ return q->get_device().get_info<sycl::info::device::device_type>() == sycl::info::device_type::cpu;+ }++ static inline bool is_gpu(sycl::queue* q) {+ return q->get_device().get_info<sycl::info::device::device_type>() == sycl::info::device_type::gpu;+ }++ void print() {+ std::cout << "Running on device: " << mQueue.get_device().get_info<sycl::info::device::name>() << "\n";+ if (is_gpu(mQueue.get_device())) {+ std::cout << "EU count:" << mQueue.get_device().get_info<sycl::ext::intel::info::device::gpu_eu_count>() << "\n";+ std::cout << "EU count per subslice:"+ << mQueue.get_device().get_info<sycl::ext::intel::info::device::gpu_eu_count_per_subslice>() << "\n";+ std::cout << "EU SIMD width:" << mQueue.get_device().get_info<sycl::ext::intel::info::device::gpu_eu_simd_width>()+ << "\n";+ std::cout << "HW threads per EU:"+ << mQueue.get_device().get_info<sycl::ext::intel::info::device::gpu_hw_threads_per_eu>() << "\n";+ std::cout << "GPU slices:" << mQueue.get_device().get_info<sycl::ext::intel::info::device::gpu_slices>() << "\n";+ std::cout << "Subslice per slice:"+ << mQueue.get_device().get_info<sycl::ext::intel::info::device::gpu_subslices_per_slice>() << "\n";+ }+ std::cout << "Global Memory size: " << getGlobalMemSizeGB() << "\n";+ std::cout << "Global Memory size: " << getMaxMemAllocSize() << "\n";+ }+ sycl::queue mQueue;+};++} // namespace sycl_device+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_epilogue.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_epilogue.hnew file mode 100644index 000000000..3fe143450--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_epilogue.h@@ -0,0 +1,56 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once++#ifdef BTLA_SYCL+#include <sycl/sycl.hpp>++#include "sycl_utils.h"++namespace bestla {+namespace sycl_epilogue {+template <typename DstT>+struct ParamOutputBase {+ DstT* C;+ int ldc;+};+template <class GemmCoreT, typename DstT>+class OutputBase {+ public:+ using CType = typename GemmCoreT::TACC;+ using DstType = DstT;+ using Param = ParamOutputBase<DstType>;+ static void store(const Param& _param, CType* tmpAcc, const sycl_utils::nd_item_helper<GemmCoreT>& helper) {+#pragma unroll+ for (int im = 0; im < GemmCoreT::TileM; im++) {+ *(sycl::vec<CType, GemmCoreT::TileN>*)&_param.C[(helper.item_g_m() + im) * _param.ldc + helper.item_g_n()] =+ *(sycl::vec<CType, GemmCoreT::TileN>*)&tmpAcc[(im)*GemmCoreT::TileN];+ }+ }++ static void store_tail(const Param& _param, CType* tmpAcc, const sycl_utils::nd_item_helper<GemmCoreT>& helper,+ int m_tail) {+ if (m_tail) {+#pragma unroll(4)+ for (int im = 0; im < m_tail; im += 1) {+ *(sycl::vec<CType, GemmCoreT::TileN>*)&_param.C[(helper.item_g_m() + im) * _param.ldc + helper.item_g_n()] =+ *(sycl::vec<CType, GemmCoreT::TileN>*)&tmpAcc[(im)*GemmCoreT::TileN];+ }+ }+ }+};++} // namespace sycl_epilogue+} // namespace bestla+#endifdiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_gemm.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_gemm.hnew file mode 100644index 000000000..2524f6693--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_gemm.h@@ -0,0 +1,1225 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once++#ifdef BTLA_SYCL+#include <array>++#include "bestla/bestla_utils.h"+#include <sycl/sycl.hpp>+#include <sycl/ext/intel/experimental/grf_size_properties.hpp>++namespace bestla {++namespace syclex = sycl::ext::oneapi::experimental;+namespace syclintelex = sycl::ext::intel::experimental;+namespace sycl_gemm {++struct GemmParam {+ void *A_d, *B_d;+ void* C_d;+ int m, n, k, lda, ldb, ldc;+ void* Bias = nullptr;+};+namespace xve {++template <class ConfigT>+class GemmCoreSharedB {+ public:+ using Helper_t = sycl_utils::nd_item_helper<GemmCoreSharedB<ConfigT>>;+ static int constexpr SgSize = ConfigT::sg_size;+ static int constexpr WgM = ConfigT::wg_m;+ static int constexpr WgN = ConfigT::wg_n;+ static int constexpr SgNStride = WgN / SgSize;+ static int constexpr WgWorkers = WgM * WgN;+ static int constexpr SgCount = WgWorkers / SgSize;+ static int constexpr TileM = ConfigT::sg_m;+ static int constexpr TileN = ConfigT::sg_n;+ static int constexpr TileK = ConfigT::sg_k;+ static int constexpr UnrollK = ConfigT::unroll_k;+ static int constexpr WgNEle = WgN * TileN;+ static int constexpr SLM_B_STRIDE = (WgNEle + 8);+ static int constexpr WgMEle = WgM * TileM;+ static int constexpr SgNEle = SgSize * TileN;+ static int constexpr SLM_B_Size = SLM_B_STRIDE * TileK;+ static int constexpr SLM_A_Size = 0;++ using TA = typename ConfigT::data_type_a;+ using TB = typename ConfigT::data_type_b;+ using TC = typename ConfigT::data_type_c;+ using TACC = typename ConfigT::data_type_acc;++ using SLM_B_Acc = sycl::local_accessor<TB, 1>;++ using AType = TA;+ using BType = TB;+ using CType = TC;+ static auto constexpr NTILE = WgNEle;+ static auto constexpr MTILE = WgMEle;+ static auto constexpr KTILE = TileK;+ static auto constexpr PACK_ROW = 1;+ static int constexpr PREFERRED_N = NTILE;+ static auto constexpr ISA = BTLA_ISA::ISA_COUNT;+ static auto constexpr ID = 0;++ static inline void compute(const TA* aptr, int lda, const SLM_B_Acc& bacc, TACC* accptr, const Helper_t& helper) {+#pragma unroll(1)+ for (int ik = 0; ik < TileK; ik += UnrollK) {+ int constexpr MReg = TileM / SgSize;+ TA regA[UnrollK * MReg];+ for (int im = 0; im < MReg; im++) {+ *(sycl::vec<TA, UnrollK>*)®A[im * UnrollK] =+ *(sycl::vec<TA, UnrollK>*)&aptr[(helper.sg_id() + im * SgSize) * lda + ik];+ }+#pragma unroll+ for (int ikk = 0; ikk < UnrollK; ikk++) {+ TB tmpB[TileN];+#pragma unroll+ for (int in = 0; in < TileN; in++) {+ tmpB[in] = bacc[helper.sg_idx_n() * SgNEle + helper.sg_id() * TileN + in + (ik + ikk) * SLM_B_STRIDE];+ }+#pragma unroll+ for (size_t im = 0; im < TileM; im++) {+ auto tmpA = sycl::select_from_group(helper.sg, regA[ikk + im / SgSize * UnrollK], im % SgSize);+#pragma unroll+ for (size_t in = 0; in < TileN; in++) {+ accptr[im * TileN + in] += tmpA * tmpB[in];+ }+ }+ }+ }+ }++ static inline void compute_mtail(const TA* aptr, int lda, const SLM_B_Acc& bacc, TACC* accptr, const Helper_t& helper,+ int& m_tail) {+ if (m_tail > 0) {+#pragma unroll(1)+ for (int ik = 0; ik < TileK; ik += UnrollK) {+ int constexpr MReg = TileM / SgSize;+ TA regA[UnrollK * MReg];+ for (int im = 0; im < MReg; im++) {+ if ((helper.sg_id() + im * SgSize) < m_tail) {+ *(sycl::vec<TA, UnrollK>*)®A[im * UnrollK] =+ *(sycl::vec<TA, UnrollK>*)&aptr[(helper.sg_id() + im * SgSize) * lda + ik];+ }+ }++#pragma unroll+ for (int ikk = 0; ikk < UnrollK; ikk++) {+ TB tmpB[TileN];+#pragma unroll+ for (int in = 0; in < TileN; in++) {+ tmpB[in] = bacc[helper.sg_idx_n() * SgNEle + helper.sg_id() * TileN + in + (ik + ikk) * SLM_B_STRIDE];+ }+ for (size_t im = 0; im < TileM; im++) {+ auto tmpA = sycl::select_from_group(helper.sg, regA[ikk + im / SgSize * UnrollK], im % SgSize);+#pragma unroll+ for (size_t in = 0; in < TileN; in++) {+ accptr[im * TileN + in] += tmpA * tmpB[in];+ }+ }+ }+ }+ }+ }+};++class Config_Fp32Fp32Fp32 {+ public:+ static int constexpr sg_size = 32;+ static int constexpr sg_m = 32;+ static int constexpr sg_n = 1;+ static int constexpr sg_k = 16;+ static int constexpr unroll_k = 4;+ static int constexpr wg_m = 8;+ static int constexpr wg_n = 64;++ using data_type_a = float;+ using data_type_b = float;+ using data_type_c = float;+ using data_type_acc = float;+};++class Config_Bf16Bf16Bf16 {+ public:+ static int constexpr sg_size = 32;+ static int constexpr sg_m = 32;+ static int constexpr sg_n = 1;+ static int constexpr sg_k = 16;+ static int constexpr unroll_k = 4;+ static int constexpr wg_m = 8;+ static int constexpr wg_n = 64;++ using data_type_a = sycl::ext::oneapi::bfloat16;+ using data_type_b = sycl::ext::oneapi::bfloat16;+ using data_type_c = sycl::ext::oneapi::bfloat16;+ using data_type_acc = sycl::ext::oneapi::bfloat16;+};++class Config_Fp16Fp16Fp16 {+ public:+ static int constexpr sg_size = 32;+ static int constexpr sg_m = 32;+ static int constexpr sg_n = 2;+ static int constexpr sg_k = 16;+ static int constexpr unroll_k = 8;+ static int constexpr wg_m = 4;+ static int constexpr wg_n = 64;++ using data_type_a = sycl::half;+ using data_type_b = sycl::half;+ using data_type_c = sycl::half;+ using data_type_acc = sycl::half;+};+using DefaultSGemmCore = GemmCoreSharedB<Config_Fp32Fp32Fp32>;+using DefaultHGemmCore = GemmCoreSharedB<Config_Fp16Fp16Fp16>;+using DefaultHGemmCoreBF16 = GemmCoreSharedB<Config_Bf16Bf16Bf16>;++template <class ConfigT>+class GemmCoreSharedBT {+ public:+ using Helper_t = sycl_utils::nd_item_helper<GemmCoreSharedBT<ConfigT>>;+ static int constexpr SgSize = ConfigT::sg_size;+ static int constexpr WgM = ConfigT::wg_m;+ static int constexpr WgN = ConfigT::wg_n;+ static int constexpr SgNStride = WgN / SgSize;+ static int constexpr WgWorkers = WgM * WgN;+ static int constexpr SgCount = WgWorkers / SgSize;+ static int constexpr TileM = ConfigT::sg_m;+ static int constexpr TileN = ConfigT::sg_n;+ static int constexpr TileK = ConfigT::sg_k;+ static int constexpr UnrollK = ConfigT::unroll_k;+ static int constexpr WgNEle = WgN * TileN;+ static int constexpr SLM_B_STRIDE = WgNEle;+ static int constexpr WgMEle = WgM * TileM;+ static int constexpr SgNEle = SgSize * TileN;+ static int constexpr SLM_B_Size = SLM_B_STRIDE * TileK;+ static int constexpr SLM_A_Size = 0;++ using TA = typename ConfigT::data_type_a;+ using TB = typename ConfigT::data_type_b;+ using TC = typename ConfigT::data_type_c;+ using TACC = typename ConfigT::data_type_acc;++ using SLM_B_Acc = sycl::local_accessor<TB, 1>;++ using AType = TA;+ using BType = TB;+ using CType = TC;+ static auto constexpr NTILE = WgNEle;+ static auto constexpr MTILE = WgMEle;+ static auto constexpr KTILE = TileK;+ static auto constexpr PACK_ROW = 1;+ static int constexpr PREFERRED_N = NTILE;+ static auto constexpr ISA = BTLA_ISA::ISA_COUNT;+ static auto constexpr ID = 0;++ static inline void compute(const TA* aptr, int lda, const SLM_B_Acc& bacc, TACC* accptr, const Helper_t& helper) {+#pragma unroll(1)+ for (int ik = 0; ik < TileK; ik += UnrollK) {+ int constexpr MReg = TileM / SgSize;+ TA regA[UnrollK * MReg];+ for (int im = 0; im < MReg; im++) {+ *(sycl::vec<TA, UnrollK>*)®A[im * UnrollK] =+ *(sycl::vec<TA, UnrollK>*)&aptr[(helper.sg_id() + im * SgSize) * lda + ik];+ }+ TB tmpB[UnrollK * TileN];+#pragma unroll+ for (int in = 0; in < TileN; in++) {+ *(sycl::vec<TB, UnrollK>*)&tmpB[in * UnrollK] =+ *(sycl::vec<TB, UnrollK>*)&bacc[(helper.sg_idx_n() * SgNEle + helper.sg_id() * TileN + in) * UnrollK ++ ik * SLM_B_STRIDE];+ }+ constexpr int unrk = std::is_same_v<TB, float> ? 8 : 8;+#pragma unroll(unrk)+ for (int ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (size_t im = 0; im < TileM; im++) {+ auto tmpA = sycl::select_from_group(helper.sg, regA[ikk + im / SgSize * UnrollK], im % SgSize);+#pragma unroll+ for (size_t in = 0; in < TileN; in++) {+ accptr[im * TileN + in] += tmpA * tmpB[in * UnrollK + ikk];+ }+ }+ }+ }+ }++ static inline void compute_mtail(const TA* aptr, int lda, const SLM_B_Acc& bacc, TACC* accptr, const Helper_t& helper,+ int& m_tail) {+ if (m_tail > 0) {+#pragma unroll(1)+ for (int ik = 0; ik < TileK; ik += UnrollK) {+ int constexpr MReg = TileM / SgSize;+ TA regA[UnrollK * MReg];+#pragma unroll+ for (int im = 0; im < MReg; im++) {+ if ((helper.sg_id() + im * SgSize) < m_tail) {+ *(sycl::vec<TA, UnrollK>*)®A[im * UnrollK] =+ *(sycl::vec<TA, UnrollK>*)&aptr[(helper.sg_id() + im * SgSize) * lda + ik];+ }+ }+ TB tmpB[UnrollK * TileN];+#pragma unroll+ for (int in = 0; in < TileN; in++) {+ *(sycl::vec<TB, UnrollK>*)&tmpB[in * UnrollK] =+ *(sycl::vec<TB, UnrollK>*)&bacc[(helper.sg_idx_n() * SgNEle + helper.sg_id() * TileN + in) * UnrollK ++ ik * SLM_B_STRIDE];+ }+ constexpr int unrk = std::is_same_v<TB, float> ? 4 : 8;+#pragma unroll(unrk)+ for (int ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (size_t im = 0; im < TileM; im++) {+ auto tmpA = sycl::select_from_group(helper.sg, regA[ikk + im / SgSize * UnrollK], im % SgSize);+#pragma unroll+ for (size_t in = 0; in < TileN; in++) {+ accptr[im * TileN + in] += tmpA * tmpB[in * UnrollK + ikk];+ }+ }+ }+ }+ }+ }+};++class Config_Fp32Fp32Fp32T {+ public:+ static int constexpr sg_size = 32;+ static int constexpr sg_m = 32;+ static int constexpr sg_n = 1;+ static int constexpr sg_k = 64;+ static int constexpr unroll_k = 8;+ static int constexpr wg_m = 4;+ static int constexpr wg_n = 64;++ using data_type_a = float;+ using data_type_b = float;+ using data_type_c = float;+ using data_type_acc = float;+};++class Config_Fp16Fp16Fp16T {+ public:+ static int constexpr sg_size = 32;+ static int constexpr sg_m = 32;+ static int constexpr sg_n = 1;+ static int constexpr sg_k = 64;+ static int constexpr unroll_k = 8;+ static int constexpr wg_m = 4;+ static int constexpr wg_n = 64;++ using data_type_a = sycl::half;+ using data_type_b = sycl::half;+ using data_type_c = sycl::half;+ using data_type_acc = sycl::half;+};+using DefaultSGemmCoreT = GemmCoreSharedBT<Config_Fp32Fp32Fp32T>;+using DefaultHGemmCoreT = GemmCoreSharedBT<Config_Fp16Fp16Fp16T>;+using DefaultHGemmCoreBF16T = GemmCoreSharedBT<Config_Bf16Bf16Bf16>;++template <typename CFG>+struct Prefetcher : CFG {+ using DT = CFG::DT;++ static int constexpr SGM = CFG::SGM;+ static int constexpr SGN = CFG::SGN;+ static int constexpr TM = CFG::TM;+ static int constexpr TN = CFG::TN;+ static int constexpr TK = CFG::TK;+ static int constexpr UnrollK = CFG::UnrollK;+ static int constexpr PrefetchDis = CFG::PrefetchDis;++ static inline void next(sycl::sub_group sg, int ik, int m, int n, int k, int g_idm, int g_idn, int sggid_col,+ int sg_col, int sggid_row, int sg_row, int lda, int ldb, const DT* Awg_d, const DT* Bwg_d) {+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;++ if (ik + TK * UnrollK * (PrefetchDis + 1) <= k) {+ for (int im = sggid_col; im < SGM; im += sg_col)+ if ((g_idm + im * TM + TM) <= m)+ joint_matrix_prefetch<TM, TK * UnrollK>(+ sg, Awg_d + (im * TM) * lda + ik + TK * UnrollK * PrefetchDis, lda, layout::row_major,+ sycl::ext::oneapi::experimental::properties{sycl::ext::oneapi::experimental::prefetch_hint_L1});+ for (int in = sggid_row; in < SGN; in += sg_row)+ if ((g_idn + in * TN + TN) <= n)+ joint_matrix_prefetch<TN, TK * UnrollK>(+ sg, Bwg_d + (in * TN) * ldb + ik + TK * UnrollK * PrefetchDis, ldb, layout::row_major,+ sycl::ext::oneapi::experimental::properties{sycl::ext::oneapi::experimental::prefetch_hint_L1});+ }+ }+};+struct SGemmCfg {+ static int constexpr sg_size = 16;+ static int constexpr sg_row = 8;+ static int constexpr sg_col = 4;+ static int constexpr TM = 16;+ static int constexpr TN = 16;+ static int constexpr TK = 16;+ static int constexpr SGM = 2;+ static int constexpr SGN = 2;+ static int constexpr UnrollK = 2;+ static int constexpr PrefetchDis = 3;+ static int constexpr G_NROW = 3;+ static int constexpr GRF = 256;+ using DT = float;+ using DT_ACC = float;+ using Param = GemmParam;+};++template <typename TP, typename CFG>+struct GemmCore {+ using DT = CFG::DT;+ using DT1 = CFG::DT_ACC;+ using Param = CFG::Param;+ static int constexpr sg_size = CFG::sg_size;+ static int constexpr sg_row = CFG::sg_row;+ static int constexpr sg_col = CFG::sg_col;+ static int constexpr TM = CFG::TM;+ static int constexpr TN = CFG::TN;+ static int constexpr TK = CFG::TK;+ static int constexpr SGM = CFG::SGM;+ static int constexpr SGN = CFG::SGN;+ static int constexpr UnrollK = CFG::UnrollK;+ static int constexpr PrefetchDis = CFG::PrefetchDis;+ static int constexpr G_NROW = CFG::G_NROW;++ static int constexpr wg_size = sg_size * sg_row * sg_col;+ static int constexpr BM = TM * sg_row * SGM;+ static int constexpr BM_STRIDE = BM;+ static int constexpr BN = TN * sg_col * SGN;+ static int constexpr BN_STRIDE = BN;++ Param param;+ TP mProp;++ int g_ncnt, g_mcnt;+ size_t wg_repeat, aligned_wg;+ GemmCore(TP Prop, CFG _cfg, Param _param) : mProp(Prop) {+ param = _param;++ g_ncnt = (param.n + BN - 1) / BN;+ g_mcnt = (param.m + BM - 1) / BM;+ wg_repeat = g_mcnt * g_ncnt;+ size_t m_tail = g_mcnt % G_NROW;+ size_t g_m_aligned = g_mcnt - m_tail;+ aligned_wg = g_m_aligned * g_ncnt;+ }++ inline sycl::nd_range<2> get_range() { return sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}); }++ void operator() [[sycl::reqd_sub_group_size(sg_size)]] (sycl::nd_item<2> it) const {+ auto sg = it.get_sub_group();+ int g_id = it.get_group(0);+ int g_n = g_id % g_ncnt;+ int g_m = g_id / g_ncnt;+ if (g_id < aligned_wg) {+ int g_m_ = g_id % G_NROW;+ g_id /= G_NROW;+ g_n = g_id % g_ncnt;+ g_id /= g_ncnt;+ g_m = g_id * G_NROW + g_m_;+ }+ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;+ int sgId = sg.get_local_id()[0];+ int g_idn = sggid_col * TN * SGN + g_n * BN;+ int g_idm = sggid_row * TM * SGM + g_m * BM;+ auto m = param.m;+ auto n = param.n;+ auto k = param.k;+ if (g_idn >= n || g_idm >= m) return;+ auto lda = param.lda;+ auto ldb = param.ldb;+ auto ldc = param.ldc;+ auto Awg_d = (DT*)param.A_d + (size_t)g_idm * lda;+ auto Bwg_d = (DT*)param.B_d + (size_t)g_idn * ldb;+ auto Cwg_d = (DT1*)param.C_d + (size_t)g_idm * ldc + g_idn;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ auto pA = syclex::annotated_ptr{+ Awg_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L3>>}};++ auto pB = syclex::annotated_ptr{+ Bwg_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};++ auto pC = syclex::annotated_ptr{+ Cwg_d, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_through, syclex::cache_level::L1, syclex::cache_level::L3>>}};+ DT sub_a[SGM * TM], sub_b[SGN * TK];+ DT1 sub_c[SGM * TM * SGN];++ DT1 zero = (DT1)0;+ if (param.Bias) {+ auto bptr = (DT1*)param.Bias + g_idn;+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ if ((in * TN + sgId) + g_idn >= n)+#pragma unroll+ for (int im = 0; im < SGM * TM; im++) sub_c[im * SGN + in] = zero;+ else+#pragma unroll+ for (int im = 0; im < SGM * TM; im++) sub_c[im * SGN + in] = bptr[in * TN + sgId];+ }++ } else {+#pragma unroll+ for (int im = 0; im < SGM * TM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) sub_c[im * SGN + in] = zero;+ }++ // Prefetcher<CFG>::next(sg, 0, m, n, k, g_idm, g_idn, sggid_col, sg_col, sggid_row, sg_row, lda, ldb, Awg_d,+ // Bwg_d);+ sycl::group_barrier(sg);++#pragma unroll(1)+ for (size_t ik = 0; ik < k; ik += TK * UnrollK) {+#pragma unroll+ for (size_t ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int _ik = 0; _ik < TK; _ik++) {+ if (ik + TK * ikk + _ik >= k) {+#pragma unroll+ for (int in = 0; in < SGN; in++) sub_b[in * TK + _ik] = zero;+ } else {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ if ((in * TN + sgId) + g_idn >= n) break;+ sub_b[in * TK + _ik] = pB[(in * TN + sgId) * ldb + ik + TK * ikk + _ik];+ }+ }+ }+ if (ik + TK * ikk + sgId < k)+#pragma unroll+ for (int im = 0; im < SGM * TM; im++) {+ if (im + g_idm >= m) break;+ sub_a[im] = pA[im * lda + ik + TK * ikk + sgId];+ }+ else+#pragma unroll+ for (int im = 0; im < SGM * TM; im++) {+ sub_a[im] = zero;+ }+ sycl::group_barrier(sg);+#pragma unroll+ for (int im = 0; im < SGM * TM; im++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+#pragma unroll+ for (int _ik = 0; _ik < TK; _ik++) {+ sub_c[im * SGN + in] += sycl::select_from_group(sg, sub_a[im], _ik) * sub_b[in * TK + _ik];+ }+ }+ }+ }+ // Prefetcher<CFG>::next(sg, ik, m, n, k, g_idm, g_idn, sggid_col, sg_col, sggid_row, sg_row, lda, ldb, Awg_d,+ // Bwg_d);+ sycl::group_barrier(sg);+ }+#pragma unroll+ for (int im = 0; im < SGM * TM; im++) {+ if (im + g_idm >= m) break;+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ if ((in * TN + sgId) + g_idn >= n) break;+ pC[im * ldc + in * TN + sgId] = sub_c[im * SGN + in];+ }+ }+ }++ auto get(syclex::properties_tag) const { return mProp; }+};+} // namespace xve++namespace xmx {+template <typename CFG>+struct Prefetcher : CFG {+ using DT = CFG::DT;++ static int constexpr SGM = CFG::SGM;+ static int constexpr SGN = CFG::SGN;+ static int constexpr TM = CFG::TM;+ static int constexpr TN = CFG::TN;+ static int constexpr TK = CFG::TK;+ static int constexpr UnrollK = CFG::UnrollK;+ static int constexpr PrefetchDis = CFG::PrefetchDis;++ static inline void next(sycl::sub_group sg, int ik, int m, int n, int k, int g_idm, int g_idn, int sggid_col,+ int sg_col, int sggid_row, int sg_row, int lda, int ldb, const DT* Awg_d, const DT* Bwg_d) {+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;++ if (ik + TK * UnrollK * (PrefetchDis + 1) <= k) {+ for (int im = sggid_col; im < SGM; im += sg_col)+ if ((g_idm + im * TM + TM) <= m)+ joint_matrix_prefetch<TM, TK * UnrollK>(+ sg, Awg_d + (im * TM) * lda + ik + TK * UnrollK * PrefetchDis, lda, layout::row_major,+ sycl::ext::oneapi::experimental::properties{sycl::ext::oneapi::experimental::prefetch_hint_L1});+ for (int in = sggid_row; in < SGN; in += sg_row)+ if ((g_idn + in * TN + TN) <= n)+ joint_matrix_prefetch<TN, TK * UnrollK>(+ sg, Bwg_d + (in * TN) * ldb + ik + TK * UnrollK * PrefetchDis, ldb, layout::row_major,+ sycl::ext::oneapi::experimental::properties{sycl::ext::oneapi::experimental::prefetch_hint_L1});+ }+ }+};++struct HGemmCfg {+ static int constexpr sg_size = 16;+ static int constexpr sg_row = 8;+ static int constexpr sg_col = 4;+ static int constexpr TM = 16;+ static int constexpr TN = 16;+ static int constexpr TK = 16;+ static int constexpr SGM = 2;+ static int constexpr SGN = 4;+ static int constexpr UnrollK = 2;+ static int constexpr PrefetchDis = 3;+ static int constexpr G_NROW = 3;+ static int constexpr GRF = 256;+ using DT = sycl::half;+ using DT_ACC = sycl::half;+ using Param = GemmParam;+};++struct HGemmAfp32Cfg {+ static int constexpr sg_size = 16;+ static int constexpr sg_row = 8;+ static int constexpr sg_col = 4;+ static int constexpr TM = 16;+ static int constexpr TN = 16;+ static int constexpr TK = 16;+ static int constexpr SGM = 2;+ static int constexpr SGN = 2;+ static int constexpr UnrollK = 2;+ static int constexpr PrefetchDis = 3;+ static int constexpr G_NROW = 3;+ static int constexpr GRF = 256;+ using DT = sycl::half;+ using DT_ACC = float;+ using Param = GemmParam;+};++struct HGemmBf16Cfg {+ static int constexpr sg_size = 16;+ static int constexpr sg_row = 8;+ static int constexpr sg_col = 4;+ static int constexpr TM = 16;+ static int constexpr TN = 16;+ static int constexpr TK = 16;+ static int constexpr SGM = 2;+ static int constexpr SGN = 4;+ static int constexpr UnrollK = 2;+ static int constexpr PrefetchDis = 3;+ static int constexpr G_NROW = 2;+ static int constexpr GRF = 256;+ using DT = sycl::ext::oneapi::bfloat16;+ using DT_ACC = sycl::ext::oneapi::bfloat16;+ using Param = GemmParam;+};++struct HGemmBf16Fp32Cfg {+ static int constexpr sg_size = 16;+ static int constexpr sg_row = 8;+ static int constexpr sg_col = 4;+ static int constexpr TM = 16;+ static int constexpr TN = 16;+ static int constexpr TK = 16;+ static int constexpr SGM = 2;+ static int constexpr SGN = 2;+ static int constexpr UnrollK = 2;+ static int constexpr PrefetchDis = 3;+ static int constexpr G_NROW = 2;+ static int constexpr GRF = 256;+ using DT = sycl::ext::oneapi::bfloat16;+ using DT_ACC = float;+ using Param = GemmParam;+};++struct IGemmCfg {+ static int constexpr sg_size = 16;+ static int constexpr sg_row = 8;+ static int constexpr sg_col = 4;+ static int constexpr TM = 8;+ static int constexpr TN = 16;+ static int constexpr TK = 32;+ static int constexpr SGM = 4;+ static int constexpr SGN = 4;+ static int constexpr UnrollK = 2;+ static int constexpr PrefetchDis = 3;+ static int constexpr G_NROW = 3;+ static int constexpr GRF = 256;+ using DT = int8_t;+ using DT_ACC = int32_t;+ using Param = GemmParam;+};++template <typename TP, typename CFG>+struct GemmCore {+ using DT = CFG::DT;+ using DT1 = CFG::DT_ACC;+ using Param = CFG::Param;+ static int constexpr sg_size = CFG::sg_size;+ static int constexpr sg_row = CFG::sg_row;+ static int constexpr sg_col = CFG::sg_col;+ static int constexpr TM = CFG::TM;+ static int constexpr TN = CFG::TN;+ static int constexpr TK = CFG::TK;+ static int constexpr SGM = CFG::SGM;+ static int constexpr SGN = CFG::SGN;+ static int constexpr UnrollK = CFG::UnrollK;+ static int constexpr PrefetchDis = CFG::PrefetchDis;+ static int constexpr G_NROW = CFG::G_NROW;++ static int constexpr wg_size = sg_size * sg_row * sg_col;+ static int constexpr BM = TM * sg_row * SGM;+ static int constexpr BM_STRIDE = BM;+ static int constexpr BN = TN * sg_col * SGN;+ static int constexpr BN_STRIDE = BN;++ Param param;+ TP mProp;++ int g_ncnt, g_mcnt;+ size_t wg_repeat, aligned_wg;+ GemmCore(TP Prop, CFG _cfg, Param _param) : mProp(Prop) {+ param = _param;++ g_ncnt = (param.n + BN - 1) / BN;+ g_mcnt = (param.m + BM - 1) / BM;+ wg_repeat = g_mcnt * g_ncnt;+ size_t m_tail = g_mcnt % G_NROW;+ size_t g_m_aligned = g_mcnt - m_tail;+ aligned_wg = g_m_aligned * g_ncnt;+ }++ inline sycl::nd_range<2> get_range() { return sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}); }++ void operator() [[sycl::reqd_sub_group_size(sg_size)]] (sycl::nd_item<2> it) const {+ auto sg = it.get_sub_group();+ int g_id = it.get_group(0);+ int g_n = g_id % g_ncnt;+ int g_m = g_id / g_ncnt;+ if (g_id < aligned_wg) {+ int g_m_ = g_id % G_NROW;+ g_id /= G_NROW;+ g_n = g_id % g_ncnt;+ g_id /= g_ncnt;+ g_m = g_id * G_NROW + g_m_;+ }+ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;+ int sgId = sg.get_local_id()[0];+ int g_idn = sggid_col * TN * SGN + g_n * BN;+ int g_idm = sggid_row * TM * SGM + g_m * BM;+ auto m = param.m;+ auto n = param.n;+ auto k = param.k;+ if (g_idn >= n || g_idm >= m) return;+ auto lda = param.lda;+ auto ldb = param.ldb;+ auto ldc = param.ldc;+ auto Awg_d = (DT*)param.A_d + (size_t)g_idm * lda;+ auto Bwg_d = (DT*)param.B_d + (size_t)g_idn * ldb;+ auto Cwg_d = (DT1*)param.C_d + (size_t)g_idm * ldc + g_idn;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ auto pA = syclex::annotated_ptr{+ Awg_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L3>>}};++ auto pB = syclex::annotated_ptr{+ Bwg_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};++ auto pC = syclex::annotated_ptr{+ Cwg_d, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_through, syclex::cache_level::L1, syclex::cache_level::L3>>}};+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::a, TM, TK, layout::row_major>+ sub_a[SGM];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::b, TK, TN, layout::col_major>+ sub_b[SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT1, use::accumulator, TM, TN>+ sub_c[SGM * SGN];++ DT1 zero = (DT1)0;+ if constexpr (!std::is_same_v<DT1, int32_t>) {+#pragma unroll(1)+ for (int in = 0; in < SGN; in++) {+ auto biasv = param.Bias ? *((DT1*)param.Bias + g_idn + in * TN + sgId) : zero;+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ joint_matrix_fill(sg, sub_c[im * SGN + in], biasv);+ }+ }+ } else {+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) joint_matrix_fill(sg, sub_c[im * SGN + in], DT1(0));+ }+ Prefetcher<CFG>::next(sg, 0, m, n, k, g_idm, g_idn, sggid_col, sg_col, sggid_row, sg_row, lda, ldb, Awg_d, Bwg_d);+ sycl::group_barrier(sg);++#pragma unroll(1)+ for (size_t ik = 0; ik < k; ik += TK * UnrollK) {+#pragma unroll+ for (size_t ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(sg, sub_b[in], pB, ldb, n - g_idn, k,+ in * TN, ik + TK * ikk);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(sg, sub_a[im], pA, lda, m - g_idm, k,+ im * TM, ik + TK * ikk);+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_mad(sg, sub_c[im * SGN + in], sub_a[im], sub_b[in], sub_c[im * SGN + in]);+ }+ }+ }+ Prefetcher<CFG>::next(sg, ik, m, n, k, g_idm, g_idn, sggid_col, sg_col, sggid_row, sg_row, lda, ldb, Awg_d,+ Bwg_d);+ }+ sycl::group_barrier(sg);+#pragma unroll+ for (int im = 0; im < SGM; im++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_store_checked(+ sg, sub_c[im * SGN + in], pC, ldc, layout::row_major, m - g_idm, n - g_idn, im * TM, in * TN);+ }+ }+ }++ auto get(syclex::properties_tag) const { return mProp; }+};++struct IGemmDQParam : GemmParam {+ void *scaleA, *scaleB;+};++template <typename T>+struct IGemmDQCfg {+ static int constexpr sg_size = 16;+ static int constexpr sg_row = 8;+ static int constexpr sg_col = 4;+ static int constexpr TM = 8;+ static int constexpr TN = 16;+ static int constexpr TK = 32;+ static int constexpr SGM = 4;+ static int constexpr SGN = 4;+ static int constexpr UnrollK = 2;+ static int constexpr PrefetchDis = 3;+ static int constexpr G_NROW = 3;+ static int constexpr GRF = 256;+ using DT = int8_t;+ using DT_ACC = int32_t;+ using DQT = T;+ using Param = IGemmDQParam;+};++template <typename TP, typename CFG>+struct IGemmDQCore {+ using DT = CFG::DT;+ using DT1 = CFG::DT_ACC;+ using DQT = CFG::DQT;+ using Param = CFG::Param;++ static int constexpr sg_size = CFG::sg_size;+ static int constexpr sg_row = CFG::sg_row;+ static int constexpr sg_col = CFG::sg_col;+ static int constexpr TM = CFG::TM;+ static int constexpr TN = CFG::TN;+ static int constexpr TK = CFG::TK;+ static int constexpr SGM = CFG::SGM;+ static int constexpr SGN = CFG::SGN;+ static int constexpr UnrollK = CFG::UnrollK;+ static int constexpr PrefetchDis = CFG::PrefetchDis;+ static int constexpr G_NROW = CFG::G_NROW;++ static int constexpr wg_size = sg_size * sg_row * sg_col;+ static int constexpr BM = TM * sg_row * SGM;+ static int constexpr BM_STRIDE = BM;+ static int constexpr BN = TN * sg_col * SGN;+ static int constexpr BN_STRIDE = BN;++ Param param;+ TP mProp;++ int g_ncnt, g_mcnt;+ size_t wg_repeat, aligned_wg;+ IGemmDQCore(TP Prop, CFG _cfg, Param _param) : mProp(Prop) {+ param = _param;++ g_ncnt = (param.n + BN - 1) / BN;+ g_mcnt = (param.m + BM - 1) / BM;+ wg_repeat = g_mcnt * g_ncnt;+ size_t m_tail = g_mcnt % G_NROW;+ size_t g_m_aligned = g_mcnt - m_tail;+ aligned_wg = g_m_aligned * g_ncnt;+ }++ inline sycl::nd_range<2> get_range() { return sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}); }++ void operator() [[sycl::reqd_sub_group_size(sg_size)]] (sycl::nd_item<2> it) const {+ auto sg = it.get_sub_group();+ int g_id = it.get_group(0);+ int g_n = g_id % g_ncnt;+ int g_m = g_id / g_ncnt;+ if (g_id < aligned_wg) {+ int g_m_ = g_id % G_NROW;+ g_id /= G_NROW;+ g_n = g_id % g_ncnt;+ g_id /= g_ncnt;+ g_m = g_id * G_NROW + g_m_;+ }+ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;+ int sgId = sg.get_local_id()[0];+ int g_idn = sggid_col * TN * SGN + g_n * BN;+ int g_idm = sggid_row * TM * SGM + g_m * BM;+ auto m = param.m;+ auto n = param.n;+ auto k = param.k;+ if (g_idn >= n || g_idm >= m) return;+ auto lda = param.lda;+ auto ldb = param.ldb;+ auto ldc = param.ldc;+ auto Awg_d = (DT*)param.A_d + (size_t)g_idm * lda;+ auto Bwg_d = (DT*)param.B_d + (size_t)g_idn * ldb;+ auto Cwg_d = (DQT*)param.C_d + (size_t)g_idm * ldc + g_idn;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ auto pA = syclex::annotated_ptr{+ Awg_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L3>>}};++ auto pB = syclex::annotated_ptr{+ Bwg_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};++ auto pC = syclex::annotated_ptr{+ Cwg_d, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_through, syclex::cache_level::L1, syclex::cache_level::L3>>}};+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::a, TM, TK, layout::row_major>+ sub_a[SGM];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::b, TK, TN, layout::col_major>+ sub_b[SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT1, use::accumulator, TM, TN>+ sub_c[SGM * SGN];++#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) joint_matrix_fill(sg, sub_c[im * SGN + in], DT1(0));+ Prefetcher<CFG>::next(sg, 0, m, n, k, g_idm, g_idn, sggid_col, sg_col, sggid_row, sg_row, lda, ldb, Awg_d, Bwg_d);+ sycl::group_barrier(sg);++#pragma unroll(1)+ for (size_t ik = 0; ik < k; ik += TK * UnrollK) {+#pragma unroll+ for (size_t ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(sg, sub_b[in], pB, ldb, n - g_idn, k,+ in * TN, ik + TK * ikk);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(sg, sub_a[im], pA, lda, m - g_idm, k,+ im * TM, ik + TK * ikk);+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_mad(sg, sub_c[im * SGN + in], sub_a[im], sub_b[in], sub_c[im * SGN + in]);+ }+ }+ }+ Prefetcher<CFG>::next(sg, ik, m, n, k, g_idm, g_idn, sggid_col, sg_col, sggid_row, sg_row, lda, ldb, Awg_d,+ Bwg_d);+ }+ sycl::group_barrier(sg);+ auto scaleA = (DQT*)param.scaleA;+ auto scaleB = (DQT*)param.scaleB;+ auto bias = (DQT*)param.Bias;++ DQT sBs[SGN], bBs[SGN];+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ if (in * TN + g_idn + sgId >= n) break;+ sBs[in] = scaleB[in * TN + g_idn + sgId];+ bBs[in] = bias ? bias[in * TN + g_idn + sgId] : DQT(0);+ }++#pragma unroll+ for (int im = 0; im < SGM; im++) {+#pragma unroll+ for (int imm = 0; imm < TM; imm++) {+ if (im * TM + g_idm + imm >= m) break;+ auto scale2 = scaleA[im * TM + g_idm + imm];+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ auto wi_data_c = sycl::ext::oneapi::detail::get_wi_data(sg, sub_c[im * SGN + in]);+ Cwg_d[(im * TM + imm) * ldc + sgId + in * TN] =+ static_cast<float>(wi_data_c[imm]) * sBs[in] * scale2 + bBs[in];+ }+ }+ }+ }++ auto get(syclex::properties_tag) const { return mProp; }+};++struct IKblockGemmDQParam : IGemmDQParam {+ int blocksize;+};++template <typename T>+struct IKblockGemmDQCfg {+ static int constexpr sg_size = 16;+ static int constexpr sg_row = 8;+ static int constexpr sg_col = 4;+ static int constexpr TM = 8;+ static int constexpr TN = 16;+ static int constexpr TK = 32;+ static int constexpr SGM = 4;+ static int constexpr SGN = 2;+ static int constexpr UnrollK = 2;+ static int constexpr PrefetchDis = 3;+ static int constexpr G_NROW = 3;+ static int constexpr GRF = 256;+ using DT = int8_t;+ using DT_ACC = int32_t;+ using DQT = T;+ using Param = IKblockGemmDQParam;+};++template <typename TP, typename CFG>+struct IKblockGemmDQCore {+ using DT = CFG::DT;+ using DT1 = CFG::DT_ACC;+ using DQT = CFG::DQT;+ using Param = CFG::Param;++ static int constexpr sg_size = CFG::sg_size;+ static int constexpr sg_row = CFG::sg_row;+ static int constexpr sg_col = CFG::sg_col;+ static int constexpr TM = CFG::TM;+ static int constexpr TN = CFG::TN;+ static int constexpr TK = CFG::TK;+ static int constexpr SGM = CFG::SGM;+ static int constexpr SGN = CFG::SGN;+ static int constexpr UnrollK = CFG::UnrollK;+ static int constexpr PrefetchDis = CFG::PrefetchDis;+ static int constexpr G_NROW = CFG::G_NROW;++ static int constexpr wg_size = sg_size * sg_row * sg_col;+ static int constexpr BM = TM * sg_row * SGM;+ static int constexpr BM_STRIDE = BM;+ static int constexpr BN = TN * sg_col * SGN;+ static int constexpr BN_STRIDE = BN;++ Param param;+ TP mProp;++ int g_ncnt, g_mcnt, blks;+ size_t wg_repeat, aligned_wg;+ IKblockGemmDQCore(TP Prop, CFG _cfg, Param _param) : mProp(Prop) {+ param = _param;++ g_ncnt = (param.n + BN - 1) / BN;+ g_mcnt = (param.m + BM - 1) / BM;+ blks = (_param.k) / _param.blocksize;+ wg_repeat = g_mcnt * g_ncnt;+ size_t m_tail = g_mcnt % G_NROW;+ size_t g_m_aligned = g_mcnt - m_tail;+ aligned_wg = g_m_aligned * g_ncnt;+ }++ inline sycl::nd_range<2> get_range() { return sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}); }++ void operator() [[sycl::reqd_sub_group_size(sg_size)]] (sycl::nd_item<2> it) const {+ auto sg = it.get_sub_group();+ int g_id = it.get_group(0);+ int g_n = g_id % g_ncnt;+ int g_m = g_id / g_ncnt;+ if (g_id < aligned_wg) {+ int g_m_ = g_id % G_NROW;+ g_id /= G_NROW;+ g_n = g_id % g_ncnt;+ g_id /= g_ncnt;+ g_m = g_id * G_NROW + g_m_;+ }+ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;+ int sgId = sg.get_local_id()[0];+ int g_idn = sggid_col * TN * SGN + g_n * BN;+ int g_idm = sggid_row * TM * SGM + g_m * BM;+ auto m = param.m;+ auto n = param.n;+ auto k = param.k;+ if (g_idn >= n || g_idm >= m) return;+ auto lda = param.lda;+ auto ldb = param.ldb;+ auto ldc = param.ldc;+ int blocksize = param.blocksize;+ auto Awg_d = (DT*)param.A_d + (size_t)g_idm * lda;+ auto Bwg_d = (DT*)param.B_d + (size_t)g_idn * ldb;+ auto Cwg_d = (DQT*)param.C_d + (size_t)g_idm * ldc + g_idn;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ auto pA = syclex::annotated_ptr{+ Awg_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};+ auto pSA = sycl::address_space_cast<sycl::access::address_space::global_space, sycl::access::decorated::yes>(+ (DQT*)param.scaleA);+ auto pB = syclex::annotated_ptr{+ Bwg_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};+ auto pSB = syclex::annotated_ptr{+ (DQT*)param.scaleB, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L3>>}};+ auto pC = syclex::annotated_ptr{+ Cwg_d, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_through, syclex::cache_level::L1, syclex::cache_level::L3>>}};+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::a, TM, TK, layout::row_major>+ sub_a[SGM];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::b, TK, TN, layout::col_major>+ sub_b[SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT1, use::accumulator, TM, TN>+ sub_c[SGM * SGN];+ float acc_c[SGM * SGN * TM];++ DT1 zero = (DT1)0;+#pragma unroll+ for (int in = 0; in < SGN; in++) {+#pragma unroll+ for (int im = 0; im < SGM; im++) {+#pragma unroll+ for (int imm = 0; imm < TM; imm++) {+ acc_c[(im * TM + imm) * SGN + in] = zero;+ }+ }+ }+ Prefetcher<CFG>::next(sg, 0, m, n, k, g_idm, g_idn, sggid_col, sg_col, sggid_row, sg_row, lda, ldb, Awg_d, Bwg_d);+ sycl::group_barrier(sg);+ for (int ib = 0; ib < blks; ib++) {+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) joint_matrix_fill(sg, sub_c[im * SGN + in], 0);+#pragma unroll(1)+ for (int ik = ib * blocksize; ik < (ib + 1) * blocksize; ik += TK * UnrollK) {+#pragma unroll+ for (int ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(sg, sub_b[in], pB, ldb, n - g_idn, k,+ in * TN, ik + ikk * TK);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(sg, sub_a[im], pA, lda, m - g_idm, k,+ im * TM, ik + TK * ikk);+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_mad(sg, sub_c[im * SGN + in], sub_a[im], sub_b[in], sub_c[im * SGN + in]);+ }+ }+ }+ Prefetcher<CFG>::next(sg, ik, m, n, k, g_idm, g_idn, sggid_col, sg_col, sggid_row, sg_row, lda, ldb, Awg_d,+ Bwg_d);+ }+ sycl::group_barrier(sg);+ DQT sBs[SGN];+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ if (in * TN + g_idn + sgId >= n) break;+ sBs[in] = pSB[(in * TN + g_idn + sgId) * blks + ib];+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ auto wi_data_c = sycl::ext::oneapi::detail::get_wi_data(sg, sub_c[im * SGN + in]);+ for (int i = 0; i < wi_data_c.length(); i++) {+ auto element = wi_data_c[i];+ acc_c[(im * TM + i) * SGN + in] += static_cast<float>(element) * static_cast<float>(sBs[in]);+ }+ }+ }+ sycl::group_barrier(sg);+ }+ DQT bBs[SGN];+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ if (in * TN + g_idn + sgId >= n) break;+ bBs[in] = param.Bias ? ((DQT*)param.Bias)[in * TN + g_idn + sgId] : DQT(0);+ }++#pragma unroll+ for (int im = 0; im < SGM; im++) {+#pragma unroll+ for (int imm = 0; imm < TM; imm++) {+ if (im * TM + g_idm + imm >= m) break;+ auto scale2 = pSA[im * TM + g_idm + imm];+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ if (in * TN + g_idn + sgId >= n) break;+ pC[(im * TM + imm) * ldc + sgId + in * TN] = acc_c[(im * TM + imm) * SGN + in] * scale2 + bBs[in];+ }+ }+ }+ }++ auto get(syclex::properties_tag) const { return mProp; }+};++} // namespace xmx++template <typename CFG, template <typename AT, typename BT> typename KER>+struct Launcher {+ static inline sycl::event run(sycl::queue* q, const typename CFG::Param& _param) {+ auto ker = [&](sycl::handler& cgh) {+ syclex::properties prop{syclintelex::grf_size<CFG::GRF>};+ KER largeker(prop, CFG(), _param);+ cgh.parallel_for(largeker.get_range(), largeker);+ };+ return q->submit(ker);+ }+};+} // namespace sycl_gemm+} // namespace bestla+#endifdiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_prologue_a.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_prologue_a.hnew file mode 100644index 000000000..32e3b005e--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_prologue_a.h@@ -0,0 +1,173 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once++#ifdef BTLA_SYCL+#include <array>++#include "bestla/bestla_utils.h"+#include <sycl/sycl.hpp>++namespace bestla {+namespace sycl_prologue_a {++template <typename SrcT>+struct ParamActivationBase {+ const SrcT* A;+ int lda;+};++template <typename T>+class ActivationBase {+ public:+ using Param = ParamActivationBase<T>;++ struct CfgQuantF32 {+ static int constexpr SgSize = 32;+ static int constexpr Unroll = 4;+ static int constexpr MaxK = 32;+ static int constexpr WG_Size = 1024;+ };++ struct CfgQuantF16 {+ static int constexpr SgSize = 32;+ static int constexpr TileK = 8;+ static int constexpr TileN = 1;+ static int constexpr Unroll = 4;+ };++ template <typename Cfg>+ static inline sycl::event quant_s8(int m, int k, int mask, const Param& in, int8_t* outptr, T* scaleptr,+ sycl::queue* q) {+ int constexpr SgSize = Cfg::SgSize;+ int constexpr Unroll = Cfg::Unroll;+ int constexpr MaxK = Cfg::MaxK;+ int constexpr WG_Size = Cfg::WG_Size;+ int constexpr SG_Num = WG_Size / SgSize;+ assert(k <= WG_Size * MaxK);++ int nsg_m = m;+ sycl::range<1> group{WG_Size};+ sycl::range<1> problem{static_cast<size_t>(nsg_m) * WG_Size};+ auto A_d = in.A;+ int lda = in.lda;+ auto ker = [&](sycl::handler& cgh) {+ sycl::local_accessor<float, 1> slm_maxabs(sycl::range(SG_Num), cgh);+ cgh.parallel_for(+ sycl::nd_range<1>(problem, group), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(SgSize)]] {+ int g_idx = it.get_group(0);+ auto sg = it.get_sub_group();+ auto wg_idx = it.get_local_id();+ int sg_id = sg.get_local_id()[0];+ int sg_group_id = sg.get_group_id()[0];+ int g_m = g_idx;+ auto aptr = A_d + g_m * lda;+ auto qptr = outptr + g_m * lda;+ float maxabs = float(0);+ T srcs[MaxK];+ auto pS8 = syclex::annotated_ptr{+ outptr, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_back, syclex::cache_level::L1, syclex::cache_level::L3>>}};+#pragma unroll+ for (int ik = 0; ik < MaxK; ik += Unroll) {+ if (wg_idx * Unroll + ik * WG_Size >= k) break;+ *(sycl::vec<T, Unroll>*)&srcs[ik] = *(sycl::vec<T, Unroll>*)(aptr + wg_idx * Unroll + ik * WG_Size);+#pragma unroll+ for (int ir = 0; ir < Unroll; ir += 1) {+ maxabs = sycl::max(maxabs, std::abs((float)srcs[ik + ir]));+ }+ }+ maxabs = sycl::reduce_over_group(sg, maxabs, sycl::maximum<float>());+ if (sg_id == 0) {+ slm_maxabs[sg_group_id] = maxabs;+ }+ it.barrier(sycl::access::fence_space::local_space);+ float g_maxabs = maxabs;+#pragma unroll+ for (size_t i = 0; i < SG_Num; i++) {+ g_maxabs = sycl::max(g_maxabs, slm_maxabs[i]);+ }+ T scale = g_maxabs / 127.f;+ float ratio = 1 / scale;+#pragma unroll+ for (int ik = 0; ik < MaxK; ik += Unroll) {+ if (wg_idx * Unroll + ik * WG_Size >= k) break;+ int8_t dst[Unroll];+#pragma unroll+ for (int ir = 0; ir < Unroll; ir += 1) {+ auto tmp = sycl::round((float)srcs[ik + ir] * ratio);+ dst[ir] = static_cast<int8_t>(tmp);+ dst[ir] = sycl::min(dst[ir], int8_t(127));+ dst[ir] = sycl::max(dst[ir], int8_t(-128));+ }+ *(sycl::vec<int8_t, Unroll>*)(pS8.get() + g_m * lda + wg_idx * Unroll + ik * WG_Size) =+ *(sycl::vec<int8_t, Unroll>*)dst;+ }+ if (wg_idx == 0) scaleptr[g_m] = scale;+ });+ };+ auto ev = q->submit(ker);+ if (mask == 0) return ev;+ auto ker2 = [&](sycl::handler& cgh) {+ sycl::local_accessor<float, 1> slm_maxabs(sycl::range(SG_Num), cgh);+ cgh.parallel_for(+ sycl::nd_range<1>(problem, group), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(SgSize)]] {+ int g_idx = it.get_group(0);+ auto sg = it.get_sub_group();+ auto wg_idx = it.get_local_id();+ int sg_id = sg.get_local_id()[0];+ int sg_group_id = sg.get_group_id()[0];+ int g_m = g_idx;+ auto aptr = outptr + g_m * lda;+ auto qptr = outptr + g_m * lda;+ float maxabs = float(0);+ for (int i = wg_idx; i < m; i += WG_Size) {+ maxabs = std::max(maxabs, (float)scaleptr[i]);+ }+ maxabs = sycl::reduce_over_group(sg, maxabs, sycl::maximum<float>());+ if (sg_id == 0) {+ slm_maxabs[sg_group_id] = maxabs;+ }+ it.barrier(sycl::access::fence_space::local_space);+ float g_maxabs = maxabs;+#pragma unroll+ for (size_t i = 0; i < SG_Num; i++) {+ g_maxabs = sycl::max(g_maxabs, slm_maxabs[i]);+ }+ float ratio = scaleptr[g_m] / g_maxabs;+ int8_t srcs[Unroll];+ float tmps[Unroll];+#pragma unroll+ for (int ik = 0; ik < MaxK; ik += Unroll) {+ if (wg_idx * Unroll + ik * WG_Size >= k) break;+ *(sycl::vec<int8_t, Unroll>*)srcs = *(sycl::vec<int8_t, Unroll>*)(aptr + wg_idx * Unroll + ik * WG_Size);+#pragma unroll+ for (int ir = 0; ir < Unroll; ir += 1) {+ tmps[ir] = sycl::round(float(srcs[ir]) * ratio);+ srcs[ir] = static_cast<int8_t>(tmps[ir]);+ srcs[ir] = sycl::min(srcs[ir], int8_t(127));+ srcs[ir] = sycl::max(srcs[ir], int8_t(-128));+ }+ *(sycl::vec<int8_t, Unroll>*)(qptr + wg_idx * Unroll + ik * WG_Size) = *(sycl::vec<int8_t, Unroll>*)srcs;+ }+ if (wg_idx == 0) scaleptr[g_m] = g_maxabs;+ });+ };+ return q->submit(ker2);+ }+};++} // namespace sycl_prologue_a+} // namespace bestla+#endifdiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_prologue_b.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_prologue_b.hnew file mode 100644index 000000000..1ec6d68bd--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_prologue_b.h@@ -0,0 +1,1057 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once++#ifdef BTLA_SYCL+#include <array>++#include "bestla/bestla_utils.h"+#include <sycl/sycl.hpp>++#include "bestla/sycl/fp8_lut.h"++namespace bestla {+namespace sycl_prologue_b {++template <typename SrcT>+struct ParamWeightBase {+ const SrcT* B;+ int ldb;+};++template <typename ScaleT>+struct ParamWeightS4 {+ const uint8_t* B;+ const ScaleT* scale;+ int ldb;+ const void* bias = nullptr;+ const int8_t* zp = nullptr;+};++template <typename ScaleT>+struct ParamWeightS8 {+ const int8_t* B;+ const ScaleT* scale;+ int ldb;+ const void* bias = nullptr;+ const int8_t* zp = nullptr;+};++template <typename ScaleT>+struct ParamWeightF8 {+ const uint8_t* B;+ const ScaleT* scale;+ int ldb;+ const void* bias = nullptr;+};++template <typename ScaleT>+struct ParamWeightS4Ext : ParamWeightS4<ScaleT> {+ const ScaleT* scaleExt = nullptr;+};++#define INT2x4_TO_0(src) (static_cast<int8_t>(src & (uint8_t)0x03))+#define INT2x4_TO_1(src) (static_cast<int8_t>((src >> (uint8_t)2) & (uint8_t)0x03))+#define INT2x4_TO_2(src) (static_cast<int8_t>((src >> (uint8_t)4) & (uint8_t)0x03))+#define INT2x4_TO_3(src) (static_cast<int8_t>(src >> (uint8_t)6))+#define INT4x2_TO_LO(src) (static_cast<int8_t>(src & (uint8_t)0x0f))+#define INT4x2_TO_HI(src) (static_cast<int8_t>(src >> (uint8_t)4))++template <typename ScaleT>+class WeightS2T {+ public:+ using Param = ParamWeightS4Ext<ScaleT>;++ static __attribute__((always_inline)) inline int8_t int2x4_to_0(const uint8_t& src) {+ return INT2x4_TO_0(src);+ }++ static __attribute__((always_inline)) inline int8_t int2x4_to_1(const uint8_t& src) {+ return INT2x4_TO_1(src);+ }++ static __attribute__((always_inline)) inline int8_t int2x4_to_2(const uint8_t& src) {+ return INT2x4_TO_2(src);+ }++ static __attribute__((always_inline)) inline int8_t int2x4_to_3(const uint8_t& src) {+ return INT2x4_TO_3(src);+ }++ struct CfgDequantF32 {+ static int constexpr SgSize = 32;+ static int constexpr TileK = 16;+ static int constexpr TileN = 1;+ static int constexpr Unroll = 8;+ };++ struct CfgDequantF16 {+ static int constexpr SgSize = 32;+ static int constexpr TileK = 16;+ static int constexpr TileN = 1;+ static int constexpr Unroll = 8;+ };++ template <typename Cfg, typename BType>+ static inline sycl::event dequant(int n, int k, int blocksize, const Param& in, BType* outptr, sycl::queue* q) {+ int constexpr SgSize = Cfg::SgSize;+ int constexpr TileK = Cfg::TileK;+ int constexpr TileN = Cfg::TileN;+ static_assert(TileN == 1);+ int constexpr Unroll = Cfg::Unroll;+ int constexpr SubGroupK = SgSize * TileK;+ int constexpr GroupK = SgSize * TileK;+ assert(blocksize % Unroll == 0);+ int nsg_k = utils::updiv(k, GroupK);+ int nsg_n = n;+ sycl::range<1> group{SgSize};+ sycl::range<1> problem{static_cast<size_t>(nsg_n) * nsg_k * SgSize};+ auto B_d = in.B;+ auto S_d = in.scale;+ auto zp_d = in.zp;+ int ldb = in.ldb;+ int ldbn = in.ldb * blocksize;+ auto deq_kernel = [&](sycl::handler& cgh) {+ cgh.parallel_for(+ sycl::nd_range<1>(problem, group), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(SgSize)]] {+ int g_idx = it.get_group(0);+ auto sg = it.get_sub_group();+ int sg_id = sg.get_local_id()[0];+ int sg_group_id = sg.get_group_id()[0];+ int g_idx_n = g_idx / nsg_k;+ int g_idx_k = g_idx % nsg_k;+ int g_n = g_idx_n * 1;+ int g_k = g_idx_k * GroupK;+ int sg_k = g_k + sg_group_id * SubGroupK;+ int sg_k_remain = k - sg_k;+ auto sptr = S_d + sg_k / blocksize + g_n * ldb;+ auto zpptr = zp_d ? zp_d + sg_k / blocksize + g_n * ldb : nullptr;+ auto bptr = B_d + (sg_k + g_n * ldbn) / 4;+ auto dbptr = outptr + sg_k + g_n * k;+#pragma unroll+ for (int ik = 0; ik < TileK; ik += Unroll) {+ if (ik * SgSize + sg_id * Unroll >= sg_k_remain) break;+ BType dst[Unroll];+ BType scale = sptr[(ik * SgSize + sg_id * Unroll) / blocksize];+ int8_t zp = zpptr ? zpptr[(ik * SgSize + sg_id * Unroll) / blocksize] + int8_t(2) : int8_t(2);+#pragma unroll+ for (int ir = 0; ir < Unroll; ir += 4) {+ uint8_t srcu8 = *(bptr + (ik * SgSize + sg_id * Unroll + ir) / 4);+ dst[ir] = static_cast<BType>(int2x4_to_0(srcu8) - zp) * scale;+ dst[ir + 1] = static_cast<BType>(int2x4_to_1(srcu8) - zp) * scale;+ dst[ir + 2] = static_cast<BType>(int2x4_to_2(srcu8) - zp) * scale;+ dst[ir + 3] = static_cast<BType>(int2x4_to_3(srcu8) - zp) * scale;+ }+ *(sycl::vec<BType, Unroll>*)&dbptr[ik * SgSize + sg_id * Unroll] = *(sycl::vec<BType, Unroll>*)dst;+ }+ });+ };+ return q->submit(deq_kernel);+ }++ struct CfgDequantS8 {+ static int constexpr SgSize = 32;+ static int constexpr TileK = 16;+ static int constexpr TileN = 1;+ static int constexpr Unroll = 16;+ static bool constexpr Rescale = 0;+ };++ struct CfgDequantS8Rescale {+ static int constexpr SgSize = 32;+ static int constexpr TileK = 32;+ static int constexpr TileN = 1;+ static int constexpr Unroll = 16;+ static bool constexpr Rescale = 1;+ };++ template <class Cfg>+ static inline sycl::event dequantS8(int n, int k, int blocksize, const Param& in, int8_t* outptr, sycl::queue* q,+ int newblocksize = -1) {+ int constexpr SgSize = Cfg::SgSize;+ int constexpr TileK = Cfg::TileK;+ int constexpr TileN = Cfg::TileN;+ int constexpr Rescale = Cfg::Rescale;+ static_assert(TileN == 1);+ int constexpr Unroll = Cfg::Unroll;+ int constexpr SubGroupK = SgSize * TileK;+ int constexpr GroupK = SgSize * TileK;+ assert(blocksize % Unroll == 0);+ int nsg_k = utils::updiv(k, GroupK);+ int nsg_n = n;+ sycl::range<1> group{SgSize};+ sycl::range<1> problem{static_cast<size_t>(nsg_n) * nsg_k * SgSize};+ auto B_d = in.B;+ auto S_d = in.scale;+ auto Sext_d = in.scaleExt;+ auto zp_d = in.zp;+ int ldb = in.ldb;+ int ldbn = in.ldb * blocksize;+ int newblks = newblocksize == -1 ? 1 : k / newblocksize;+ auto deq_kernel = [&](sycl::handler& cgh) {+ cgh.parallel_for(+ sycl::nd_range<1>(problem, group), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(SgSize)]] {+ int g_idx = it.get_group(0);+ auto sg = it.get_sub_group();+ int sg_id = sg.get_local_id()[0];+ int sg_group_id = sg.get_group_id()[0];+ int g_idx_n = g_idx / nsg_k;+ int g_idx_k = g_idx % nsg_k;+ int g_n = g_idx_n * 1;+ int g_k = g_idx_k * GroupK;+ int sg_k = g_k + sg_group_id * SubGroupK;+ int sg_k_remain = k - sg_k;+ auto sptr = S_d + sg_k / blocksize + g_n * ldb;+ auto newsptr = Sext_d + sg_k / newblocksize + g_n * newblks;+ auto zpptr = zp_d ? zp_d + sg_k / blocksize + g_n * ldb : nullptr;+ auto bptr = B_d + (sg_k + g_n * ldbn) / 4;+ auto dbptr = outptr + sg_k + g_n * k;++#pragma unroll+ for (int ik = 0; ik < TileK; ik += Unroll) {+ if (ik * SgSize + sg_id * Unroll >= sg_k_remain) break;+ int8_t zp = zpptr ? zpptr[(ik * SgSize + sg_id * Unroll) / blocksize] + int8_t(2) : int8_t(2);+ int8_t dst[Unroll];+ if constexpr (Rescale) {+ float scaleN = (float)newsptr[(ik * SgSize + sg_id * Unroll) / newblocksize];+ float scale = (float)sptr[(ik * SgSize + sg_id * Unroll) / blocksize] / scaleN;+#pragma unroll+ for (int ir = 0; ir < Unroll; ir += 4) {+ uint8_t srcu8 = *(bptr + (ik * SgSize + sg_id * Unroll + ir) / 4);+ dst[ir] = std::round(static_cast<float>(int2x4_to_0(srcu8) - zp) * scale);+ dst[ir + 1] = std::round(static_cast<float>(int2x4_to_1(srcu8) - zp) * scale);+ dst[ir + 2] = std::round(static_cast<float>(int2x4_to_2(srcu8) - zp) * scale);+ dst[ir + 3] = std::round(static_cast<float>(int2x4_to_3(srcu8) - zp) * scale);+ }+ } else {+#pragma unroll+ for (int ir = 0; ir < Unroll; ir += 4) {+ uint8_t srcu8 = *(bptr + (ik * SgSize + sg_id * Unroll + ir) / 4);+ dst[ir] = static_cast<float>(int2x4_to_0(srcu8) - zp);+ dst[ir + 1] = static_cast<float>(int2x4_to_1(srcu8) - zp);+ dst[ir + 2] = static_cast<float>(int2x4_to_2(srcu8) - zp);+ dst[ir + 3] = static_cast<float>(int2x4_to_3(srcu8) - zp);+ }+ }+ *(sycl::vec<int8_t, Unroll>*)&dbptr[ik * SgSize + sg_id * Unroll] = *(sycl::vec<int8_t, Unroll>*)dst;+ }+ });+ };+ return q->submit(deq_kernel);+ }++ struct CfgGemvF32 {+ static int constexpr SgSize = 32;+ static int constexpr TileK0 = 8;+ static int constexpr TileK1 = 8;+ static int constexpr TileM = 1;+ static int constexpr Unroll = 4;+ static int constexpr PrefetchDis = 2;+ };++ struct CfgGemvF16 {+ static int constexpr SgSize = 32;+ static int constexpr TileK0 = 16;+ static int constexpr TileK1 = 8;+ static int constexpr TileM = 1;+ static int constexpr Unroll = 4;+ static int constexpr PrefetchDis = 2;+ };++ template <typename Cfg, typename T>+ static sycl::event gemv(const T* A, const Param& paramB, T* C, int n, int k, int blocksize, sycl::queue* q) {+ auto B = paramB.B;+ auto B_scale = paramB.scale;+ auto zp_d = paramB.zp;+ auto bias = (const T*)paramB.bias;+ int ldb = paramB.ldb;+ int constexpr SgSize = Cfg::SgSize;+ int constexpr TileK0 = Cfg::TileK0;+ int constexpr TileK1 = Cfg::TileK1;+ int constexpr TileM = Cfg::TileM;+ int constexpr Unroll = Cfg::Unroll;+ int constexpr PrefetchDis = Cfg::PrefetchDis;+ sycl::range<1> group{SgSize};+ sycl::range<1> problem{static_cast<size_t>(n) * SgSize};+ assert(k % TileK1 == 0);+ int constexpr SGK0 = TileK0 * SgSize * Unroll;+ int constexpr SGK1 = TileK1 * SgSize;+ int k_0 = utils::padto_le(k, SGK0);+ auto ev = q->submit([&](sycl::handler& cgh) {+ cgh.parallel_for(+ sycl::nd_range<1>(problem, group), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(SgSize)]] {+ int g_idx = it.get_group(0);+ auto sg = it.get_sub_group();+ int sg_id = sg.get_local_id()[0];+ int g_n = g_idx;+ auto sptr = B_scale + g_n * ldb;+ auto zpptr = zp_d ? zp_d + g_n * ldb : nullptr;+ auto bptr = B + g_n * k / 4;+ auto aptr = A;+ auto cptr = C + g_n;+ T tmpAcc = 0.f;+ int i = 0;+ if (i + SGK0 * PrefetchDis + SGK0 < k)+ for (int j = 0; j < PrefetchDis; j++) {+ sycl::ext::oneapi::experimental::joint_prefetch(sg, bptr + j * SGK0 / 4, SGK0 / 4);+ }++ sycl::group_barrier(sg);+ for (; i < k_0; i += SGK0) {+#pragma unroll+ for (size_t ir = 0; ir < Unroll; ir++) {+ auto tmpu8 = *(sycl::vec<uint8_t, TileK0 / 4>*)(bptr + sg_id * TileK0 / 4);+ T scale = *(sptr + sg_id * TileK0 / blocksize);+ int8_t zp = zpptr ? *(zpptr + sg_id * TileK0 / blocksize) + int8_t(2) : int8_t(2);+ T tmpA[TileK0];+ *(sycl::vec<T, TileK0>*)tmpA = *(sycl::vec<T, TileK0>*)&aptr[sg_id * TileK0];+ T tacc[4] = {0.f, 0.f, 0.f, 0.f};+#pragma unroll+ for (int ikk = 0; ikk < TileK0; ikk += 4) {+ auto pack = tmpu8[ikk / 4];+ tacc[0] += tmpA[ikk + 0] * static_cast<T>(int2x4_to_0(pack) - zp);+ tacc[1] += tmpA[ikk + 1] * static_cast<T>(int2x4_to_1(pack) - zp);+ tacc[2] += tmpA[ikk + 2] * static_cast<T>(int2x4_to_2(pack) - zp);+ tacc[3] += tmpA[ikk + 3] * static_cast<T>(int2x4_to_3(pack) - zp);+ }+ tmpAcc += (tacc[0] + tacc[1] + tacc[2] + tacc[3]) * scale;+ sptr += TileK0 * SgSize / blocksize;+ zpptr += zpptr ? TileK0 * SgSize / blocksize : 0;+ aptr += TileK0 * SgSize;+ bptr += TileK0 * SgSize / 4;+ }+ if (i + SGK0 * PrefetchDis + SGK0 < k)+ sycl::ext::oneapi::experimental::joint_prefetch(sg, bptr + SGK0 * PrefetchDis / 4, SGK0 / 4);+ sycl::group_barrier(sg);+ }+ if (i < k) {+ for (; i < k; i += SGK1) {+#pragma unroll+ for (size_t ir = 0; ir < 1; ir++) {+ if (i + ir * TileK1 * SgSize + sg_id * TileK1 >= k) break;+ auto tmpu8 = *(sycl::vec<uint8_t, TileK1 / 4>*)(bptr + sg_id * TileK1 / 4);+ T scale = *(sptr + sg_id * TileK1 / blocksize);+ int8_t zp = zpptr ? *(zpptr + sg_id * TileK1 / blocksize) + int8_t(2) : int8_t(2);+ T tmpA[TileK1];+ *(sycl::vec<T, TileK1>*)tmpA = *(sycl::vec<T, TileK1>*)&aptr[sg_id * TileK1];+ T tacc[4] = {0.f, 0.f, 0.f, 0.f};+#pragma unroll+ for (int ikk = 0; ikk < TileK1; ikk += 4) {+ auto pack = tmpu8[ikk / 4];+ tacc[0] += tmpA[ikk + 0] * static_cast<T>(int2x4_to_0(pack) - zp);+ tacc[1] += tmpA[ikk + 1] * static_cast<T>(int2x4_to_1(pack) - zp);+ tacc[2] += tmpA[ikk + 2] * static_cast<T>(int2x4_to_2(pack) - zp);+ tacc[3] += tmpA[ikk + 3] * static_cast<T>(int2x4_to_3(pack) - zp);+ }+ tmpAcc += (tacc[0] + tacc[1] + tacc[2] + tacc[3]) * scale;+ sptr += TileK1 * SgSize / blocksize;+ zpptr += zpptr ? TileK1 * SgSize / blocksize : 0;+ aptr += TileK1 * SgSize;+ bptr += TileK1 * SgSize / 4;+ }+ sycl::group_barrier(sg);+ }+ }+ sycl::group_barrier(sg);+ auto sum = sycl::reduce_over_group(sg, tmpAcc, std::plus<>());+ if (bias) sum += bias[g_n];+ if (sg_id == 0) {+ *cptr = sum;+ }+ });+ });+ return ev;+ }+};++template <typename ScaleT>+class WeightS4T {+ public:+ using Param = ParamWeightS4Ext<ScaleT>;++ static __attribute__((always_inline)) inline int8_t int4x2_to_low(const uint8_t& src) {+ return static_cast<int8_t>(src & (uint8_t)0x0f) - (int8_t)8;+ }++ static __attribute__((always_inline)) inline int8_t int4x2_to_high(const uint8_t& src) {+ return static_cast<int8_t>(src >> (uint8_t)4) - (int8_t)8;+ }++ struct CfgDequantF32 {+ static int constexpr SgSize = 32;+ static int constexpr TileK = 16;+ static int constexpr TileN = 1;+ static int constexpr Unroll = 4;+ };++ struct CfgDequantF16 {+ static int constexpr SgSize = 32;+ static int constexpr TileK = 16;+ static int constexpr TileN = 1;+ static int constexpr Unroll = 4;+ };++ template <typename Cfg, typename BType>+ static inline sycl::event dequant(int n, int k, int blocksize, const Param& in, BType* outptr, sycl::queue* q) {+ int constexpr SgSize = Cfg::SgSize;+ int constexpr TileK = Cfg::TileK;+ int constexpr TileN = Cfg::TileN;+ static_assert(TileN == 1);+ int constexpr Unroll = Cfg::Unroll;+ int constexpr SubGroupK = SgSize * TileK;+ int constexpr GroupK = SgSize * TileK;+ assert(blocksize % Unroll == 0);+ int nsg_k = utils::updiv(k, GroupK);+ int nsg_n = n;+ sycl::range<1> group{SgSize};+ sycl::range<1> problem{static_cast<size_t>(nsg_n) * nsg_k * SgSize};+ auto B_d = in.B;+ auto S_d = in.scale;+ auto zp_d = in.zp;+ int ldb = in.ldb;+ int ldbn = in.ldb * blocksize;+ auto deq_kernel = [&](sycl::handler& cgh) {+ cgh.parallel_for(+ sycl::nd_range<1>(problem, group), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(SgSize)]] {+ int g_idx = it.get_group(0);+ auto sg = it.get_sub_group();+ int sg_id = sg.get_local_id()[0];+ int sg_group_id = sg.get_group_id()[0];+ int g_idx_n = g_idx / nsg_k;+ int g_idx_k = g_idx % nsg_k;+ int g_n = g_idx_n * 1;+ int g_k = g_idx_k * GroupK;+ int sg_k = g_k + sg_group_id * SubGroupK;+ int sg_k_remain = k - sg_k;+ auto sptr = S_d + sg_k / blocksize + g_n * ldb;+ auto zpptr = zp_d ? zp_d + sg_k / blocksize + g_n * ldb : nullptr;+ auto bptr = B_d + (sg_k + g_n * ldbn) / 2;+ auto dbptr = outptr + sg_k + g_n * k;+#pragma unroll+ for (int ik = 0; ik < TileK; ik += Unroll) {+ if (ik * SgSize + sg_id * Unroll >= sg_k_remain) break;+ BType dst[Unroll];+ BType scale = sptr[(ik * SgSize + sg_id * Unroll) / blocksize];+ int8_t zp = zpptr ? zpptr[(ik * SgSize + sg_id * Unroll) / blocksize] + int8_t(8) : int8_t(8);+#pragma unroll+ for (int ir = 0; ir < Unroll; ir += 2) {+ uint8_t srcu8 = *(bptr + (ik * SgSize + sg_id * Unroll + ir) / 2);+ dst[ir] = static_cast<BType>(INT4x2_TO_LO(srcu8) - zp) * scale;+ dst[ir + 1] = static_cast<BType>(INT4x2_TO_HI(srcu8) - zp) * scale;+ }+ *(sycl::vec<BType, Unroll>*)&dbptr[ik * SgSize + sg_id * Unroll] = *(sycl::vec<BType, Unroll>*)dst;+ }+ });+ };+ return q->submit(deq_kernel);+ }++ struct CfgDequantS8 {+ static int constexpr SgSize = 32;+ static int constexpr TileK = 16;+ static int constexpr TileN = 1;+ static int constexpr Unroll = 8;+ static bool constexpr Rescale = 0;+ };++ struct CfgDequantS8Rescale {+ static int constexpr SgSize = 32;+ static int constexpr TileK = 32;+ static int constexpr TileN = 1;+ static int constexpr Unroll = 16;+ static bool constexpr Rescale = 1;+ };++ template <class Cfg>+ static inline sycl::event dequantS8(int n, int k, int blocksize, const Param& in, int8_t* outptr, sycl::queue* q,+ int newblocksize = -1) {+ int constexpr SgSize = Cfg::SgSize;+ int constexpr TileK = Cfg::TileK;+ int constexpr TileN = Cfg::TileN;+ int constexpr Rescale = Cfg::Rescale;+ static_assert(TileN == 1);+ int constexpr Unroll = Cfg::Unroll;+ int constexpr SubGroupK = SgSize * TileK;+ int constexpr GroupK = SgSize * TileK;+ assert(blocksize % Unroll == 0);+ int nsg_k = utils::updiv(k, GroupK);+ int nsg_n = n;+ sycl::range<1> group{SgSize};+ sycl::range<1> problem{static_cast<size_t>(nsg_n) * nsg_k * SgSize};+ auto B_d = in.B;+ auto S_d = in.scale;+ auto Sext_d = in.scaleExt;+ auto zp_d = in.zp;+ int ldb = in.ldb;+ int ldbn = in.ldb * blocksize;+ int newblks = newblocksize == -1 ? 1 : k / newblocksize;+ auto deq_kernel = [&](sycl::handler& cgh) {+ cgh.parallel_for(+ sycl::nd_range<1>(problem, group), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(SgSize)]] {+ int g_idx = it.get_group(0);+ auto sg = it.get_sub_group();+ int sg_id = sg.get_local_id()[0];+ int sg_group_id = sg.get_group_id()[0];+ int g_idx_n = g_idx / nsg_k;+ int g_idx_k = g_idx % nsg_k;+ int g_n = g_idx_n * 1;+ int g_k = g_idx_k * GroupK;+ int sg_k = g_k + sg_group_id * SubGroupK;+ int sg_k_remain = k - sg_k;+ auto sptr = S_d + sg_k / blocksize + g_n * ldb;+ auto newsptr = Sext_d + sg_k / newblocksize + g_n * newblks;+ auto zpptr = zp_d ? zp_d + sg_k / blocksize + g_n * ldb : nullptr;+ auto bptr = B_d + (sg_k + g_n * ldbn) / 2;+ auto dbptr = outptr + sg_k + g_n * k;++#pragma unroll+ for (int ik = 0; ik < TileK; ik += Unroll) {+ if (ik * SgSize + sg_id * Unroll >= sg_k_remain) break;+ int8_t zp = zpptr ? zpptr[(ik * SgSize + sg_id * Unroll) / blocksize] + int8_t(8) : int8_t(8);+ int8_t dst[Unroll];+ if constexpr (Rescale) {+ float scaleN = (float)newsptr[(ik * SgSize + sg_id * Unroll) / newblocksize];+ float scale = (float)sptr[(ik * SgSize + sg_id * Unroll) / blocksize] / scaleN;+#pragma unroll+ for (int ir = 0; ir < Unroll; ir += 2) {+ uint8_t srcu8 = *(bptr + (ik * SgSize + sg_id * Unroll + ir) / 2);+ dst[ir] = std::round(static_cast<float>(INT4x2_TO_LO(srcu8) - zp) * scale);+ dst[ir + 1] = std::round(static_cast<float>(INT4x2_TO_HI(srcu8) - zp) * scale);+ }+ } else {+#pragma unroll+ for (int ir = 0; ir < Unroll; ir += 2) {+ uint8_t srcu8 = *(bptr + (ik * SgSize + sg_id * Unroll + ir) / 2);+ dst[ir] = static_cast<float>(INT4x2_TO_LO(srcu8) - zp);+ dst[ir + 1] = static_cast<float>(INT4x2_TO_HI(srcu8) - zp);+ }+ }+ *(sycl::vec<int8_t, Unroll>*)&dbptr[ik * SgSize + sg_id * Unroll] = *(sycl::vec<int8_t, Unroll>*)dst;+ }+ });+ };+ return q->submit(deq_kernel);+ }++ struct CfgGemvF32 {+ static int constexpr SgSize = 32;+ static int constexpr TileK0 = 8;+ static int constexpr TileK1 = 8;+ static int constexpr TileM = 1;+ static int constexpr Unroll = 2;+ static int constexpr PrefetchDis = 2;+ };++ struct CfgGemvF16 {+ static int constexpr SgSize = 32;+ static int constexpr TileK0 = 8;+ static int constexpr TileK1 = 8;+ static int constexpr TileM = 1;+ static int constexpr Unroll = 2;+ static int constexpr PrefetchDis = 2;+ };++ template <typename Cfg, typename T>+ static sycl::event gemv(const T* A, const Param& paramB, T* C, int n, int k, int blocksize, sycl::queue* q) {+ auto B = paramB.B;+ auto B_scale = paramB.scale;+ auto zp_d = paramB.zp;+ auto bias = (const T*)paramB.bias;+ int ldb = paramB.ldb;+ int constexpr SgSize = Cfg::SgSize;+ int constexpr TileK0 = Cfg::TileK0;+ int constexpr TileK1 = Cfg::TileK1;+ int constexpr TileM = Cfg::TileM;+ int constexpr Unroll = Cfg::Unroll;+ int constexpr PrefetchDis = Cfg::PrefetchDis;+ sycl::range<1> group{SgSize};+ sycl::range<1> problem{static_cast<size_t>(n) * SgSize};+ assert(k % TileK1 == 0);+ int constexpr SGK0 = TileK0 * SgSize * Unroll;+ int constexpr SGK1 = TileK1 * SgSize;+ int k_0 = utils::padto_le(k, SGK0);+ auto ev = q->submit([&](sycl::handler& cgh) {+ cgh.parallel_for(+ sycl::nd_range<1>(problem, group), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(SgSize)]] {+ int g_idx = it.get_group(0);+ auto sg = it.get_sub_group();+ int sg_id = sg.get_local_id()[0];+ int g_n = g_idx;+ auto sptr = B_scale + g_n * ldb;+ auto zpptr = zp_d ? zp_d + g_n * ldb : nullptr;+ auto bptr = B + g_n * k / 2;+ auto aptr = A;+ auto cptr = C + g_n;+ T tmpAcc = 0.f;+ int i = 0;+ if (i + SGK0 * PrefetchDis + SGK0 < k)+ for (int j = 0; j < PrefetchDis; j++) {+ sycl::ext::oneapi::experimental::joint_prefetch(sg, bptr + j * SGK0, SGK0 / 2);+ }++ sycl::group_barrier(sg);+ for (; i < k_0; i += SGK0) {+#pragma unroll+ for (size_t ir = 0; ir < Unroll; ir++) {+ auto tmpu8 = *(sycl::vec<uint8_t, TileK0 / 2>*)(bptr + sg_id * TileK0 / 2);+ T scale = *(sptr + sg_id * TileK0 / blocksize);+ int8_t zp = zpptr ? *(zpptr + sg_id * TileK0 / blocksize) + int8_t(8) : int8_t(8);+ T tmpA[TileK0];+ *(sycl::vec<T, TileK0>*)tmpA = *(sycl::vec<T, TileK0>*)&aptr[sg_id * TileK0];+ T tacc[2] = {0.f, 0.f};+#pragma unroll+ for (int ikk = 0; ikk < TileK0; ikk += 2) {+ tacc[0] += tmpA[ikk + 0] * static_cast<T>(INT4x2_TO_LO(tmpu8[ikk / 2]) - zp);+ tacc[1] += tmpA[ikk + 1] * static_cast<T>(INT4x2_TO_HI(tmpu8[ikk / 2]) - zp);+ }+ tmpAcc += (tacc[0] + tacc[1]) * scale;+ sptr += TileK0 * SgSize / blocksize;+ zpptr += zpptr ? TileK0 * SgSize / blocksize : 0;+ aptr += TileK0 * SgSize;+ bptr += TileK0 * SgSize / 2;+ }+ if (i + SGK0 * PrefetchDis + SGK0 < k)+ sycl::ext::oneapi::experimental::joint_prefetch(sg, bptr + i + SGK0 * PrefetchDis, SGK0 / 2);+ sycl::group_barrier(sg);+ }+ if (i < k) {+ for (; i < k; i += SGK1) {+#pragma unroll+ for (size_t ir = 0; ir < 1; ir++) {+ if (i + ir * TileK1 * SgSize + sg_id * TileK1 >= k) break;+ auto tmpu8 = *(sycl::vec<uint8_t, TileK1 / 2>*)(bptr + sg_id * TileK1 / 2);+ T scale = *(sptr + sg_id * TileK1 / blocksize);+ int8_t zp = zpptr ? *(zpptr + sg_id * TileK1 / blocksize) + int8_t(8) : int8_t(8);+ T tmpA[TileK1];+ *(sycl::vec<T, TileK1>*)tmpA = *(sycl::vec<T, TileK1>*)&aptr[sg_id * TileK1];+ T tacc[2] = {0.f, 0.f};+#pragma unroll+ for (int ikk = 0; ikk < TileK1; ikk += 2) {+ tacc[0] += tmpA[ikk + 0] * static_cast<T>(INT4x2_TO_LO(tmpu8[ikk / 2]) - zp);+ tacc[1] += tmpA[ikk + 1] * static_cast<T>(INT4x2_TO_HI(tmpu8[ikk / 2]) - zp);+ }+ tmpAcc += (tacc[0] + tacc[1]) * scale;+ sptr += TileK1 * SgSize / blocksize;+ zpptr += zpptr ? TileK0 * SgSize / blocksize : 0;+ aptr += TileK1 * SgSize;+ bptr += TileK1 * SgSize / 2;+ }+ sycl::group_barrier(sg);+ }+ }+ sycl::group_barrier(sg);+ auto sum = sycl::reduce_over_group(sg, tmpAcc, std::plus<>());+ if (bias) sum += bias[g_n];+ if (sg_id == 0) {+ *cptr = sum;+ }+ });+ });+ return ev;+ }+};++template <typename ScaleT>+class WeightS8T {+ public:+ using Param = ParamWeightS8<ScaleT>;++ struct CfgDequantF32 {+ static int constexpr SgSize = 16;+ static int constexpr TileK = 8;+ static int constexpr TileN = 1;+ static int constexpr Unroll = 4;+ };++ struct CfgDequantF16 {+ static int constexpr SgSize = 32;+ static int constexpr TileK = 8;+ static int constexpr TileN = 1;+ static int constexpr Unroll = 4;+ };++ template <typename Cfg, typename BType>+ static inline sycl::event dequant(int n, int k, int blocksize, const Param& in, BType* outptr, sycl::queue* q) {+ int constexpr SgSize = Cfg::SgSize;+ int constexpr TileK = Cfg::TileK;+ int constexpr Unroll = Cfg::Unroll;+ int constexpr SubGroupK = SgSize * TileK;+ int constexpr GroupK = SgSize * TileK;+ assert(blocksize % Unroll == 0);++ int nsg_k = utils::updiv(k, GroupK);+ int nsg_n = n;+ sycl::range<1> group{SgSize};+ sycl::range<1> problem{static_cast<size_t>(nsg_n) * nsg_k * SgSize};+ auto B_d = in.B;+ auto S_d = in.scale;+ auto zp_d = in.zp;+ int ldb = in.ldb;+ int ldbn = in.ldb * blocksize;+ auto deq_kernel = [&](sycl::handler& cgh) {+ cgh.parallel_for(+ sycl::nd_range<1>(problem, group), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(SgSize)]] {+ int g_idx = it.get_group(0);+ auto sg = it.get_sub_group();+ int sg_id = sg.get_local_id()[0];+ int sg_group_id = sg.get_group_id()[0];+ int g_idx_n = g_idx / nsg_k;+ int g_idx_k = g_idx % nsg_k;+ int g_n = g_idx_n * 1;+ int g_k = g_idx_k * GroupK;+ int sg_k = g_k + sg_group_id * SubGroupK;+ int sg_k_remain = k - sg_k;+ auto sptr = S_d + sg_k / blocksize + g_n * ldb;+ auto zpptr = zp_d ? zp_d + sg_k / blocksize + g_n * ldb : nullptr;+ auto bptr = B_d + (sg_k + g_n * ldbn);+ auto dbptr = outptr + sg_k + g_n * k;+#pragma unroll+ for (int ik = 0; ik < TileK; ik += Unroll) {+ if (ik * SgSize + sg_id * Unroll >= sg_k_remain) break;+ BType dst[Unroll];+ BType scale = sptr[(ik * SgSize + sg_id * Unroll) / blocksize];+ int8_t zp = zpptr ? zpptr[(ik * SgSize + sg_id * Unroll) / blocksize] : int8_t(0);+ int8_t srcs8[Unroll];+ *(sycl::vec<int8_t, Unroll>*)srcs8 = *(sycl::vec<int8_t, Unroll>*)(bptr + (ik * SgSize + sg_id * Unroll));+#pragma unroll+ for (int ir = 0; ir < Unroll; ir += 1) {+ dst[ir] = (BType)((int16_t)srcs8[ir] - (int16_t)zp) * scale;+ }+ *(sycl::vec<BType, Unroll>*)&dbptr[ik * SgSize + sg_id * Unroll] = *(sycl::vec<BType, Unroll>*)dst;+ }+ });+ };+ return q->submit(deq_kernel);+ }++ struct CfgGemvF32 {+ static int constexpr SgSize = 32;+ static int constexpr TileK0 = 16;+ static int constexpr TileK1 = 4;+ static int constexpr TileM = 1;+ };++ struct CfgGemvF16 {+ static int constexpr SgSize = 32;+ static int constexpr TileK0 = 16;+ static int constexpr TileK1 = 4;+ static int constexpr TileM = 1;+ };++ template <typename Cfg, typename T>+ static sycl::event gemv(const T* A, const Param& paramB, T* C, int n, int k, int blocksize, sycl::queue* q) {+ auto B = paramB.B;+ auto B_scale = paramB.scale;+ auto zp_d = paramB.zp;+ auto bias = (const T*)paramB.bias;+ int ldb = paramB.ldb;+ int constexpr SgSize = Cfg::SgSize;+ int constexpr TileK0 = Cfg::TileK0;+ int constexpr TileK1 = Cfg::TileK1;+ int constexpr TileM = Cfg::TileM;+ sycl::range<1> group{SgSize};+ sycl::range<1> problem{static_cast<size_t>(n) * SgSize};+ int constexpr SGK0 = TileK0 * SgSize;+ int constexpr SGK1 = TileK1 * SgSize;+ int k_0 = utils::padto_le(k, SGK0);+ auto ev = q->submit([&](sycl::handler& cgh) {+ cgh.parallel_for(sycl::nd_range<1>(problem, group),+ [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(SgSize)]] {+ int g_idx = it.get_group(0);+ auto sg = it.get_sub_group();+ int sg_id = sg.get_local_id()[0];+ int g_n = g_idx;+ auto sptr = B_scale + g_n * ldb;+ auto zpptr = zp_d ? zp_d + g_n * ldb : nullptr;+ auto bptr = B + g_n * k;+ auto aptr = A;+ auto cptr = C + g_n;+ T tmpAcc = 0.f;+ int i = 0;+ for (; i < k_0; i += SGK0) {+ auto tmps8 = *(sycl::vec<int8_t, TileK0>*)(bptr + sg_id * TileK0);+ T scale = *(sptr + sg_id * TileK0 / blocksize);+ int16_t zp = zpptr ? *(zpptr + sg_id * TileK0 / blocksize) : int16_t(0);+#pragma unroll+ for (int ikk = 0; ikk < TileK0; ikk += 1) {+ tmpAcc += T(aptr[sg_id * TileK0 + ikk]) * T((int16_t)tmps8[ikk] - zp) * scale;+ }+ sptr += SGK0 / blocksize;+ zpptr += zpptr ? SGK0 / blocksize : 0;+ aptr += SGK0;+ bptr += SGK0;+ }+ if (i < k) {+ for (; i < k; i += SGK1) {+ if (i + sg_id * TileK1 >= k) break;+ auto tmps8 = *(sycl::vec<int8_t, TileK1>*)(bptr + sg_id * TileK1);+ T scale = *(sptr + sg_id * TileK1 / blocksize);+ int16_t zp = zpptr ? *(zpptr + sg_id * TileK1 / blocksize) : int16_t(0);+#pragma unroll+ for (int ikk = 0; ikk < TileK1; ikk += 1) {+ tmpAcc += T(aptr[sg_id * TileK1 + ikk + 0]) * T((int16_t)tmps8[ikk] - zp) * scale;+ }++ sptr += SGK1 / blocksize;+ zpptr += zpptr ? SGK1 / blocksize : 0;+ aptr += SGK1;+ bptr += SGK1;+ }+ }++ auto sum = sycl::reduce_over_group(sg, tmpAcc, std::plus<>());+ if (bias) sum += bias[g_n];+ if (sg_id == 0) {+ *cptr = sum;+ }+ });+ });+ return ev;+ }+};++// FP8 weight dequantization and GEMM support+// FP8 can be E4M3 or E5M2 format, stored as uint8_t+template <typename ScaleT, bool E4M3 = true>+class WeightF8T {+ public:+ using Param = ParamWeightF8<ScaleT>;+ static constexpr bool IsE4M3 = E4M3;++ static inline float scale_to_f32(const ScaleT& s) {+ if constexpr (std::is_same_v<ScaleT, bestla::utils::f8>) {+ return s.tofloat();+ } else {+ return static_cast<float>(s);+ }+ }++ template <typename ItemT, typename BType>+ static inline void init_slm_lut(const sycl::local_accessor<BType, 1>& slm_lut, const ItemT& it) {+ int local_id = it.get_local_id(0);+ int local_range = it.get_local_range(0);+ const float* lut_data = E4M3 ? fp8_lut::lut_e4m3_128.data() : fp8_lut::lut_e5m2_128.data();+ for (int i = local_id; i < 128; i += local_range) {+ slm_lut[i] = BType(lut_data[i]);+ }+ sycl::group_barrier(it.get_group());+ }++ class Cfg {+ public:+ static int constexpr SgSize = 32;+ static int constexpr TileK = 16;+ static int constexpr TileN = 1;+ };++ template <typename Cfg, typename BType>+ static inline sycl::event dequant(int n, int k, int blocksize, const Param& in, BType* outptr, sycl::queue* q) {+ int constexpr SgSize = Cfg::SgSize;+ int constexpr TileK = Cfg::TileK;+ int constexpr SubGroupK = SgSize * TileK;+ int constexpr GroupK = SgSize * TileK;+ assert(blocksize % TileK == 0);++ int nsg_k = utils::updiv(k, GroupK);+ int nsg_n = n;+ sycl::range<1> group{SgSize};+ sycl::range<1> problem{static_cast<size_t>(nsg_n) * nsg_k * SgSize};+ auto B_d = in.B;+ auto S_d = in.scale;+ int ldb = in.ldb;+ int ldbn = in.ldb * blocksize;+ auto deq_kernel = [&](sycl::handler& cgh) {+ sycl::local_accessor<BType, 1> slm_lut(sycl::range<1>(128), cgh);+ cgh.parallel_for(+ sycl::nd_range<1>(problem, group), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(SgSize)]] {+ init_slm_lut(slm_lut, it);+ int g_idx = it.get_group(0);+ auto sg = it.get_sub_group();+ int sg_id = sg.get_local_id()[0];+ int sg_group_id = sg.get_group_id()[0];+ int g_idx_n = g_idx / nsg_k;+ int g_idx_k = g_idx % nsg_k;+ int g_n = g_idx_n * 1;+ int g_k = g_idx_k * GroupK;+ int sg_k = g_k + sg_group_id * SubGroupK;+ int sg_k_remain = k - sg_k;+ if (sg_k_remain >= SubGroupK) {+ auto sptr = S_d + sg_k / blocksize + g_n * ldb;+ auto bptr = B_d + (sg_k + g_n * ldbn);+ auto dbptr = outptr + sg_k + g_n * k;+ int constexpr Unroll = 4;+#pragma unroll+ for (int ik = 0; ik < TileK; ik += Unroll) {+ BType dst[Unroll];+ BType scale = static_cast<BType>(scale_to_f32(sptr[(ik * SgSize + sg_id * Unroll) / blocksize]));+#pragma unroll+ for (int ir = 0; ir < Unroll; ir += 1) {+ uint8_t srcfp8 = *(bptr + (ik * SgSize + sg_id * Unroll + ir));+ BType val = slm_lut[srcfp8 & 0x7F];+ if (srcfp8 & 0x80) val = -val;+ dst[ir] = val * scale;+ }+ *(sycl::vec<BType, Unroll>*)&dbptr[ik * SgSize + sg_id * Unroll] = *(sycl::vec<BType, Unroll>*)dst;+ }+ } else {+ auto sptr = S_d + sg_k / blocksize + g_n * ldb;+ auto bptr = B_d + (sg_k + g_n * ldbn);+ auto dbptr = outptr + sg_k + g_n * k;+ int constexpr Unroll = 4;+ for (int ik = 0; ik < TileK; ik += Unroll) {+ if (ik * SgSize + sg_id * Unroll >= sg_k_remain) break;+ BType dst[Unroll];+ BType scale = static_cast<BType>(scale_to_f32(sptr[(ik * SgSize + sg_id * Unroll) / blocksize]));+#pragma unroll+ for (int ir = 0; ir < Unroll; ir += 1) {+ uint8_t srcfp8 = *(bptr + (ik * SgSize + sg_id * Unroll + ir));+ BType val = slm_lut[srcfp8 & 0x7F];+ if (srcfp8 & 0x80) val = -val;+ dst[ir] = val * scale;+ }+ *(sycl::vec<BType, Unroll>*)&dbptr[ik * SgSize + sg_id * Unroll] = *(sycl::vec<BType, Unroll>*)dst;+ }+ }+ });+ };+ return q->submit(deq_kernel);+ }++ template <typename T>+ static inline sycl::event gemv(const T* A, const Param& paramB, T* C, int n, int k, int blocksize, sycl::queue* q) {+ auto B = paramB.B;+ auto B_scale = paramB.scale;+ int ldb = paramB.ldb;+ const T* bias_ptr = reinterpret_cast<const T*>(paramB.bias);+ int constexpr SgSize = 32;+ sycl::range<1> group{SgSize};+ sycl::range<1> problem{static_cast<size_t>(n) * SgSize};++ // Configuration based on data type+ constexpr bool is_fp32 = std::is_same_v<T, float>;+ int constexpr TileK0 = is_fp32 ? 8 : 16; // Reduce tile size for FP32+ int constexpr TileK1 = 8;+ // FP32: Use 8 accumulators (match TileK0) and 4x unroll for max parallelism+ // FP16: Use 4 accumulators and 2x unroll (balanced)+ int constexpr Unroll0 = is_fp32 ? 4 : 2;+ int constexpr NumAcc = is_fp32 ? 8 : 4;++ int constexpr SGK0 = TileK0 * SgSize;+ int constexpr SGK1 = TileK1 * SgSize;+ int k_0 = utils::padto_le(k, SGK0 * Unroll0);+ int k_1 = utils::padto_le(k, SGK1);+ auto ev = q->submit([&](sycl::handler& cgh) {+ sycl::local_accessor<T, 1> slm_lut(sycl::range<1>(128), cgh);+ cgh.parallel_for(sycl::nd_range<1>(problem, group),+ [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(SgSize)]] {+ init_slm_lut(slm_lut, it);+ int g_idx = it.get_group(0);+ auto sg = it.get_sub_group();+ int sg_id = sg.get_local_id()[0];+ int g_n = g_idx;+ auto sptr = B_scale + g_n * ldb;+ auto bptr = B + g_n * k;+ auto aptr = A;+ auto cptr = C + g_n;++ // Multiple accumulators+ T tmpAcc[NumAcc];+#pragma unroll+ for (int i = 0; i < NumAcc; ++i) tmpAcc[i] = 0.f;++ int i = 0;+ // Main loop with 2x unrolling + vectorized LUT lookup+ for (; i < k_0; i += SGK0 * Unroll0) {+#pragma unroll+ for (int iu = 0; iu < Unroll0; iu++) {+ auto tmpfp8 = *(sycl::vec<uint8_t, TileK0>*)(bptr + sg_id * TileK0);+ T scale = static_cast<T>(scale_to_f32(*(sptr + sg_id * TileK0 / blocksize)));+ auto tmpA = *(sycl::vec<T, TileK0>*)&aptr[sg_id * TileK0];++ // Vectorized LUT lookup + fused scale multiply+ T tmpB[TileK0];+#pragma unroll+ for (int ikk = 0; ikk < TileK0; ikk += 1) {+ uint8_t u8 = tmpfp8[ikk];+ T val = slm_lut[u8 & 0x7F];+ if (u8 & 0x80) val = -val;+ tmpB[ikk] = val * scale;+ }++ // Fused multiply-accumulate with multiple accumulators+#pragma unroll+ for (int ikk = 0; ikk < TileK0; ikk += NumAcc) {+#pragma unroll+ for (int ia = 0; ia < NumAcc; ++ia) {+ tmpAcc[ia] += T(tmpA[ikk + ia]) * tmpB[ikk + ia];+ }+ }+ sptr += SGK0 / blocksize;+ aptr += SGK0;+ bptr += SGK0;+ }+ }+ // Secondary loop with larger TileK1+ for (; i < k_1; i += SGK1) {+ auto tmpfp8 = *(sycl::vec<uint8_t, TileK1>*)(bptr + sg_id * TileK1);+ T scale = static_cast<T>(scale_to_f32(*(sptr + sg_id * TileK1 / blocksize)));+ auto tmpA = *(sycl::vec<T, TileK1>*)&aptr[sg_id * TileK1];++ // Vectorized LUT lookup + fused scale multiply+ T tmpB[TileK1];+#pragma unroll+ for (int ikk = 0; ikk < TileK1; ikk += 1) {+ uint8_t u8 = tmpfp8[ikk];+ T val = slm_lut[u8 & 0x7F];+ if (u8 & 0x80) val = -val;+ tmpB[ikk] = val * scale;+ }++ // Fused multiply-accumulate+#pragma unroll+ for (int ikk = 0; ikk < TileK1; ikk += 1) {+ tmpAcc[ikk % NumAcc] += T(tmpA[ikk]) * tmpB[ikk];+ }++ sptr += SGK1 / blocksize;+ aptr += SGK1;+ bptr += SGK1;+ }++ // Combine accumulators+ T finalAcc = 0.f;+#pragma unroll+ for (int i = 0; i < NumAcc; ++i) finalAcc += tmpAcc[i];++ float sum = sycl::reduce_over_group(sg, static_cast<float>(finalAcc), sycl::plus<float>());+ if (sg_id == 0) {+ if (bias_ptr) {+ sum += static_cast<float>(bias_ptr[g_n]);+ }+ *cptr = static_cast<T>(sum);+ }+ });+ });+ return ev;+ }+};++template <class PrologueT>+struct needs_fp8_lut : std::false_type {};++template <typename ScaleT, bool Flag>+struct needs_fp8_lut<WeightF8T<ScaleT, Flag>> : std::true_type {};+} // namespace sycl_prologue_b+} // namespace bestla+#endifdiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_storage.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_storage.hnew file mode 100644index 000000000..f7e98c1fa--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_storage.h@@ -0,0 +1,124 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include "sycl_utils.h"+#include "bestla/bestla_storage.h"++namespace bestla {+namespace sycl_storage {+class StorageWeightKBlockNInteger {+ public:+ BTLA_PROLOGUEB_IDS mPrologueID = BTLA_PROLOGUEB_IDS::Undef;+ uint64_t mCoreId = 0;+ BTLA_DTYPE mDType = BTLA_DTYPE::F32;+ BTLA_DTYPE mScaT = BTLA_DTYPE::F32, mZpT = BTLA_DTYPE::F32, mRedT = BTLA_DTYPE::S32;+ int mNPad = 0, mKPad = 0;+ int mN = 0, mK = 0;+ int mBlockSize = 1;+ int mDqBlockSize = 0;+ int mCStep = 0;+ int8_t* mQBuf = nullptr;+ size_t mWSize = 0;+ int8_t* mSBuf = nullptr;+ size_t mCSize = 0;+ int8_t *mZpBuf = nullptr, *mRedBuf = nullptr;+ size_t mZpSize = 0, mRedSize = 0;+ int8_t* mDQCorrectionBuf = nullptr;+ int8_t* mShuffleIndices = nullptr;+ size_t mDQCorSize = 0, mShufSize = 0;++ StorageWeightKBlockNInteger(bestla::storage::gemm::StorageWeightKBlockNInteger& _hoststor) {+ mPrologueID = BTLA_PROLOGUEB_IDS::WeightKBlockNInteger;+ mCoreId = 0;+ mDType = _hoststor.mDType;+ mScaT = _hoststor.SDtype();+ mZpT = _hoststor.ZDtype();+ mNPad = _hoststor.mNPad;+ mKPad = _hoststor.mKPad;+ mN = _hoststor.mN;+ mK = _hoststor.mK;+ mBlockSize = _hoststor.mBlockSize;+ mDqBlockSize = _hoststor.mDqBlockSize;+ mWSize = _hoststor.template WSize<int8_t>();+ mCSize = _hoststor.CSize() * utils::bestla_dtype_bytes(mScaT);+ mCStep = _hoststor.CStep();++ if (_hoststor.template ZPtr<void>()) {+ mZpSize = _hoststor.CSize() * utils::bestla_dtype_bytes(mZpT);+ }+ if (_hoststor.template RPtr<void>()) {+ mRedSize = _hoststor.CSize() * utils::bestla_dtype_bytes(mRedT);+ }+ // TODO DQ,shuffle support+ }++ size_t getDeviceSize() { return mWSize + mCSize + mZpSize + mRedSize + mDQCorSize + mShufSize; }++ void assign(int8_t* dptr) {+ mQBuf = dptr;+ dptr += mWSize;+ mSBuf = dptr;+ dptr += mCSize;+ if (mZpSize) {+ mZpBuf = dptr;+ dptr += mZpSize;+ }+ if (mRedSize) {+ mRedBuf = dptr;+ dptr += mRedSize;+ }+ if (mDQCorSize) {+ mDQCorrectionBuf = dptr;+ dptr += mDQCorSize;+ }+ if (mShuffleIndices) {+ mDQCorrectionBuf = dptr;+ dptr += mShufSize;+ }+ }++ void fromHost(bestla::storage::gemm::StorageWeightKBlockNInteger& _hoststor, sycl::queue* queue) {+ if (_hoststor.template WPtr<void>() && mQBuf) {+ queue->memcpy(mQBuf, _hoststor.template WPtr<void>(), mWSize);+ }+ if (_hoststor.template SPtr<void>() && mSBuf) {+ queue->memcpy(mSBuf, _hoststor.template SPtr<void>(), mCSize);+ }+ if (_hoststor.template ZPtr<void>() && mZpBuf) {+ queue->memcpy(mZpBuf, _hoststor.template ZPtr<void>(), mZpSize);+ }+ if (_hoststor.template RPtr<void>() && mRedBuf) {+ queue->memcpy(mRedBuf, _hoststor.template RPtr<void>(), mRedSize);+ }+ queue->wait();+ }++ void toHost(bestla::storage::gemm::StorageWeightKBlockNInteger& _hoststor, sycl::queue* queue) {+ if (mQBuf) {+ queue->memcpy(_hoststor.template WPtr<void>(), mQBuf, mWSize);+ }+ if (mSBuf) {+ queue->memcpy(_hoststor.template SPtr<void>(), mSBuf, mCSize);+ }+ if (mZpBuf) {+ queue->memcpy(_hoststor.template ZPtr<void>(), mZpBuf, mZpSize);+ }+ if (mRedBuf) {+ queue->memcpy(_hoststor.template RPtr<void>(), mRedBuf, mRedSize);+ }+ queue->wait();+ }+};+} // namespace sycl_storage+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_utils.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_utils.hnew file mode 100644index 000000000..606355a4e--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_utils.h@@ -0,0 +1,110 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once+#include "sycl_device.h"+#include "bestla/bestla_utils.h"++namespace bestla {+namespace sycl_utils {++struct sycl_deleter {+ sycl::queue* queue_;+ sycl_deleter(sycl::queue* _q) : queue_(_q) {}+ template <class T>+ void operator()(T* obj) const {+ if (obj) {+ sycl::free(obj, *queue_);+ }+ }+};++template <typename _T>+struct sycl_vector {+ sycl_vector(uint64_t _size = 0, sycl::queue* _q = nullptr) : size_(_size) {+ if (_q && _size) {+ resize(_size, _q);+ }+ }++ void resize(uint64_t _size, sycl::queue* _q) {+ size_ = _size;+ _T* tmp = sycl::malloc_device<_T>(_size, *_q);+ ptr_ = std::shared_ptr<_T>(tmp, sycl_deleter(_q));+ }++ inline uint64_t size() { return size_; }++ inline _T* data() { return ptr_.get(); }++ std::shared_ptr<_T> ptr_;+ uint64_t size_;+};++template <typename T>+__inline__ std::vector<T> sycl2host(const T* syclptr, size_t elecount, sycl::queue* q) {+ std::vector<T> tmp(elecount);+ q->memcpy(tmp.data(), syclptr, elecount * sizeof(T)).wait();+ return tmp;+}++class event_helper {+ public:+ static float elapsed_time(sycl::event& evt) {+ float t = 0.f;+ const auto startKernExecutionTimePoint = evt.get_profiling_info<sycl::info::event_profiling::command_submit>();+ const auto endKernExecutionTimePoint = evt.get_profiling_info<sycl::info::event_profiling::command_end>();+ t = (endKernExecutionTimePoint - startKernExecutionTimePoint) / 1e6;+ return t;+ }++ static float execute_time(sycl::event& evt) {+ float t = 0.f;+ const auto startKernExecutionTimePoint = evt.get_profiling_info<sycl::info::event_profiling::command_start>();+ const auto endKernExecutionTimePoint = evt.get_profiling_info<sycl::info::event_profiling::command_end>();+ t = (endKernExecutionTimePoint - startKernExecutionTimePoint) / 1e6;+ return t;+ }+};+template <class GemmCoreT>+class nd_item_helper {+ public:+ const sycl::nd_item<2> it;+ const sycl::sub_group sg;+ nd_item_helper(sycl::nd_item<2>& _it) : it(_it), sg(it.get_sub_group()) {}++ constexpr inline void local_barrier() const { it.barrier(sycl::access::fence_space::local_space); }++ constexpr inline int sg_group_id() const { return sg.get_group_id()[0]; }++ constexpr inline int wg_idx_m() const { return it.get_group(0); }+ constexpr inline int wg_size_m() const { return GemmCoreT::WgM * GemmCoreT::TileM; }+ constexpr inline int wg_g_m() const { return wg_idx_m() * wg_size_m(); }++ constexpr inline int wg_idx_n() const { return it.get_group(1); }+ constexpr inline int wg_size_n() const { return GemmCoreT::WgN * GemmCoreT::TileN; }+ constexpr inline int wg_g_n() const { return wg_idx_n() * wg_size_n(); }++ constexpr inline int sg_idx_m() const { return sg_group_id() / GemmCoreT::SgNStride; }+ constexpr inline int sg_g_m() const { return wg_g_m() + sg_idx_m() * GemmCoreT::TileM; }++ constexpr inline int sg_idx_n() const { return sg_group_id() % GemmCoreT::SgNStride; }+ constexpr inline int sg_g_n() const { return wg_g_n() + sg_idx_n() * GemmCoreT::SgSize * GemmCoreT::TileN; }++ constexpr inline int sg_id() const { return sg.get_local_id()[0]; }+ constexpr inline int item_g_m() const { return sg_g_m(); }+ constexpr inline int item_g_n() const { return sg_g_n() + sg_id() * GemmCoreT::TileN; }+};++} // namespace sycl_utils+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_wrapper.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_wrapper.hnew file mode 100644index 000000000..238de670b--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/sycl/sycl_wrapper.h@@ -0,0 +1,403 @@+// Copyright (c) 2023 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once++#ifdef BTLA_SYCL+#include <sycl/sycl.hpp>++#include "bestla/bestla_utils.h"+#include "sycl_utils.h"+#include "sycl_device.h"+#include "sycl_gemm.h"+#include "sycl_epilogue.h"+#include "sycl_prologue_a.h"+#include "sycl_prologue_b.h"++namespace bestla {+namespace sycl_wrapper {+template <template <class GCT> class ProAT, template <class GCT> class ProBT, template <class GCT> class EpiT,+ class GemmCoreT>+class Launcher {+ public:+ using GemmCore = GemmCoreT;+ using PrologueA = ProAT<GemmCore>;+ using PrologueB = ProBT<GemmCore>;+ using Epilogue = EpiT<GemmCore>;+ using AType = typename GemmCore::TA;+ using AParam = typename PrologueA::Param;+ using BType = typename GemmCore::TB;+ using BParam = typename PrologueB::Param;+ using CType = typename GemmCore::TC;+ using ACCType = typename GemmCore::TACC;+ using EpiParam = typename Epilogue::Param;+ struct Param {+ const AParam paramA;+ const BParam paramB;+ const EpiParam paramC;+ };+ template <bool debug = false>+ static inline sycl::event compute(sycl::queue* q, int m, int n, int k, const Param& _param) {+ sycl::range<2> group{GemmCore::WgM, GemmCore::WgN};+ auto A = _param.paramA.A;+ auto B = _param.paramB.B;+ auto C = _param.paramC.C;+ int lda = _param.paramA.lda;+ int ldb = _param.paramB.ldb;+ int ldc = _param.paramC.ldc;+ int m_pad = utils::padto(utils::updiv(m, GemmCore::TileM), GemmCore::WgM);+ sycl::range<2> problem{static_cast<size_t>(m_pad), static_cast<size_t>(n) / GemmCore::TileN};+ auto ev = q->submit([&](sycl::handler& cgh) {+ sycl::local_accessor<BType, 1> slm_b(sycl::range(GemmCore::SLM_B_Size), cgh);+ cgh.parallel_for(sycl::nd_range<2>(problem, group),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(GemmCore::SgSize)]] {+ sycl_utils::nd_item_helper<GemmCore> helper(it);+ if constexpr (debug) {+ compute_tile(k, B, ldb, slm_b, A, lda, C, ldc, it);+ } else {+ int m_tail = m - helper.sg_g_m();+ m_tail = m_tail > GemmCore::TileM ? GemmCore::TileM : m_tail;+ if (m_tail == GemmCore::TileM) {+ compute_tile(k, B, ldb, slm_b, A, lda, C, ldc, it);+ } else {+ compute_tail(k, B, ldb, slm_b, A, lda, C, ldc, m_tail, it);+ }+ }+ });+ });+ return ev;+ }++ static void compute_tile(int k, const BType* B, int ldb, const sycl::local_accessor<BType, 1>& slm_b, const AType* A,+ int lda, CType* C, int ldc, sycl::nd_item<2>& it) {+ sycl_utils::nd_item_helper<GemmCore> helper(it);+ ACCType tmp[GemmCore::TileM * GemmCore::TileN];+ for (size_t im = 0; im < GemmCore::TileM; im++)+ for (size_t in = 0; in < GemmCore::TileN; in++) tmp[im * GemmCore::TileN + in] = ACCType(0.f);+ for (int i = 0; i < k; i += GemmCore::TileK) {+ PrologueB::getWeight({B, ldb}, slm_b, i, helper);+ it.barrier(sycl::access::fence_space::local_space);+ GemmCore::compute(&A[helper.item_g_m() * lda + i], lda, slm_b, tmp, helper);+ it.barrier(sycl::access::fence_space::local_space);+ }+ Epilogue::store({C, ldc}, tmp, helper);+ }++ static void compute_tail(int k, const BType* B, int ldb, const sycl::local_accessor<BType, 1>& slm_b, const AType* A,+ int lda, CType* C, int ldc, int m_tail, sycl::nd_item<2>& it) {+ sycl_utils::nd_item_helper<GemmCore> helper(it);+ ACCType tmp[GemmCore::TileM * GemmCore::TileN];+ for (size_t im = 0; im < GemmCore::TileM; im++)+ for (size_t in = 0; in < GemmCore::TileN; in++) tmp[im * GemmCore::TileN + in] = ACCType(0.f);+ for (int i = 0; i < k; i += GemmCore::TileK) {+ PrologueB::getWeight({B, ldb}, slm_b, i, helper);+ it.barrier(sycl::access::fence_space::local_space);+ GemmCore::compute_mtail(&A[helper.item_g_m() * lda + i], lda, slm_b, tmp, helper, m_tail);+ it.barrier(sycl::access::fence_space::local_space);+ }+ Epilogue::store_tail({C, ldc}, tmp, helper, m_tail);+ }+};++template <template <class GCT> class ProAT, template <class GCT> class ProBT, template <class GCT> class EpiT,+ class GemmCoreT>+class LauncherWOQ {+ public:+ using GemmCore = GemmCoreT;+ using PrologueA = ProAT<GemmCore>;+ using PrologueB = ProBT<GemmCore>;+ using Epilogue = EpiT<GemmCore>;+ using AType = typename GemmCore::TA;+ using AParam = typename PrologueA::Param;+ using BType = typename GemmCore::TB;+ using BParam = typename PrologueB::Param;+ using CType = typename GemmCore::TC;+ using ACCType = typename GemmCore::TACC;+ using EpiParam = typename Epilogue::Param;+ struct Param {+ const AParam paramA;+ const BParam paramB;+ const EpiParam paramC;+ };+ static constexpr bool NeedFp8Lut = sycl_prologue_b::needs_fp8_lut<PrologueB>::value;++ template <typename ParamT>+ static inline void load_weight_tile(const ParamT& paramB, const sycl::local_accessor<BType, 1>& slm_b, int koffset,+ int blocksize, const sycl_utils::nd_item_helper<GemmCore>& helper,+ const sycl::local_accessor<BType, 1>* slm_lut) {+ if constexpr (NeedFp8Lut) {+ PrologueB::getWeight(paramB, slm_b, koffset, blocksize, helper, *slm_lut);+ } else {+ PrologueB::getWeight(paramB, slm_b, koffset, blocksize, helper);+ }+ }++ template <bool debug = false>+ static inline sycl::event compute(sycl::queue* q, int m, int n, int k, int blocksize, const Param& _param) {+ sycl::range<2> group{GemmCore::WgM, GemmCore::WgN};+ auto A = _param.paramA.A;+ auto B = _param.paramB.B;+ auto B_scale = _param.paramB.scale;+ auto C = _param.paramC.C;+ int lda = _param.paramA.lda;+ int ldb = _param.paramB.ldb;+ int ldc = _param.paramC.ldc;+ int m_pad = utils::padto(utils::updiv(m, GemmCore::TileM), GemmCore::WgM);+ sycl::range<2> problem{static_cast<size_t>(m_pad), static_cast<size_t>(n) / GemmCore::TileN};+ auto ev = q->submit([&](sycl::handler& cgh) {+ sycl::local_accessor<BType, 1> slm_b(sycl::range(GemmCore::SLM_B_Size), cgh);+ [[maybe_unused]] sycl::local_accessor<BType, 1> slm_lut(sycl::range<1>(NeedFp8Lut ? 256 : 1), cgh);+ cgh.parallel_for(sycl::nd_range<2>(problem, group),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(GemmCore::SgSize)]] {+ sycl_utils::nd_item_helper<GemmCore> helper(it);+ if constexpr (debug) {+ compute_tile(k, blocksize, B, B_scale, ldb, slm_b, A, lda, C, ldc, it,+ NeedFp8Lut ? &slm_lut : nullptr);+ } else {+ int m_tail = m - helper.sg_g_m();+ m_tail = m_tail > GemmCore::TileM ? GemmCore::TileM : m_tail;+ if (m_tail == GemmCore::TileM) {+ compute_tile(k, blocksize, B, B_scale, ldb, slm_b, A, lda, C, ldc, it,+ NeedFp8Lut ? &slm_lut : nullptr);+ } else {+ compute_tail(k, blocksize, m_tail, B, B_scale, ldb, slm_b, A, lda, C, ldc, it,+ NeedFp8Lut ? &slm_lut : nullptr);+ }+ }+ });+ });+ return ev;+ }++ template <typename ScaleT>+ static void compute_tile(int k, int blocksize, const uint8_t* B, const ScaleT* B_scale, int ldb,+ const sycl::local_accessor<BType, 1>& slm_b, const AType* A, int lda, CType* C, int ldc,+ sycl::nd_item<2>& it, const sycl::local_accessor<BType, 1>* slm_lut) {+ sycl_utils::nd_item_helper<GemmCore> helper(it);+ if constexpr (NeedFp8Lut) {+ PrologueB::init_slm_lut(*slm_lut, it);+ }+ ACCType tmp[GemmCore::TileM * GemmCore::TileN];+ for (size_t im = 0; im < GemmCore::TileM; im++)+ for (size_t in = 0; in < GemmCore::TileN; in++) tmp[im * GemmCore::TileN + in] = ACCType(0.f);+#pragma forceinline recursive+ for (int i = 0; i < k; i += GemmCore::TileK) {+ load_weight_tile(typename PrologueB::Param{B, B_scale, ldb}, slm_b, i, blocksize, helper, slm_lut);+ it.barrier(sycl::access::fence_space::local_space);+ GemmCore::compute(&A[helper.item_g_m() * k + i], k, slm_b, tmp, helper);+ it.barrier(sycl::access::fence_space::local_space);+ }+#pragma forceinline recursive+ Epilogue::store({C, ldc}, tmp, helper);+ }++ template <typename ScaleT>+ static void compute_tail(int k, int blocksize, int m_tail, const uint8_t* B, const ScaleT* B_scale, int ldb,+ const sycl::local_accessor<BType, 1>& slm_b, const AType* A, int lda, CType* C, int ldc,+ sycl::nd_item<2>& it, const sycl::local_accessor<BType, 1>* slm_lut) {+ sycl_utils::nd_item_helper<GemmCore> helper(it);+ if constexpr (NeedFp8Lut) {+ PrologueB::init_slm_lut(*slm_lut, it);+ }+ ACCType tmp[GemmCore::TileM * GemmCore::TileN];+ for (size_t im = 0; im < GemmCore::TileM; im++)+ for (size_t in = 0; in < GemmCore::TileN; in++) tmp[im * GemmCore::TileN + in] = ACCType(0.f);+#pragma forceinline recursive+ for (int i = 0; i < k; i += GemmCore::TileK) {+ load_weight_tile(typename PrologueB::Param{B, B_scale, ldb}, slm_b, i, blocksize, helper, slm_lut);+ it.barrier(sycl::access::fence_space::local_space);+ GemmCore::compute_mtail(&A[helper.item_g_m() * k + i], k, slm_b, tmp, helper, m_tail);+ it.barrier(sycl::access::fence_space::local_space);+ }+#pragma forceinline recursive+ Epilogue::store_tail({C, ldc}, tmp, helper, m_tail);+ }+};++struct SDPAParam {+ void *Q, *K, *V;+ void* O;+ void* mask;+ int seq, seq_kv, hn_q, hn_kv, h_dim, h_vdim;+ float scale;+};++struct HGemmCfg {+ static int constexpr sg_size = 16;+ static int constexpr sg_row = 8;+ static int constexpr sg_col = 4;+ static int constexpr TM = 16;+ static int constexpr TN = 16;+ static int constexpr TK = 16;+ static int constexpr SGM = 2;+ static int constexpr SGN = 4;+ static int constexpr UnrollK = 2;+ static int constexpr PrefetchDis = 3;+ static int constexpr G_NROW = 3;+ static int constexpr REPEAT = 1;+ static int constexpr GRF = 256;+ using DT = sycl::half;+ using DT_ACC = sycl::half;+ using DT_MASK = bool;+ using Param = sycl_gemm::GemmParam;+};++template <typename TP, typename CFG>+struct SDPA {+ using DT = CFG::DT;+ using DT1 = CFG::DT_ACC;+ using Param = CFG::Param;+ static int constexpr sg_size = CFG::sg_size;+ static int constexpr sg_row = CFG::sg_row;+ static int constexpr sg_col = CFG::sg_col;+ static int constexpr TM = CFG::TM;+ static int constexpr TN = CFG::TN;+ static int constexpr TK = CFG::TK;+ static int constexpr SGM = CFG::SGM;+ static int constexpr SGN = CFG::SGN;+ static int constexpr UnrollK = CFG::UnrollK;+ static int constexpr PrefetchDis = CFG::PrefetchDis;+ static int constexpr G_NROW = CFG::G_NROW;++ static int constexpr wg_size = sg_size * sg_row * sg_col;+ static int constexpr BM = TM * sg_row * SGM;+ static int constexpr BM_STRIDE = BM;+ static int constexpr BN = TN * sg_col * SGN;+ static int constexpr BN_STRIDE = BN;++ Param param;+ TP mProp;++ int g_ncnt, g_mcnt;+ size_t wg_repeat, aligned_wg;+ SDPA(TP Prop, CFG _cfg, Param _param) : mProp(Prop) {+ param = _param;++ g_ncnt = (param.n + BN - 1) / BN;+ g_mcnt = (param.m + BM - 1) / BM;+ wg_repeat = g_mcnt * g_ncnt;+ size_t m_tail = g_mcnt % G_NROW;+ size_t g_m_aligned = g_mcnt - m_tail;+ aligned_wg = g_m_aligned * g_ncnt;+ }++ inline sycl::nd_range<2> get_range() { return sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}); }++ void operator() [[sycl::reqd_sub_group_size(sg_size)]] (sycl::nd_item<2> it) const {+ auto sg = it.get_sub_group();+ int g_id = it.get_group(0);+ int g_n = g_id % g_ncnt;+ int g_m = g_id / g_ncnt;+ if (g_id < aligned_wg) {+ int g_m_ = g_id % G_NROW;+ g_id /= G_NROW;+ g_n = g_id % g_ncnt;+ g_id /= g_ncnt;+ g_m = g_id * G_NROW + g_m_;+ }+ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;+ int sgId = sg.get_local_id()[0];+ int g_idn = sggid_col * TN * SGN + g_n * BN;+ int g_idm = sggid_row * TM * SGM + g_m * BM;+ auto m = param.m;+ auto n = param.n;+ auto k = param.k;+ if (g_idn >= n || g_idm >= m) return;+ auto lda = param.lda;+ auto ldb = param.ldb;+ auto ldc = param.ldc;+ auto Awg_d = (DT*)param.A_d + (size_t)g_idm * lda;+ auto Bwg_d = (DT*)param.B_d + (size_t)g_idn * ldb;+ auto Cwg_d = (DT1*)param.C_d + (size_t)g_idm * ldc + g_idn;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ auto pA = syclex::annotated_ptr{+ Awg_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L3>>}};++ auto pB = syclex::annotated_ptr{+ Bwg_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};++ auto pC = syclex::annotated_ptr{+ Cwg_d, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_through, syclex::cache_level::L1, syclex::cache_level::L3>>}};+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::a, TM, TK, layout::row_major>+ sub_a[SGM];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::b, TK, TN, layout::col_major>+ sub_b[SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT1, use::accumulator, TM, TN>+ sub_c[SGM * SGN];++ DT1 zero = (DT1)0;+ if constexpr (!std::is_same_v<DT1, int32_t>) {+#pragma unroll(1)+ for (int in = 0; in < SGN; in++) {+ auto biasv = param.Bias ? *((DT1*)param.Bias + g_idn + in * TN + sgId) : zero;+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ joint_matrix_fill(sg, sub_c[im * SGN + in], biasv);+ }+ }+ } else {+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) joint_matrix_fill(sg, sub_c[im * SGN + in], DT1(0));+ }+ sycl_gemm::xmx::Prefetcher<CFG>::next(sg, 0, m, n, k, g_idm, g_idn, sggid_col, sg_col, sggid_row, sg_row, lda, ldb, Awg_d, Bwg_d);+ sycl::group_barrier(sg);++#pragma unroll(1)+ for (size_t ik = 0; ik < k; ik += TK * UnrollK) {+#pragma unroll+ for (size_t ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(sg, sub_b[in], pB, ldb, n - g_idn, k,+ in * TN, ik + TK * ikk);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(sg, sub_a[im], pA, lda, m - g_idm, k,+ im * TM, ik + TK * ikk);+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_mad(sg, sub_c[im * SGN + in], sub_a[im], sub_b[in], sub_c[im * SGN + in]);+ }+ }+ }+ sycl_gemm::xmx::Prefetcher<CFG>::next(sg, ik, m, n, k, g_idm, g_idn, sggid_col, sg_col, sggid_row, sg_row, lda, ldb, Awg_d,+ Bwg_d);+ }+ sycl::group_barrier(sg);+#pragma unroll+ for (int im = 0; im < SGM; im++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_store_checked(+ sg, sub_c[im * SGN + in], pC, ldc, layout::row_major, m - g_idm, n - g_idn, im * TM, in * TN);+ }+ }+ }++ auto get(syclex::properties_tag) const { return mProp; }+};+} // namespace sycl_wrapper+} // namespace bestla+#endifdiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla.cppnew file mode 100644index 000000000..dccf567e2--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla.cpp@@ -0,0 +1 @@+#include "../bestla.h"diff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_benchmark.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_benchmark.cppnew file mode 100644index 000000000..977bae7a1--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_benchmark.cpp@@ -0,0 +1,980 @@+#include <stdio.h>+#include "bestla_wrapper.h"+#include "bestla_gemm_kblock.h"+#include "bestla_ut.h"++namespace bestla {+using namespace utils;+namespace ut {+int constexpr TestMs = 500;+class Benchmark_Fp32Fp32 {+ public:+ Benchmark_Fp32Fp32() {+ UT_START();+ benchmark_all(1, 4096, 4096);+ benchmark_all(1024, 4096, 4096);+ }++ using AType = float;+ using BType = float;+ using CType = float;+ template <typename Core_T, typename LOG_T>+ void benchmark(int m, int n, int k, int batch, AType* A, BType* B, CType* C, float timems, int threads) {+ LOG_T log;+ using Parallel = parallel::gemm::SchedulerBase<Core_T>;+ using Launcher = wrapper::gemm::LauncherBase<Core_T, prologue_a::gemm::ActivationBase, prologue_b::gemm::WeightPack,+ epilogue::gemm::AccumulatorWriteBackFp32>;++ UT_Threading::set_threads(threads);+ auto corestr = gemm::CoreAttr::to_str(Core_T::ID);+ utils::timer<std::chrono::milliseconds> tm;+ auto tmpB = Launcher::PrologueB::createStorage(n, k);+ std::vector<storage::gemm::StoragePackedWeight> packBs(batch, 0);+ avector<int8_t> bufB(tmpB.mSize * batch);+ for (size_t i = 0; i < batch; i++) {+ packBs[i] = tmpB;+ packBs[i].assign(bufB.data() + i * tmpB.mSize);+ Launcher::PrologueB::packWeight(n, k, {B + i * n * k, n, &packBs[i]}, UT_Threading::get());+ }+ auto psize = (size_t)m * n * k * 2;+ tm.start();+ while (tm.stop() < timems) {+ for (size_t i = 0; i < batch; i++) {+ log.start();+ utils::GemmProblem gp(1, m, n, k);+ typename Launcher::Param args{gp, {A + i * m * k, k}, {0, 0, &packBs[i]}, {C + i * m * n, n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ log.stop();+ if (tm.stop() >= timems) {+ break;+ }+ }+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ printf("Threads %d %s %s Flops:%.3f PerCoreFlops:%.3f\n", threads, corestr, log.get_log_str(), flops,+ flops / threads);+ }++ void benchmark_all(int m, int n, int k) {+ auto memsize = gemm_memsize(m, n, k, BTLA_DTYPE::F32, BTLA_DTYPE::F32, BTLA_DTYPE::F32);+ auto batch = auto_batch(memsize);+ printf("%d %d %d %d %s %s %s\n", m, n, k, batch, bestla_dtype_str(BTLA_DTYPE::F32),+ bestla_dtype_str(BTLA_DTYPE::F32), bestla_dtype_str(BTLA_DTYPE::F32));+ avector<AType> A(size_t(m) * k * batch);+ avector<BType> B(size_t(k) * n * batch);+ avector<CType> C(size_t(m) * n * batch, 0);+ fill_buffer_randn(A.data(), m * k, -0.5f, 0.5f);+ fill_buffer_randn(B.data(), n * k, -0.5f, 0.5f);+ for (size_t i = 0; i < batch - 1; i++) {+ memcpy(A.data() + i * m * k, A.data(), m * k * sizeof(AType));+ memcpy(B.data() + i * n * k, B.data(), n * k * sizeof(BType));+ }+ using LOG = timer_statistics_logger<TestMs * 2>;+ float testtime = float(TestMs);+ GetCPUDevice();+ auto threads_cfg = UT_Threading::get_threads_config();+ for (auto threads : threads_cfg) {+ if (_cd->AVX512F()) {+ benchmark<gemm::SCoreRowNAvx512f<48, 8>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime, threads);+ }+ if (_cd->AVX2()) {+ benchmark<gemm::SCoreRowNAvx2<24, 4>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime, threads);+ }+ }+ }+};+#ifdef BTLA_UT_WRAPPER+static Benchmark_Fp32Fp32 sBenchmark_Fp32Fp32;+#endif++class Benchmark_U8S8S32 {+ public:+ Benchmark_U8S8S32() {+ UT_START();+ benchmark_all(1, 4096, 4096);+ benchmark_all(1024, 4096, 4096);+ }++ using AType = uint8_t;+ using BType = int8_t;+ using CType = int;+ template <typename Core_T, typename LOG_T>+ void benchmark(int m, int n, int k, int batch, AType* A, BType* B, CType* C, float timems, int threads) {+ LOG_T log;+ using Parallel = parallel::gemm::SchedulerBase<Core_T>;+ using Launcher = wrapper::gemm::LauncherBase<Core_T, prologue_a::gemm::ActivationBase, prologue_b::gemm::WeightPack,+ epilogue::gemm::AccumulatorWriteBackInt32>;+ UT_Threading::set_threads(threads);+ auto corestr = gemm::CoreAttr::to_str(Core_T::ID);+ utils::timer<std::chrono::milliseconds> tm;+ auto tmpB = Launcher::PrologueB::createStorage(n, k);+ std::vector<storage::gemm::StoragePackedWeight> packBs(batch, 0);+ avector<int8_t> bufB(tmpB.mSize * batch);+ for (size_t i = 0; i < batch; i++) {+ packBs[i] = tmpB;+ packBs[i].assign(bufB.data() + i * tmpB.mSize);+ Launcher::PrologueB::packWeight(n, k, {B + i * n * k, n, &packBs[i]}, UT_Threading::get());+ }+ auto psize = (size_t)m * n * k * 2;+ tm.start();+ while (tm.stop() < timems) {+ for (size_t i = 0; i < batch; i++) {+ log.start();+ utils::GemmProblem gp(1, m, n, k);+ typename Launcher::Param args{gp, {A + i * m * k, k}, {0, 0, &packBs[i]}, {C + i * m * n, n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ log.stop();+ if (tm.stop() >= timems) {+ break;+ }+ }+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ printf("Threads %d %s %s Flops:%.3f PerCoreFlops:%.3f\n", threads, corestr, log.get_log_str(), flops,+ flops / threads);+ }++ void benchmark_all(int m, int n, int k) {+ auto memsize = gemm_memsize(m, n, k, BTLA_DTYPE::U8, BTLA_DTYPE::S8, BTLA_DTYPE::S32);+ auto batch = auto_batch(memsize);+ printf("%d %d %d %d %s %s %s\n", m, n, k, batch, bestla_dtype_str(BTLA_DTYPE::U8), bestla_dtype_str(BTLA_DTYPE::S8),+ bestla_dtype_str(BTLA_DTYPE::S32));+ avector<AType> A(size_t(m) * k * batch);+ avector<BType> B(size_t(k) * n * batch);+ avector<CType> C(size_t(m) * n * batch);+ fill_buffer_randn(A.data(), m * k, AType(0), AType(255));+ fill_buffer_randn(B.data(), k * n, BType(-127), BType(127));+ for (size_t i = 0; i < batch - 1; i++) {+ memcpy(A.data() + i * m * k, A.data(), m * k * sizeof(AType));+ memcpy(B.data() + i * n * k, B.data(), n * k * sizeof(BType));+ }+ using LOG = timer_statistics_logger<TestMs * 2>;+ float testtime = float(TestMs);+ GetCPUDevice();+ auto threads_cfg = UT_Threading::get_threads_config();+ for (auto threads : threads_cfg) {+ if (_cd->AMX_INT8()) {+ benchmark<gemm::ICoreRowNAmxint8<32, 32>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime, threads);+ benchmark<gemm::ICoreRowNAmxint8<48, 16>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime, threads);+ benchmark<gemm::ICoreRowNAmxint8<64, 16>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime, threads);+ }+ if (_cd->AVX512_VNNI()) {+ benchmark<gemm::ICoreRowNAvx512vnni<48, 8>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime,+ threads);+ }+ if (_cd->AVX512BW()) {+ benchmark<gemm::ICoreRowNAvx512bw<48, 8>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime, threads);+ }+ if (_cd->AVX_VNNI()) {+ benchmark<gemm::ICoreRowNAvxvnni<24, 4>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime, threads);+ }+ if (_cd->AVX2()) {+ benchmark<gemm::ICoreRowNAvx2vnni<24, 4>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime, threads);+ }+ }+ }+};+#ifdef BTLA_UT_WRAPPER+static Benchmark_U8S8S32 sBenchmark_U8S8S32;+#endif++class Benchmark_S8S8S32 {+ public:+ Benchmark_S8S8S32() {+ UT_START();+ benchmark_all(1, 4096, 4096);+ benchmark_all(1024, 4096, 4096);+ }++ using AType = int8_t;+ using BType = int8_t;+ using CType = int;+ template <typename Core_T, typename LOG_T>+ void benchmark(int m, int n, int k, int batch, AType* A, BType* B, CType* C, float timems, int threads) {+ LOG_T log;+ using Parallel = parallel::gemm::SchedulerBase<Core_T>;+ using Launcher = wrapper::gemm::LauncherBase<Core_T, prologue_a::gemm::ActivationBase, prologue_b::gemm::WeightPack,+ epilogue::gemm::AccumulatorWriteBackInt32>;++ UT_Threading::set_threads(threads);+ auto corestr = gemm::CoreAttr::to_str(Core_T::ID);+ utils::timer<std::chrono::milliseconds> tm;+ auto tmpB = Launcher::PrologueB::createStorage(n, k);+ std::vector<storage::gemm::StoragePackedWeight> packBs(batch, 0);+ avector<int8_t> bufB(tmpB.mSize * batch);+ for (size_t i = 0; i < batch; i++) {+ packBs[i] = tmpB;+ packBs[i].assign(bufB.data() + i * tmpB.mSize);+ Launcher::PrologueB::packWeight(n, k, {B + i * n * k, n, &packBs[i]}, UT_Threading::get());+ }+ auto psize = (size_t)m * n * k * 2;+ tm.start();+ while (tm.stop() < timems) {+ for (size_t i = 0; i < batch; i++) {+ log.start();+ utils::GemmProblem gp(1, m, n, k);+ typename Launcher::Param args{gp, {A + i * m * k, k}, {0, 0, &packBs[i]}, {C + i * m * n, n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ log.stop();+ if (tm.stop() >= timems) {+ break;+ }+ }+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ printf("Threads %d %s %s Flops:%.3f PerCoreFlops:%.3f\n", threads, corestr, log.get_log_str(), flops,+ flops / threads);+ }++ void benchmark_all(int m, int n, int k) {+ auto memsize = gemm_memsize(m, n, k, BTLA_DTYPE::S8, BTLA_DTYPE::S8, BTLA_DTYPE::S32);+ auto batch = auto_batch(memsize);+ printf("%d %d %d %d %s %s %s\n", m, n, k, batch, bestla_dtype_str(BTLA_DTYPE::S8), bestla_dtype_str(BTLA_DTYPE::S8),+ bestla_dtype_str(BTLA_DTYPE::S32));+ avector<AType> A(size_t(m) * k * batch);+ avector<BType> B(size_t(k) * n * batch);+ avector<CType> C(size_t(m) * n * batch);+ fill_buffer_randn(A.data(), m * k, AType(0), AType(255));+ fill_buffer_randn(B.data(), k * n, BType(-127), BType(127));+ for (size_t i = 0; i < batch - 1; i++) {+ memcpy(A.data() + i * m * k, A.data(), m * k * sizeof(AType));+ memcpy(B.data() + i * n * k, B.data(), n * k * sizeof(AType));+ }+ using LOG = timer_statistics_logger<TestMs * 2>;+ float testtime = float(TestMs);+ GetCPUDevice();+ auto threads_cfg = UT_Threading::get_threads_config();+ for (auto threads : threads_cfg) {+ if (_cd->AVX2()) {+ benchmark<gemm::ICoreRowNAvx2vnniSS<24, 2>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime,+ threads);+ }+ if (_cd->AVX_VNNI()) {+ benchmark<gemm::ICoreRowNAvxvnniSS<24, 4>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime,+ threads);+ benchmark<gemm::ICoreRowNAvxvnniSS<24, 3>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime,+ threads);+ }+ if (_cd->AMX_INT8()) {+ benchmark<gemm::ICoreRowNAmxint8SS<32, 32>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime,+ threads);+ benchmark<gemm::ICoreRowNAmxint8SS<48, 16>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime,+ threads);+ benchmark<gemm::ICoreRowNAmxint8SS<64, 16>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime,+ threads);+ }+ }+ }+};+#ifdef BTLA_UT_WRAPPER+static Benchmark_S8S8S32 sBenchmark_S8S8S32;+#endif++class Benchmark_Bf16Bf16Fp32 {+ public:+ Benchmark_Bf16Bf16Fp32() {+ UT_START();+ benchmark_all(1, 4096, 4096);+ benchmark_all(1024, 4096, 4096);+ }++ using AType = utils::bf16;+ using BType = utils::bf16;+ using CType = float;+ template <typename Core_T, typename LOG_T>+ void benchmark(int m, int n, int k, int batch, AType* A, BType* B, CType* C, float timems, int threads) {+ LOG_T log;+ using Parallel = parallel::gemm::SchedulerBase<Core_T>;+ using Launcher = wrapper::gemm::LauncherBase<Core_T, prologue_a::gemm::ActivationBase, prologue_b::gemm::WeightPack,+ epilogue::gemm::AccumulatorWriteBackFp32>;++ UT_Threading::set_threads(threads);+ auto corestr = gemm::CoreAttr::to_str(Core_T::ID);+ utils::timer<std::chrono::milliseconds> tm;+ auto tmpB = Launcher::PrologueB::createStorage(n, k);+ std::vector<storage::gemm::StoragePackedWeight> packBs(batch, 0);+ avector<int8_t> bufB(tmpB.mSize * batch);+ for (size_t i = 0; i < batch; i++) {+ packBs[i] = tmpB;+ packBs[i].assign(bufB.data() + i * tmpB.mSize);+ Launcher::PrologueB::packWeight(n, k, {B + i * n * k, n, &packBs[i]}, UT_Threading::get());+ }+ auto psize = (size_t)m * n * k * 2;+ tm.start();+ while (tm.stop() < timems) {+ for (size_t i = 0; i < batch; i++) {+ log.start();+ utils::GemmProblem gp(1, m, n, k);+ typename Launcher::Param args{gp, {A + i * m * k, k}, {0, 0, &packBs[i]}, {C + i * m * n, n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ log.stop();+ if (tm.stop() >= timems) {+ break;+ }+ }+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ printf("Threads %d %s %s Flops:%.3f PerCoreFlops:%.3f\n", threads, corestr, log.get_log_str(), flops,+ flops / threads);+ }++ void benchmark_all(int m, int n, int k) {+ auto memsize = gemm_memsize(m, n, k, BTLA_DTYPE::BF16, BTLA_DTYPE::BF16, BTLA_DTYPE::F32);+ auto batch = auto_batch(memsize);+ printf("%d %d %d %d %s %s %s\n", m, n, k, batch, bestla_dtype_str(BTLA_DTYPE::BF16),+ bestla_dtype_str(BTLA_DTYPE::BF16), bestla_dtype_str(BTLA_DTYPE::F32));+ avector<AType> A(size_t(m) * k * batch);+ avector<BType> B(size_t(k) * n * batch);+ avector<CType> C(size_t(m) * n * batch);+ fill_buffer_randn(A.data(), k * m, AType(-0.5f), AType(0.5f));+ fill_buffer_randn(B.data(), k * n, BType(-0.5f), BType(0.5f));+ for (size_t i = 0; i < batch - 1; i++) {+ memcpy(A.data() + i * m * k, A.data(), m * k * sizeof(AType));+ memcpy(B.data() + i * n * k, B.data(), n * k * sizeof(BType));+ }+ using LOG = timer_statistics_logger<TestMs * 2>;+ float testtime = float(TestMs);+ GetCPUDevice();+ auto threads_cfg = UT_Threading::get_threads_config();+ for (auto threads : threads_cfg) {+ if (_cd->AMX_BF16()) {+ benchmark<gemm::HCoreRowNAmxbf16<32, 32>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime, threads);+ benchmark<gemm::HCoreRowNAmxbf16<48, 16>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime, threads);+ benchmark<gemm::HCoreRowNAmxbf16<64, 16>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime, threads);+ }+ }+ }+};+#ifdef BTLA_UT_WRAPPER+static Benchmark_Bf16Bf16Fp32 sBenchmark_Bf16Bf16Fp32;+#endif++class Benchmark_Fp16Fp16Fp32 {+ public:+ Benchmark_Fp16Fp16Fp32() {+ UT_START();+ benchmark_all(1, 4096, 4096);+ benchmark_all(1024, 4096, 4096);+ }++ using AType = utils::fp16;+ using BType = utils::fp16;+ using CType = float;+ template <typename Core_T, typename LOG_T>+ void benchmark(int m, int n, int k, int batch, AType* A, BType* B, CType* C, float timems, int threads) {+ LOG_T log;+ using Parallel = parallel::gemm::SchedulerBase<Core_T>;+ using Launcher = wrapper::gemm::LauncherBase<Core_T, prologue_a::gemm::ActivationBase, prologue_b::gemm::WeightPack,+ epilogue::gemm::AccumulatorWriteBackFp32>;++ UT_Threading::set_threads(threads);+ auto corestr = gemm::CoreAttr::to_str(Core_T::ID);+ utils::timer<std::chrono::milliseconds> tm;+ auto tmpB = Launcher::PrologueB::createStorage(n, k);+ std::vector<storage::gemm::StoragePackedWeight> packBs(batch, 0);+ avector<int8_t> bufB(tmpB.mSize * batch);+ for (size_t i = 0; i < batch; i++) {+ packBs[i] = tmpB;+ packBs[i].assign(bufB.data() + i * tmpB.mSize);+ Launcher::PrologueB::packWeight(n, k, {B + i * n * k, n, &packBs[i]}, UT_Threading::get());+ }+ auto psize = (size_t)m * n * k * 2;+ tm.start();+ while (tm.stop() < timems) {+ for (size_t i = 0; i < batch; i++) {+ log.start();+ GemmProblem gp(1, m, n, k);+ typename Launcher::Param args{gp, {A + i * m * k, k}, {0, 0, &packBs[i]}, {C + i * m * n, n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ log.stop();+ if (tm.stop() >= timems) {+ break;+ }+ }+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ printf("Threads %d %s %s Flops:%.3f PerCoreFlops:%.3f\n", threads, corestr, log.get_log_str(), flops,+ flops / threads);+ }++ void benchmark_all(int m, int n, int k) {+ auto memsize = gemm_memsize(m, n, k, BTLA_DTYPE::F16, BTLA_DTYPE::F16, BTLA_DTYPE::F32);+ auto batch = auto_batch(memsize);+ printf("%d %d %d %d %s %s %s\n", m, n, k, batch, bestla_dtype_str(BTLA_DTYPE::F16),+ bestla_dtype_str(BTLA_DTYPE::F16), bestla_dtype_str(BTLA_DTYPE::F32));+ avector<AType> A(size_t(m) * k * batch);+ avector<BType> B(size_t(k) * n * batch);+ avector<CType> C(size_t(m) * n * batch);+ fill_buffer_randn(A.data(), k * m, AType(-0.5f), AType(0.5f));+ fill_buffer_randn(B.data(), k * n, AType(-0.5f), AType(0.5f));+ for (size_t i = 0; i < batch - 1; i++) {+ memcpy(A.data() + i * m * k, A.data(), m * k * sizeof(AType));+ memcpy(B.data() + i * n * k, B.data(), n * k * sizeof(BType));+ }+ using LOG = timer_statistics_logger<TestMs * 2>;+ float testtime = float(TestMs);+ GetCPUDevice();+ auto threads_cfg = UT_Threading::get_threads_config();+ for (auto threads : threads_cfg) {+ if (_cd->AMX_FP16()) {+ benchmark<gemm::HCoreRowNAmxfp16<32, 32>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime, threads);+ benchmark<gemm::HCoreRowNAmxfp16<64, 16>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime, threads);+ }+ }+ }+};+#ifdef BTLA_UT_WRAPPER+static Benchmark_Fp16Fp16Fp32 sBenchmark_Fp16Fp16Fp32;+#endif++class Benchmark_Fp16Fp16Fp16 {+ public:+ Benchmark_Fp16Fp16Fp16() {+ UT_START();+ benchmark_all(1, 4096, 4096);+ benchmark_all(1024, 4096, 4096);+ }++ using AType = utils::fp16;+ using BType = utils::fp16;+ using CType = utils::fp16;+ template <typename Core_T, typename LOG_T>+ void benchmark(int m, int n, int k, int batch, AType* A, BType* B, CType* C, float timems, int threads) {+ LOG_T log;+ using Parallel = parallel::gemm::SchedulerBase<Core_T>;+ using Launcher = wrapper::gemm::LauncherBase<Core_T, prologue_a::gemm::ActivationBase, prologue_b::gemm::WeightPack,+ epilogue::gemm::AccumulatorWriteBackFp16>;++ UT_Threading::set_threads(threads);+ auto corestr = gemm::CoreAttr::to_str(Core_T::ID);+ utils::timer<std::chrono::milliseconds> tm;+ auto tmpB = Launcher::PrologueB::createStorage(n, k);+ std::vector<storage::gemm::StoragePackedWeight> packBs(batch, 0);+ avector<int8_t> bufB(tmpB.mSize * batch);+ for (size_t i = 0; i < batch; i++) {+ packBs[i] = tmpB;+ packBs[i].assign(bufB.data() + i * tmpB.mSize);+ Launcher::PrologueB::packWeight(n, k, {B + i * n * k, n, &packBs[i]}, UT_Threading::get());+ }+ auto psize = (size_t)m * n * k * 2;+ tm.start();+ while (tm.stop() < timems) {+ for (size_t i = 0; i < batch; i++) {+ log.start();+ GemmProblem gp(1, m, n, k);+ typename Launcher::Param args{gp, {A + i * m * k, k}, {0, 0, &packBs[i]}, {C + i * m * n, n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ log.stop();+ if (tm.stop() >= timems) {+ break;+ }+ }+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ printf("Threads %d %s %s Flops:%.3f PerCoreFlops:%.3f\n", threads, corestr, log.get_log_str(), flops,+ flops / threads);+ }++ void benchmark_all(int m, int n, int k) {+ auto memsize = gemm_memsize(m, n, k, BTLA_DTYPE::F16, BTLA_DTYPE::F16, BTLA_DTYPE::F16);+ auto batch = auto_batch(memsize);+ printf("%d %d %d %d %s %s %s\n", m, n, k, batch, bestla_dtype_str(BTLA_DTYPE::F16),+ bestla_dtype_str(BTLA_DTYPE::F16), bestla_dtype_str(BTLA_DTYPE::F16));+ avector<AType> A(size_t(m) * k * batch);+ avector<BType> B(size_t(k) * n * batch);+ avector<CType> C(size_t(m) * n * batch);+ fill_buffer_randn(A.data(), k * m, AType(-0.5f), AType(0.5f));+ fill_buffer_randn(B.data(), k * n, AType(-0.5f), AType(0.5f));+ for (size_t i = 0; i < batch - 1; i++) {+ memcpy(A.data() + i * m * k, A.data(), m * k * sizeof(AType));+ memcpy(B.data() + i * n * k, B.data(), n * k * sizeof(BType));+ }+ using LOG = timer_statistics_logger<TestMs * 2>;+ float testtime = float(TestMs);+ GetCPUDevice();+ auto threads_cfg = UT_Threading::get_threads_config();+ for (auto threads : threads_cfg) {+ if (_cd->AVX512_FP16()) {+ benchmark<sAVX512_FP16, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime, threads);+ benchmark<gemm::HCoreRowNAvx512fp16<64, 12>, LOG>(m, n, k, batch, A.data(), B.data(), C.data(), testtime,+ threads);+ }+ }+ }+};+#ifdef BTLA_UT_WRAPPER+static Benchmark_Fp16Fp16Fp16 sBenchmark_Fp16Fp16Fp16;+#endif++class UTWOQ_CompFp32 {+ public:+ UTWOQ_CompFp32() {+ UT_START();+ ut_s4_full();+ ut_new_dtype(BTLA_DTYPE::S1_CLIP);+ ut_new_dtype(BTLA_DTYPE::S2_CLIP);+ ut_new_dtype(BTLA_DTYPE::S3_CLIP);+ ut_new_dtype(BTLA_DTYPE::S5_CLIP);+ ut_new_dtype(BTLA_DTYPE::S6_CLIP);+ ut_new_dtype(BTLA_DTYPE::S7_CLIP);+ ut_new_dtype(BTLA_DTYPE::S8);+ ut_f4();+ }++ void ut_new_dtype(BTLA_DTYPE qtype) {+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, float>(1, 4096, 4096, qtype, true);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(1, 4096, 4096, qtype);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(1, 4096, 4096, qtype, true);+ }++ void ut_s4_full() {+ BTLA_DTYPE qtype = BTLA_DTYPE::S4_CLIP;+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, float>(1, 4096, 4096, qtype, true);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(1, 4096, 4096, qtype);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(1, 4096, 4096, qtype, true);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(1, 4096, 4096, qtype);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(1024, 4096, 4096, qtype);+ }++ void ut_f4() {+ benchmark_all<prologue_b::gemm::WeightKBlockNFloat, utils::bf16>(1, 4096, 4096, BTLA_DTYPE::F4_BNB);+ benchmark_all<prologue_b::gemm::WeightKBlockNFloat, utils::bf16>(1, 4096, 4096, BTLA_DTYPE::F4_E2M1);+ benchmark_all<prologue_b::gemm::WeightKBlockNFloat, utils::bf16>(1, 4096, 4096, BTLA_DTYPE::F4_NF4);+ benchmark_all<prologue_b::gemm::WeightKBlockNFloat, utils::bf16>(1024, 4096, 4096, BTLA_DTYPE::F4_BNB);+ }++ template <typename Core_T, typename LOG_T, template <class _T> class Wei, typename Scale_T>+ void benchmark(int m, int n, int k, int batch, int blocksize, float* A, float* B, float* C, float timems, int threads,+ BTLA_DTYPE qtype, bool isasym) {+ LOG_T log;+ using Parallel = parallel::gemm::SchedulerBase<Core_T>;+ using Launcher = wrapper::gemm::LauncherBase<Core_T, prologue_a::gemm::ActivationBase, Wei,+ epilogue::gemm::AccumulatorWriteBackFp32>;++ UT_Threading::set_threads(threads);+ auto corestr = gemm::CoreAttr::to_str(Core_T::ID);+ utils::timer<std::chrono::milliseconds> tm;+ using WType = typename Wei<Core_T>::StorageWeight;+ WType tmpB(0);+ if constexpr (std::is_same_v<Wei<Core_T>, prologue_b::gemm::WeightKBlockNInteger<Core_T>>) {+ tmpB = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, bestla_dtype<Scale_T>, isasym);++ } else if constexpr (std::is_same_v<Wei<Core_T>, prologue_b::gemm::WeightKBlockNFloat<Core_T>>) {+ tmpB = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, bestla_dtype<Scale_T>);+ }+ std::vector<WType> packBs(batch, 0);+ avector<int8_t> bufB(tmpB.mSize * batch);+ for (size_t i = 0; i < batch; i++) {+ packBs[i] = tmpB;+ packBs[i].assign(bufB.data() + i * tmpB.mSize);+ }+ Launcher::PrologueB::packWeight(n, k, B, n, &packBs[0], UT_Threading::get());+ for (size_t i = 1; i < batch; i++) {+ memcpy(packBs[i].template WPtr<void>(), packBs[0].template WPtr<void>(), packBs[0].template WSize<char>());+ memcpy(packBs[i].template SPtr<void>(), packBs[0].template SPtr<void>(), packBs[0].CSize() * sizeof(Scale_T));+ }+ auto psize = (size_t)m * n * k * 2;+ int blks = k / blocksize;+ size_t nbits = utils::bestla_dtype_bits(qtype);+ auto memsize = (size_t)(n * k * nbits / 8 + n * blks * sizeof(Scale_T)) + (m * k + m * n) * sizeof(float);+ tm.start();+ while (tm.stop() < timems) {+ for (int i = 0; i < batch; i++) {+ log.start();+ GemmProblem gp(1, m, n, k, blocksize);+ typename Launcher::Param args{gp, {A + i * m * k, k}, {&packBs[i]}, {C + i * m * n, n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ log.stop();+ if (tm.stop() >= timems) {+ break;+ }+ }+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ double band = double(memsize) / log.min_val / 1e6;+ int cores = std::min(threads, device::CpuDevice::getInstance()->getCores());+ printf("Threads %d Block %d %s %s Flops:%.3fG PerCoreFlops:%.3fG MemoryBandwidth:%.3fGB/s\n", threads, blocksize,+ corestr, log.get_log_str(), flops, flops / cores, band);+ }++ template <template <class _T> class Wei, typename Scale_T>+ void benchmark_all(int m, int n, int k, BTLA_DTYPE qtype, bool isasym = false) {+ auto memsize = gemm_memsize(m, n, k, BTLA_DTYPE::F32, qtype, BTLA_DTYPE::F32);+ int batch = auto_batch(memsize);+ printf("%d %d %d %d %s %s %s isasym:%d\n", m, n, k, batch, bestla_dtype_str(BTLA_DTYPE::F32),+ bestla_dtype_str(qtype), bestla_dtype_str(BTLA_DTYPE::F32), isasym);+ avector<float> A(size_t(m) * k * batch);+ avector<float> B(size_t(k) * n);+ avector<float> C(size_t(m) * n * batch);+ fill_buffer_randn(A.data(), k * m, (-0.5f), (0.5f));+ fill_buffer_randn(B.data(), k * n, (-0.5f), (0.5f));+ for (int i = 1; i < batch; i++) {+ memcpy(A.data() + i * m * k, A.data(), m * k * sizeof(float));+ }+ using LOG = timer_statistics_logger<TestMs * 2>;+ float testtime = float(TestMs);+ GetCPUDevice();+ auto threads_cfg = UT_Threading::get_threads_config();+ for (auto threads : threads_cfg) {+ for (auto blocksize : {32, 128}) {+ if (_cd->AVX512F()) {+ benchmark<gemm::SCoreRowNAvx512f<48, 8>, LOG, Wei, Scale_T>(m, n, k, batch, blocksize, A.data(), B.data(),+ C.data(), testtime, threads, qtype, isasym);+ }+ if (_cd->AVX2()) {+ benchmark<gemm::SCoreRowNAvx2<24, 4>, LOG, Wei, Scale_T>(m, n, k, batch, blocksize, A.data(), B.data(),+ C.data(), testtime, threads, qtype, isasym);+ }+ }+ }+ }+};+#ifdef BTLA_UT_PROLOGUE_B+static UTWOQ_CompFp32 sUTWOQ_CompFp32;+#endif++class UTWOQ_CompBf16 {+ public:+ UTWOQ_CompBf16() {+ UT_START();+ ut_s4();+ ut_s8();+ ut_f4();+ }++ void ut_s4() {+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(1, 4096, 4096, BTLA_DTYPE::S4_CLIP);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(1024, 4096, 4096, BTLA_DTYPE::S4_CLIP);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(2048, 4096, 4096, BTLA_DTYPE::S4_CLIP);+ }++ void ut_s8() {+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(1, 4096, 4096, BTLA_DTYPE::S8);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(1024, 4096, 4096, BTLA_DTYPE::S8);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(2048, 4096, 4096, BTLA_DTYPE::S8);+ }++ void ut_f4() {+ benchmark_all<prologue_b::gemm::WeightKBlockNFloat, utils::bf16>(1, 4096, 4096, BTLA_DTYPE::F4_BNB);+ benchmark_all<prologue_b::gemm::WeightKBlockNFloat, utils::bf16>(1024, 4096, 4096, BTLA_DTYPE::F4_BNB);+ benchmark_all<prologue_b::gemm::WeightKBlockNFloat, utils::bf16>(2048, 4096, 4096, BTLA_DTYPE::F4_BNB);+ }++ template <typename Core_T, typename LOG_T, template <class _T> class Wei, typename Scale_T>+ void benchmark(int m, int n, int k, int batch, int blocksize, float* A, float* B, float* C, float timems, int threads,+ BTLA_DTYPE qtype) {+ LOG_T log;+ using Parallel = parallel::gemm::SchedulerBase<Core_T>;+ using Launcher = wrapper::gemm::LauncherBase<Core_T, prologue_a::gemm::ActivationConverterFp32, Wei,+ epilogue::gemm::AccumulatorWriteBackFp32>;++ UT_Threading::set_threads(threads);+ auto corestr = gemm::CoreAttr::to_str(Core_T::ID);+ utils::timer<std::chrono::milliseconds> tm;+ using WType = typename Wei<Core_T>::StorageWeight;+ WType tmpB(0);+ if constexpr (std::is_same_v<Wei<Core_T>, prologue_b::gemm::WeightKBlockNInteger<Core_T>>) {+ tmpB = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, bestla_dtype<Scale_T>, false);++ } else if constexpr (std::is_same_v<Wei<Core_T>, prologue_b::gemm::WeightKBlockNFloat<Core_T>>) {+ tmpB = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, bestla_dtype<Scale_T>);+ }+ std::vector<WType> packBs(batch, 0);+ avector<int8_t> bufB(tmpB.mSize * batch);+ for (size_t i = 0; i < batch; i++) {+ packBs[i] = tmpB;+ packBs[i].assign(bufB.data() + i * tmpB.mSize);+ }+ Launcher::PrologueB::packWeight(n, k, B, n, &packBs[0], UT_Threading::get());+ for (size_t i = 1; i < batch; i++) {+ memcpy(packBs[i].template WPtr<void>(), packBs[0].template WPtr<void>(), packBs[0].template WSize<char>());+ memcpy(packBs[i].template SPtr<void>(), packBs[0].template SPtr<void>(), packBs[0].CSize() * sizeof(Scale_T));+ }+ auto psize = (size_t)m * n * k * 2;+ int blks = k / blocksize;+ size_t nbits = utils::bestla_dtype_bits(qtype);+ auto memsize = (size_t)(n * k * nbits / 8 + n * blks * sizeof(Scale_T)) + (m * k + m * n) * sizeof(float);+ tm.start();+ while (tm.stop() < timems) {+ for (int i = 0; i < batch; i++) {+ log.start();+ GemmProblem gp(1, m, n, k);+ typename Launcher::Param args{gp, {A + i * m * k, k}, {&packBs[i]}, {C + i * m * n, n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ log.stop();+ if (tm.stop() >= timems) {+ break;+ }+ }+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ double band = double(memsize) / log.min_val / 1e6;+ int cores = std::min(threads, device::CpuDevice::getInstance()->getCores());+ printf("Threads %d Block %d %s %s Flops:%.3fG PerCoreFlops:%.3fG MemoryBandwidth:%.3fGB/s\n", threads, blocksize,+ corestr, log.get_log_str(), flops, flops / cores, band);+ }++ template <template <class _T> class Wei, typename Scale_T>+ void benchmark_all(int m, int n, int k, BTLA_DTYPE qtype) {+ auto memsize = gemm_memsize(m, n, k, BTLA_DTYPE::F32, qtype, BTLA_DTYPE::F32);+ int batch = auto_batch(memsize);+ printf("%d %d %d %d %s %s %s\n", m, n, k, batch, bestla_dtype_str(BTLA_DTYPE::F32), bestla_dtype_str(qtype),+ bestla_dtype_str(BTLA_DTYPE::F32));+ avector<float> A(size_t(m) * k * batch);+ avector<float> B(size_t(k) * n);+ avector<float> C(size_t(m) * n * batch);+ fill_buffer_randn(A.data(), k * m, (-0.5f), (0.5f));+ fill_buffer_randn(B.data(), k * n, (-0.5f), (0.5f));+ for (int i = 1; i < batch; i++) {+ memcpy(A.data() + i * m * k, A.data(), m * k * sizeof(float));+ }+ using LOG = timer_statistics_logger<TestMs * 2>;+ float testtime = float(TestMs);+ GetCPUDevice();+ auto threads_cfg = UT_Threading::get_threads_config();+ for (auto threads : threads_cfg) {+ for (auto blocksize : {32, 128}) {+ if (_cd->AMX_BF16()) {+ benchmark<gemm::HCoreRowNAmxbf16<64, 16>, LOG, Wei, Scale_T>(m, n, k, batch, blocksize, A.data(), B.data(),+ C.data(), testtime, threads, qtype);+ }+ }+ }+ }+};+#ifdef BTLA_UT_PROLOGUE_B+static UTWOQ_CompBf16 sUTWOQ_CompBf16;+#endif++class UTWOQ_CompInt8 {+ public:+ UTWOQ_CompInt8() {+ UT_START();+ ut_s4_full();+ ut_new_dtype(BTLA_DTYPE::S1_CLIP);+ ut_new_dtype(BTLA_DTYPE::S2_CLIP);+ ut_new_dtype(BTLA_DTYPE::S3_CLIP);+ ut_new_dtype(BTLA_DTYPE::S5_CLIP);+ ut_new_dtype(BTLA_DTYPE::S6_CLIP);+ ut_new_dtype(BTLA_DTYPE::S7_CLIP);+ ut_new_dtype(BTLA_DTYPE::S8);+ }+ void ut_new_dtype(BTLA_DTYPE qtype) {+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, float>(1, 4096, 4096, qtype, true);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(1, 4096, 4096, qtype);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(1, 4096, 4096, qtype, true);+ }++ void ut_s4_full() {+ BTLA_DTYPE qtype = BTLA_DTYPE::S4_CLIP;+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, float>(1, 4096, 4096, qtype);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(1, 4096, 4096, qtype);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(1, 4096, 4096, qtype, true);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(4, 4096, 4096, qtype);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(8, 4096, 4096, qtype);+ benchmark_all<prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(1024, 4096, 4096, qtype);+ }++ using PcWriteBack = epilogue::gemm::PcKBlockCompInt8Epilogue<epilogue::gemm::AccumulatorWriteBackFp32>;++ template <typename Core_T, typename LOG_T, template <class _T> class Wei, typename Scale_T>+ void benchmark_pc(int m, int n, int k, int batch, int blocksize, float* A, float* B, float* C, float timems,+ int threads, BTLA_DTYPE qtype, bool isasym) {+ LOG_T log;+ using Parallel = parallel::gemm::SchedulerBase<Core_T>;+ using Launcher =+ wrapper::gemm::LauncherBase<Core_T, prologue_a::gemm::ActivationF32KBlockQuantize, Wei, PcWriteBack>;++ UT_Threading::set_threads(threads);+ auto corestr = gemm::CoreAttr::to_str(Core_T::ID);+ utils::timer<std::chrono::milliseconds> tm;+ using WType = typename Wei<Core_T>::StorageWeight;+ WType tmpB = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, bestla_dtype<Scale_T>, isasym);+ std::vector<WType> packBs(batch, 0);+ avector<int8_t> bufB(tmpB.mSize * batch);+ for (size_t i = 0; i < batch; i++) {+ packBs[i] = tmpB;+ packBs[i].assign(bufB.data() + i * tmpB.mSize);+ }+ Launcher::PrologueB::packWeight(n, k, B, n, &packBs[0], UT_Threading::get());+ for (size_t i = 1; i < batch; i++) {+ memcpy(packBs[i].template WPtr<void>(), packBs[0].template WPtr<void>(), packBs[0].template WSize<char>());+ memcpy(packBs[i].template SPtr<void>(), packBs[0].template SPtr<void>(), packBs[0].CSize() * sizeof(Scale_T));+ }+ auto quanA = Launcher::PrologueA::createStorage(m, k, blocksize, false);+ utils::avector<int8_t> bufferA(quanA.mSize);+ quanA.assign(bufferA.data());+ auto psize = (size_t)m * n * k * 2;+ int blks = k / blocksize;+ auto nbits = utils::bestla_dtype_bits(qtype);+ auto memsize = (size_t)(n * k * nbits / 8 + n * blks * sizeof(Scale_T)) + (m * k + m * n) * sizeof(float);+ if (isasym) {+ memsize += n * blks * sizeof(int8_t);+ }+ tm.start();+ while (tm.stop() < timems) {+ for (int i = 0; i < batch; i++) {+ log.start();+ GemmProblem gp(1, m, n, k, blocksize);+ typename Launcher::Param args{+ gp,+ {A + i * m * k, k, &quanA},+ {&packBs[i]},+ {{packBs[i].template SPtr<char>(), packBs[i].SDtype(), quanA.template SPtr<float>(),+ quanA.template ZPtr<uint8_t>(), packBs[i].template RPtr<int32_t>(), nullptr, nullptr, k},+ {C + i * m * n, n}}};+ parallel::GemmRunWithA<Parallel, Launcher>(args, UT_Threading::get());+ log.stop();+ if (tm.stop() >= timems) {+ break;+ }+ }+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ double band = double(memsize) / log.min_val / 1e6;+ int cores = std::min(threads, device::CpuDevice::getInstance()->getCores());+ printf("Threads %d Block %d %s %s Flops:%.3fG PerCoreFlops:%.3fG MemoryBandwidth:%.3fGB/s\n", threads, blocksize,+ corestr, log.get_log_str(), flops, flops / cores, band);+ }++ template <typename Core_T, typename LOG_T, template <class _T> class Wei, typename Scale_T>+ void benchmark(int m, int n, int k, int batch, int blocksize, float* A, float* B, float* C, float timems, int threads,+ BTLA_DTYPE qtype, bool isasym) {+ LOG_T log;+ using Parallel = parallel::gemm::SchedulerIntKBlock<Core_T>;+ using Launcher = wrapper::gemm::LauncherIntKBlock<Core_T, prologue_a::gemm::ActivationF32KBlockQuantize, Wei,+ epilogue::gemm::AccumulatorWriteBackFp32>;++ UT_Threading::set_threads(threads);+ auto corestr = gemm::CoreAttr::to_str(Core_T::ID);+ utils::timer<std::chrono::milliseconds> tm;+ using WType = typename Wei<Core_T>::StorageWeight;+ WType tmpB = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, bestla_dtype<Scale_T>, isasym);+ std::vector<WType> packBs(batch, 0);+ avector<int8_t> bufB(tmpB.mSize * batch);+ for (size_t i = 0; i < batch; i++) {+ packBs[i] = tmpB;+ packBs[i].assign(bufB.data() + i * tmpB.mSize);+ }+ Launcher::PrologueB::packWeight(n, k, B, n, &packBs[0], UT_Threading::get());+ for (size_t i = 1; i < batch; i++) {+ memcpy(packBs[i].template WPtr<void>(), packBs[0].template WPtr<void>(), packBs[0].template WSize<char>());+ memcpy(packBs[i].template SPtr<void>(), packBs[0].template SPtr<void>(), packBs[0].CSize() * sizeof(Scale_T));+ }+ auto quanA = Launcher::PrologueA::createStorage(m, k, blocksize, false);+ utils::avector<int8_t> bufferA(quanA.mSize);+ quanA.assign(bufferA.data());+ auto psize = (size_t)m * n * k * 2;+ int blks = k / blocksize;+ auto nbits = utils::bestla_dtype_bits(qtype);+ auto memsize = (size_t)(n * k * nbits / 8 + n * blks * sizeof(Scale_T)) + (m * k + m * n) * sizeof(float);+ if (isasym) {+ memsize += n * blks * sizeof(int8_t);+ }+ tm.start();+ while (tm.stop() < timems) {+ for (int i = 0; i < batch; i++) {+ log.start();+ GemmProblem gp(1, m, n, k, blocksize);+ typename Launcher::Param args{gp, {A + i * m * k, k, &quanA}, {&packBs[i]}, {C + i * m * n, n}};+ parallel::GemmRunWithA<Parallel, Launcher>(args, UT_Threading::get());+ log.stop();+ if (tm.stop() >= timems) {+ break;+ }+ }+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ double band = double(memsize) / log.min_val / 1e6;+ printf("Threads %d Block %d %s %s Flops:%.3fG PerCoreFlops:%.3fG MemoryBandwidth:%.3fGB/s\n", threads, blocksize,+ corestr, log.get_log_str(), flops, flops / threads, band);+ }++ template <template <class _T> class Wei, typename Scale_T>+ void benchmark_all(int m, int n, int k, BTLA_DTYPE qtype, bool isasym = false) {+ auto memsize = gemm_memsize(m, n, k, BTLA_DTYPE::F32, qtype, BTLA_DTYPE::F32);+ int batch = auto_batch(memsize);+ printf("%d %d %d %d %s %s %s Asym:%d Scale:%s\n", m, n, k, batch, bestla_dtype_str(BTLA_DTYPE::F32),+ bestla_dtype_str(qtype), bestla_dtype_str(BTLA_DTYPE::F32), isasym, bestla_dtype_str(bestla_dtype<Scale_T>));+ avector<float> A(size_t(m) * k * batch);+ avector<float> B(size_t(k) * n);+ avector<float> C(size_t(m) * n * batch);+ fill_buffer_randn(A.data(), k * m, (-0.5f), (0.5f));+ fill_buffer_randn(B.data(), k * n, (-0.5f), (0.5f));+ for (int i = 1; i < batch; i++) {+ memcpy(A.data() + i * m * k, A.data(), m * k * sizeof(float));+ }+ using LOG = timer_statistics_logger<TestMs / 2>;+ float testtime = float(TestMs);+ GetCPUDevice();+ auto threads_cfg = UT_Threading::get_threads_config();+ for (auto threads : threads_cfg) {+ for (auto blocksize : {k}) {+ if (_cd->AMX_INT8() && blocksize % 64 == 0) {+ benchmark_pc<gemm::ICoreRowNAmxint8SS<64, 16>, LOG, Wei, Scale_T>(+ m, n, k, batch, blocksize, A.data(), B.data(), C.data(), testtime, threads, qtype, isasym);+ }+ if (_cd->AVX512_VNNI()) {+ benchmark_pc<gemm::ICoreRowNAvx512vnni<48, 8>, LOG, Wei, Scale_T>(+ m, n, k, batch, blocksize, A.data(), B.data(), C.data(), testtime, threads, qtype, isasym);+ }+ if (_cd->AVX512BW()) {+ benchmark_pc<gemm::ICoreRowNAvx512bw<48, 8>, LOG, Wei, Scale_T>(m, n, k, batch, blocksize, A.data(), B.data(),+ C.data(), testtime, threads, qtype, isasym);+ }+ if (_cd->AVX_VNNI()) {+ benchmark_pc<gemm::ICoreRowNAvxvnni<24, 4>, LOG, Wei, Scale_T>(m, n, k, batch, blocksize, A.data(), B.data(),+ C.data(), testtime, threads, qtype, isasym);+ }+ if (_cd->AVX2()) {+ benchmark_pc<gemm::ICoreRowNAvx2vnni<24, 4>, LOG, Wei, Scale_T>(m, n, k, batch, blocksize, A.data(), B.data(),+ C.data(), testtime, threads, qtype, isasym);+ }+ }+ }++ for (auto threads : threads_cfg) {+ for (auto blocksize : {32, 128}) {+ if (_cd->AMX_INT8() && blocksize % 64 == 0) {+ benchmark<gemm::KblockGemmAmxint8<48, 16>, LOG, Wei, Scale_T>(m, n, k, batch, blocksize, A.data(), B.data(),+ C.data(), testtime, threads, qtype, isasym);+ }+ if (_cd->AVX512_VNNI()) {+ benchmark<gemm::KblockGemmAvx512vnni<48, 4>, LOG, Wei, Scale_T>(m, n, k, batch, blocksize, A.data(), B.data(),+ C.data(), testtime, threads, qtype, isasym);+ }+ if (_cd->AVX512BW()) {+ benchmark<gemm::KblockGemmAvx512bw<48, 4>, LOG, Wei, Scale_T>(m, n, k, batch, blocksize, A.data(), B.data(),+ C.data(), testtime, threads, qtype, isasym);+ }+ if (_cd->AVX_VNNI()) {+ benchmark<gemm::KblockGemmAvxvnni<24, 2>, LOG, Wei, Scale_T>(m, n, k, batch, blocksize, A.data(), B.data(),+ C.data(), testtime, threads, qtype, isasym);+ }+ if (_cd->AVX2()) {+ benchmark<gemm::KblockGemmAvx2vnni<24, 2>, LOG, Wei, Scale_T>(m, n, k, batch, blocksize, A.data(), B.data(),+ C.data(), testtime, threads, qtype, isasym);+ }+ }+ }+ }+};+#ifdef BTLA_UT_PROLOGUE_B+static UTWOQ_CompInt8 sUTWOQ_CompInt8;+#endif+} // namespace ut+} // namespace bestla+int main() {+ printf("BesTLA Benchmark done\n");+ return 0;+}diff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_epilogue.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_epilogue.cppnew file mode 100644index 000000000..b5357c985--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_epilogue.cpp@@ -0,0 +1,143 @@+#include "bestla_epilogue.h"+#include "bestla_ut.h"++#ifdef BTLA_UT_EPILOGUE+namespace bestla {+using namespace utils;+namespace ut {+class UT_AccumulatorWriteBack {+ public:+ UT_AccumulatorWriteBack() {+ UT_START();+ CheckISA(AVX2);+ fp32ut<BTLA_ISA::AVX2>(127, 255, 0, 0, 127, 255);+ fp32ut<BTLA_ISA::AVX2>(101, 237, 10, 63, 30, 33);+ fp32ut_with_custom_gelu<BTLA_ISA::AVX2>(15, 15, 0, 0, 15, 15);+ fp32ut_with_custom_swish<BTLA_ISA::AVX2>(15, 15, 0, 0, 15, 15);+ bf16ut<BTLA_ISA::AVX2>(127, 255, 0, 0, 127, 255);+ bf16ut<BTLA_ISA::AVX2>(101, 237, 10, 63, 30, 33);+ bf16fp32ut<BTLA_ISA::AVX2>(101, 237, 10, 63, 30, 33);+ bf16fp32ut<BTLA_ISA::AVX2>(127, 255, 0, 0, 127, 255);+ CheckISA(AVX512F);+ fp32ut<BTLA_ISA::AVX512F>(127, 255, 0, 0, 127, 255);+ fp32ut<BTLA_ISA::AVX512F>(101, 237, 10, 63, 30, 33);+ bf16ut<BTLA_ISA::AVX512F>(127, 255, 0, 0, 127, 255);+ bf16ut<BTLA_ISA::AVX512F>(101, 237, 10, 63, 30, 33);+ fp32ut_with_custom_gelu<BTLA_ISA::AVX512F>(15, 15, 0, 0, 15, 15);+ fp32ut_with_custom_swish<BTLA_ISA::AVX512F>(15, 15, 0, 0, 15, 15);++ bf16fp32ut<BTLA_ISA::AVX512F>(101, 237, 10, 63, 30, 33);+ bf16fp32ut<BTLA_ISA::AVX512F>(127, 255, 0, 0, 127, 255);+ }+ template <BTLA_ISA _RT_ISA_T>+ void bf16fp32ut(int _M, int _N, int _M_offset, int _N_offset, int _cpy_M, int _cpy_N) {+ printf("Test Case %s %d %d %d %d %d %d\n", __FUNCTION__, _M, _N, _M_offset, _N_offset, _cpy_M, _cpy_N);+ std::vector<bf16> src(_M * _N);+ for (int i = 0; i < _M * _N; i++) src[i].fromfloat(static_cast<float>(i));+ std::vector<float> dstref(_M * _N, 0), dstker(_M * _N, 0);++ epilogue::gemm::AccumulatorWriteBackBf16Fp32::template forward<BTLA_ISA::NoSIMD>(+ src.data(), _N, _M_offset, _N_offset, _cpy_M, _cpy_N, {dstref.data(), _N}, cache, CacheSize);+ epilogue::gemm::AccumulatorWriteBackBf16Fp32::template forward<_RT_ISA_T>(+ src.data(), _N, _M_offset, _N_offset, _cpy_M, _cpy_N, {dstker.data(), _N}, cache, CacheSize);+ ut::buffer_error(dstref.data(), dstker.data(), dstref.size());+ }+ template <BTLA_ISA _RT_ISA_T>+ void bf16ut(int _M, int _N, int _M_offset, int _N_offset, int _cpy_M, int _cpy_N) {+ printf("Test Case %s %d %d %d %d %d %d\n", __FUNCTION__, _M, _N, _M_offset, _N_offset, _cpy_M, _cpy_N);+ std::vector<float> src(_M * _N);+ for (int i = 0; i < _M * _N; i++) src[i] = float(i);+ std::vector<uint16_t> dstref(_M * _N, 0), dstker(_M * _N, 0);++ epilogue::gemm::AccumulatorWriteBackFp32Bf16::template forward<BTLA_ISA::NoSIMD>(+ src.data(), _N, _M_offset, _N_offset, _cpy_M, _cpy_N, {reinterpret_cast<bf16*>(dstref.data()), _N}, cache,+ CacheSize);+ epilogue::gemm::AccumulatorWriteBackFp32Bf16::template forward<_RT_ISA_T>(+ src.data(), _N, _M_offset, _N_offset, _cpy_M, _cpy_N, {reinterpret_cast<bf16*>(dstker.data()), _N}, cache,+ CacheSize);+ ut::buffer_error<uint16_t>(dstref.data(), dstker.data(), dstref.size());+ }+ template <BTLA_ISA _RT_ISA_T>+ void fp32ut(int _M, int _N, int _M_offset, int _N_offset, int _cpy_M, int _cpy_N) {+ printf("Test Case %s %d %d %d %d %d %d\n", __FUNCTION__, _M, _N, _M_offset, _N_offset, _cpy_M, _cpy_N);+ std::vector<float> src(_M * _N);+ for (int i = 0; i < _M * _N; i++) src[i] = float(i);+ std::vector<float> dstref(_M * _N, 0), dstker(_M * _N, 0);++ epilogue::gemm::AccumulatorWriteBackFp32::template forward<BTLA_ISA::NoSIMD>(+ src.data(), _N, _M_offset, _N_offset, _cpy_M, _cpy_N, {dstref.data(), _N}, cache, CacheSize);+ epilogue::gemm::AccumulatorWriteBackFp32::template forward<_RT_ISA_T>(+ src.data(), _N, _M_offset, _N_offset, _cpy_M, _cpy_N, {dstker.data(), _N}, cache, CacheSize);+ ut::buffer_error<float>(dstref.data(), dstker.data(), dstref.size());+ }+ template <BTLA_ISA _RT_ISA_T>+ void fp32ut_with_custom_gelu(int _M, int _N, int _M_offset, int _N_offset, int _cpy_M, int _cpy_N) {+ printf("Test Case %s %d %d %d %d %d %d\n", __FUNCTION__, _N, _M, _M_offset, _N_offset, _cpy_M, _cpy_N);+ std::vector<float> src(_M * _N);+ for (int i = 0; i < _M * _N; i++) src[i] = float(i);+ std::vector<float> dstref(_M * _N, 0), dstker(_M * _N, 0);+ epilogue::gemm::AccumulatorWriteBackWithGeluFp32 ::template forward<_RT_ISA_T>(+ src.data(), _N, _M_offset, _N_offset, _cpy_M, _cpy_N, {dstker.data(), _N}, cache, CacheSize);+ auto gelu = [&](float x) {+ return 0.5f * x * (1.f + tanhf(0.7978845834732056f * (x + 0.044714998453855515f * x * x * x)));+ };+ for (int i = 0; i < _M * _N; i++) src[i] = gelu(src[i]);+ ut::buffer_error<float>(src.data(), dstker.data(), dstker.size(), 0.000001f);+ }++ template <BTLA_ISA _RT_ISA_T>+ void fp32ut_with_custom_swish(int _M, int _N, int _M_offset, int _N_offset, int _cpy_M, int _cpy_N) {+ printf("Test Case %s %d %d %d %d %d %d\n", __FUNCTION__, _N, _M, _M_offset, _N_offset, _cpy_M, _cpy_N);+ std::vector<float> src(_M * _N);+ for (int i = 0; i < _M * _N; i++) src[i] = float(i);+ std::vector<float> dstref(_M * _N, 0), dstker(_M * _N, 0);+ float elt_const_v[] = {-1.0f};+ epilogue::gemm::AccumulatorWriteBackWithSwishFp32::template forward<_RT_ISA_T>(+ src.data(), _N, _M_offset, _N_offset, _cpy_M, _cpy_N, {dstker.data(), _N, elt_const_v}, cache, CacheSize);+ auto swish = [&](float x) { return x / (1 + exp(-x)); };+ for (int i = 0; i < _M * _N; i++) src[i] = swish(src[i]);+ ut::buffer_error<float>(src.data(), dstker.data(), dstker.size(), 0.2f); // swish use low lprecision exp+ }+};++class UT_AlphaBetaProcessFp32 {+ public:+ UT_AlphaBetaProcessFp32() {+ UT_START();+ // ut(45, 8, padto(45, 48), 0, 45, 1.f, 1.f);+ // ut(45, 8, padto(45, 48), 0, 45, 1.f, 0.f);+ // ut(45, 8, padto(45, 48), 45, 45, 1.f, 0.f);+ CheckISA(AVX512F);+ ut(3, 8, padto(3, 48), 3, 3, 1.f, 1.f);+ ut(3, 8, padto(3, 48), 0, 3, 1.f, 1.f);+ ut(3, 8, padto(3, 48), 3, 3, 1.f, 0.f);+ }++ void ut(int _N, int _M, int _srcstep, int _src1step, int _dststep, float alpha, float beta) {+ printf("Test Case %d %d %d %d %d %f %f\n", _N, _M, _srcstep, _src1step, _dststep, alpha, beta);+ std::vector<float> src(_M * _srcstep), src1, dst(_M * _dststep, 0.f), dstref(_M * _dststep, 0.f);+ if (_src1step == 0) {+ src1.resize(_N, 10.f);+ } else {+ src1.resize(_M * _src1step, 10.f);+ }+ for (int i = 0; i < src.size(); i++) {+ src[i] = float(i);+ }+ using Epi = epilogue::gemm::AlphaBetaProcessFp32;+ Epi::template forward<BTLA_ISA::NoSIMD>(src.data(), _srcstep, 0, 0, _M, _N,+ {dstref.data(), src1.data(), _dststep, _src1step, alpha, beta}, cache,+ CacheSize);+ Epi::template forward<BTLA_ISA::AVX512F>(src.data(), _srcstep, 0, 0, _M, _N,+ {dst.data(), src1.data(), _dststep, _src1step, alpha, beta}, cache,+ CacheSize);+ ut::buffer_error<float>(dstref.data(), dst.data(), dstref.size());+ }+};+#ifdef BTLA_UT_EPILOGUE+static UT_AccumulatorWriteBack sUT_AccumulatorWriteBack;+static UT_AlphaBetaProcessFp32 sUT_AlphaBetaProcessFp32;+#endif+} // namespace ut+} // namespace bestla+#endifdiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_gemm.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_gemm.cppnew file mode 100644index 000000000..be81952fc--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_gemm.cpp@@ -0,0 +1,837 @@+#include "bestla_gemm.h"+#include "bestla_gemm_kblock.h"+#include "bestla_utils.h"+#include "bestla_ut.h"++namespace bestla {+using namespace utils;++template <int NTILE>+void ref_bf16(utils::bf16* matA, utils::bf16* matB, float* matC, int _m, int _n, int _k, int _astride, int _bstride,+ int _cstride, int kpos) {+ int lda = _astride / sizeof(utils::bf16);+ int ldb = _bstride / sizeof(utils::bf16);+ int ldc = _cstride / sizeof(float);+ int constexpr KPack = 4 / sizeof(utils::bf16);+ for (int i = 0; i < _m; i++) {+ for (int j = 0; j < _n; j += NTILE) {+ for (int ij = 0; ij < NTILE; ij++) {+ if (j + ij >= _n) {+ continue;+ }+ float tmp = 0;+ for (int k = 0; k < _k; k += KPack) {+ for (int ik = 0; ik < KPack; ik++) {+ if (k + ik >= _k) {+ continue;+ }+ auto tmpA = utils::cast<utils::bf16, float>(utils::bf16{matA[i * lda + k + ik]});+ auto tmpB = utils::cast<utils::bf16, float>(utils::bf16{matB[k * NTILE + ij * KPack + ik + j * ldb]});+ tmp += tmpA * tmpB;+ }+ }+ matC[i * ldc + j + ij] = tmp;+ }+ }+ }+}++template <int NTILE>+void ref_fp16(utils::fp16* matA, utils::fp16* matB, utils::fp16* matC, int _m, int _n, int _k, int _astride,+ int _bstride, int _cstride, int kpos) {+ int lda = _astride / sizeof(utils::fp16);+ int ldb = _bstride / sizeof(utils::fp16);+ int ldc = _cstride / sizeof(utils::fp16);+ int constexpr KPack = 1;+ for (int i = 0; i < _m; i++) {+ for (int j = 0; j < _n; j += NTILE) {+ for (int ij = 0; ij < NTILE; ij++) {+ if (j + ij >= _n) {+ continue;+ }+ utils::fp16 tmp = utils::fp16(0.f);+ for (int k = 0; k < _k; k += KPack) {+ for (int ik = 0; ik < KPack; ik++) {+ if (k + ik >= _k) {+ continue;+ }+ auto tmpA = utils::cast<utils::fp16, float>(matA[i * lda + k + ik]);+ auto tmpB = utils::cast<utils::fp16, float>(matB[k * NTILE + ij * KPack + ik + j * ldb]);+ tmp = float(tmp) + tmpA * tmpB;+ }+ }+ matC[i * ldc + j + ij] = tmp;+ }+ }+ }+}++template <int NTILE>+void ref_fp16_fp32(utils::fp16* matA, utils::fp16* matB, float* matC, int _m, int _n, int _k, int _astride,+ int _bstride, int _cstride, int kpos) {+ int lda = _astride / sizeof(utils::fp16);+ int ldb = _bstride / sizeof(utils::fp16);+ int ldc = _cstride / sizeof(float);+ int constexpr KPack = 4 / sizeof(utils::fp16);+ for (int i = 0; i < _m; i++) {+ for (int j = 0; j < _n; j += NTILE) {+ for (int ij = 0; ij < NTILE; ij++) {+ if (j + ij >= _n) {+ continue;+ }+ float tmp = 0;+ for (int k = 0; k < _k; k += KPack) {+ for (int ik = 0; ik < KPack; ik++) {+ if (k + ik >= _k) {+ continue;+ }+ auto tmpA = utils::cast<utils::fp16, float>(utils::fp16{matA[i * lda + k + ik]});+ auto tmpB = utils::cast<utils::fp16, float>(utils::fp16{matB[k * NTILE + ij * KPack + ik + j * ldb]});+ tmp += tmpA * tmpB;+ }+ }+ matC[i * ldc + j + ij] = tmp;+ }+ }+ }+}++template <int NTILE>+void ref_fp32(float* matA, float* matB, float* matC, int _m, int _n, int _k, int _astride, int _bstride, int _cstride,+ int kpos) {+ int lda = _astride / sizeof(float);+ int ldb = _bstride / sizeof(float);+ int ldc = _cstride / sizeof(float);+ int constexpr KPack = 4 / sizeof(float);+ for (int i = 0; i < _m; i++) {+ for (int j = 0; j < _n; j += NTILE) {+ for (int ij = 0; ij < NTILE; ij++) {+ if (j + ij >= _n) {+ continue;+ }+ float tmp = 0;+ for (int k = 0; k < _k; k += KPack) {+ for (int ik = 0; ik < KPack; ik++) {+ if (k + ik >= _k) {+ continue;+ }+ auto tmpA = matA[i * lda + k + ik];+ auto tmpB = matB[k * NTILE + ij * KPack + ik + j * ldb];+ tmp += tmpA * tmpB;+ }+ }+ matC[i * ldc + j + ij] = tmp;+ }+ }+ }+}++template <int NTILE, class T_A_, class T_B_>+void ref_int8(T_A_* matA, T_B_* matB, int32_t* matC, int _m, int _n, int _k, int _astride, int _bstride, int _cstride,+ int kpos) {+ int lda = _astride / sizeof(T_A_);+ int ldb = _bstride / sizeof(T_B_);+ int ldc = _cstride / sizeof(int32_t);+ int constexpr KPack = 4 / sizeof(T_B_);+ for (int i = 0; i < _m; i++) {+ for (int j = 0; j < _n; j += NTILE) {+ for (int ij = 0; ij < NTILE; ij++) {+ if (j + ij >= _n) {+ continue;+ }+ int32_t tmp = 0;+ for (int k = 0; k < _k; k += KPack) {+ for (int ik = 0; ik < KPack; ik++) {+ if (k + ik >= _k) {+ continue;+ }+ auto tmpA = utils::cast<T_A_, int32_t>(matA[i * lda + k + ik]);+ auto tmpB = utils::cast<T_B_, int32_t>(matB[k * NTILE + ij * KPack + ik + j * ldb]);+ tmp += tmpA * tmpB;+ }+ }+ matC[i * ldc + j + ij] = tmp;+ }+ }+ }+}++template <int NTILE, typename T_SB_>+void ref_kblock_int8(uint8_t* matA, int8_t* matB, float* matC, uint8_t* zpA, float* scaleA, int _ldsa, T_SB_* scaleB,+ int* reduceB, int _ldsb, int _m, int _n, int _k, int _kblock, int _astride, int _bstride,+ int _cstride, int kpos) {+ int lda = _astride / sizeof(matA[0]);+ int ldb = _bstride / sizeof(matB[0]);+ int ldc = _cstride / sizeof(matC[0]);+ for (int i = 0; i < _m; i++) {+ for (int j = 0; j < _n; j += NTILE) {+ for (int ij = 0; ij < NTILE; ij++) {+ if (j + ij >= _n) {+ break;+ }+ float tmpf = 0.f;+ for (int k = 0; k < _k; k += _kblock) {+ int tmp = 0;+ int zpval = int(zpA[i * _ldsa + k / _kblock]);+ for (int ik = 0; ik < _kblock; ik += 4) {+ if (k + ik >= _k) {+ break;+ }+ for (int ikk = 0; ikk < 4; ikk++) {+ tmp += (int(matA[i * lda + k + ik + ikk])) * int(matB[(k + ik) * NTILE + ij * 4 + ikk + j * ldb]);+ }+ }+ tmp -= zpval * reduceB[j + ij + k / _kblock * _ldsb];+ tmpf += tmp * scaleA[i * _ldsa + k / _kblock] * float(scaleB[j + ij + k / _kblock * _ldsb]);+ }+ matC[i * ldc + j + ij] = tmpf;+ }+ }+ }+}++template <int NTILE, typename T_SB_>+void ref_kblock_int8_ss(int8_t* matA, int8_t* matB, float* matC, float* scaleA, int _ldsa, T_SB_* scaleB, int _ldsb,+ int _m, int _n, int _k, int _kblock, int _astride, int _bstride, int _cstride, int kpos) {+ int lda = _astride / sizeof(matA[0]);+ int ldb = _bstride / sizeof(matB[0]);+ int ldc = _cstride / sizeof(matC[0]);+ for (int i = 0; i < _m; i++) {+ for (int j = 0; j < _n; j += NTILE) {+ for (int ij = 0; ij < NTILE; ij++) {+ if (j + ij >= _n) {+ break;+ }+ float tmpf = 0.f;+ for (int k = 0; k < _k; k += _kblock) {+ int tmp = 0;+ for (int ik = 0; ik < _kblock; ik += 4) {+ if (k + ik >= _k) {+ break;+ }+ for (int ikk = 0; ikk < 4; ikk++) {+ tmp += (int(matA[i * lda + k + ik + ikk])) * int(matB[(k + ik) * NTILE + ij * 4 + ikk + j * ldb]);+ }+ }+ tmpf += tmp * scaleA[i * _ldsa + k / _kblock] * float(scaleB[j + ij + k / _kblock * _ldsb]);+ }+ matC[i * ldc + j + ij] = tmpf;+ }+ }+ }+}+namespace ut {+class UT_GEMM_AVX2 {+ public:+ UT_GEMM_AVX2() {+ UT_START();+ CheckISA(AVX2);+ ut_24(4, 24, 3);+ ut_24(4, 48, 3);++ ut_48(1, 48, 3);+ ut_48(1, 144, 3);+ }++ void ut_24(int m, int n, int k) {+ printf("Test Case: %d %d %d\n", m, n, k);+ using Core = gemm::SCoreRowNAvx2<24>;+ static Core gemm;+ if (n % Core::NTILE != 0) {+ return;+ }+ if (k % Core::Code::KTILE != 0) {+ return;+ }+ if (m > Core::Code::MTILE) {+ return;+ }+ avector<float> A(m * k), B(k * n), C(m * n, 0.f), RefC(m * n, 0.f);+ fill_buffer_randn(A.data(), A.size(), -0.5f, 0.5f);+ fill_buffer_randn(B.data(), B.size(), -0.5f, 0.5f);+ ref_fp32<Core::Code::NTILE>(A.data(), B.data(), RefC.data(), m, n, k, k * 4, k * 4, n * 4, 0);++ gemm.forward(A.data(), B.data(), C.data(), m, n, k, k * 4, k * 4, n * 4, 0, cache, CacheSize);+ ut::buffer_error(RefC.data(), C.data(), RefC.size(), 0.001f);+ }++ void ut_48(int m, int n, int k) {+ printf("Test Case: %d %d %d\n", m, n, k);+ using Core = gemm::SCoreRowNAvx2<48>;+ static Core gemm;+ if (n % Core::Code::NTILE != 0) {+ return;+ }+ if (k % Core::Code::KTILE != 0) {+ return;+ }+ if (m > Core::Code::MTILE) {+ return;+ }+ avector<float> A(m * k), B(k * n), C(m * n, 0.f), RefC(m * n, 0.f);+ fill_buffer_randn(A.data(), A.size(), -0.5f, 0.5f);+ fill_buffer_randn(B.data(), B.size(), -0.5f, 0.5f);+ ref_fp32<Core::Code::NTILE>(A.data(), B.data(), RefC.data(), m, n, k, k * 4, k * 4, n * 4, 0);++ gemm.forward(A.data(), B.data(), C.data(), m, n, k, k * 4, k * 4, n * 4, 0, cache, CacheSize);+ ut::buffer_error(RefC.data(), C.data(), RefC.size(), 0.001f);+ }+};+#ifdef BTLA_UT_GEMM+static UT_GEMM_AVX2 sUT_GEMM_AVX2;+#endif++class UT_GEMM_AVX512F {+ public:+ UT_GEMM_AVX512F() {+ UT_START();+ CheckISA(AVX512F);+ ut_32(4, 32, 3);+ ut_32(4, 64, 3);++ ut_48(1, 48, 3);+ ut_48(1, 144, 3);+ }++ void ut_32(int m, int n, int k) {+ printf("Test Case: %d %d %d\n", m, n, k);+ using Core = gemm::SCoreRowNAvx512f<32>;+ static Core gemm;+ if (n % Core::Code::NTILE != 0) {+ return;+ }+ if (k % Core::Code::KTILE != 0) {+ return;+ }+ avector<float> A(m * k), B(k * n), C(m * n, 0.f), RefC(m * n, 0.f);+ fill_buffer_randn(A.data(), A.size(), -0.5f, 0.5f);+ fill_buffer_randn(B.data(), B.size(), -0.5f, 0.5f);+ ref_fp32<Core::Code::NTILE>(A.data(), B.data(), RefC.data(), m, n, k, k * 4, k * 4, n * 4, 0);++ gemm.forward(A.data(), B.data(), C.data(), m, n, k, k * 4, k * 4, n * 4, 0, cache, CacheSize);+ ut::buffer_error(RefC.data(), C.data(), RefC.size(), 0.001f);+ }++ void ut_48(int m, int n, int k) {+ printf("Test Case: %d %d %d\n", m, n, k);+ using Core = gemm::SCoreRowNAvx512f<48>;+ static Core gemm;+ if (n % Core::Code::NTILE != 0) {+ return;+ }+ if (k % Core::Code::KTILE != 0) {+ return;+ }+ avector<float> A(m * k), B(k * n), C(m * n, 0.f), RefC(m * n, 0.f);+ fill_buffer_randn(A.data(), A.size(), -0.5f, 0.5f);+ fill_buffer_randn(B.data(), B.size(), -0.5f, 0.5f);+ ref_fp32<Core::Code::NTILE>(A.data(), B.data(), RefC.data(), m, n, k, k * 4, k * 4, n * 4, 0);++ gemm.forward(A.data(), B.data(), C.data(), m, n, k, k * 4, k * 4, n * 4, 0, cache, CacheSize);+ ut::buffer_error(RefC.data(), C.data(), RefC.size(), 0.001f);+ }+};+#ifdef BTLA_UT_GEMM+static UT_GEMM_AVX512F sUT_GEMM_AVX512F;+#endif++class UT_GEMM_AVX512VNNI {+ public:+ UT_GEMM_AVX512VNNI() {+ UT_START();+ CheckISA(AVX512_VNNI);+ ut<32, 0>(4, 64, 12);+ ut<32, 12>(4, 64, 12);++ ut<48, 0>(4, 96, 12);+ ut<48, 8>(4, 96, 12);+ }++ template <int NTile, int MTile>+ void ut(int m, int n, int k) {+ printf("Test Case: %d %d %d\n", m, n, k);+ using Core = gemm::ICoreRowNAvx512vnni<NTile, MTile>;+ static Core gemm;+ if (n % Core::Code::NTILE != 0) {+ return;+ }+ if (k % Core::Code::KTILE != 0) {+ return;+ }+ avector<uint8_t> A(m * k);+ avector<int8_t> B(k * n);+ avector<int> C(m * n, 0), RefC(m * n, 0);+ fill_buffer_randn(A.data(), A.size(), (uint8_t)0, (uint8_t)255);+ fill_buffer_randn(B.data(), B.size(), (int8_t)-127, (int8_t)127);+ ref_int8<Core::Code::NTILE>(A.data(), B.data(), RefC.data(), m, n, k, k * sizeof(A[0]), k * sizeof(B[0]),+ n * sizeof(C[0]), 0);++ gemm.forward(A.data(), B.data(), C.data(), m, n, k, k * sizeof(A[0]), k * sizeof(B[0]), n * sizeof(C[0]), 0, cache,+ CacheSize);+ ut::buffer_error(RefC.data(), C.data(), RefC.size(), 1);+ }+};+#ifdef BTLA_UT_GEMM+static UT_GEMM_AVX512VNNI sUT_GEMM_AVX512VNNI;+#endif++class UT_GEMM_AVX512BW {+ public:+ UT_GEMM_AVX512BW() {+ UT_START();+ CheckISA(AVX512BW);+ ut<32, 0>(4, 64, 12);+ ut<32, 12>(4, 64, 12);++ ut<48, 0>(4, 96, 12);+ ut<48, 8>(4, 96, 12);+ }++ template <int NTile, int MTile>+ void ut(int m, int n, int k) {+ printf("Test Case: %d %d %d\n", m, n, k);+ using Core = gemm::ICoreRowNAvx512bw<NTile, MTile>;+ static Core gemm;+ if (n % Core::Code::NTILE != 0) {+ return;+ }+ if (k % Core::Code::KTILE != 0) {+ return;+ }+ avector<uint8_t> A(m * k);+ avector<int8_t> B(k * n);+ avector<int> C(m * n, 0), RefC(m * n, 0);+ fill_buffer_randn(A.data(), A.size(), (uint8_t)0, (uint8_t)128);+ fill_buffer_randn(B.data(), B.size(), (int8_t)-127, (int8_t)127);+ ref_int8<Core::Code::NTILE>(A.data(), B.data(), RefC.data(), m, n, k, k * sizeof(A[0]), k * sizeof(B[0]),+ n * sizeof(C[0]), 0);++ gemm.forward(A.data(), B.data(), C.data(), m, n, k, k * sizeof(A[0]), k * sizeof(B[0]), n * sizeof(C[0]), 0, cache,+ CacheSize);+ ut::buffer_error(RefC.data(), C.data(), RefC.size(), 1);+ }+};+#ifdef BTLA_UT_GEMM+static UT_GEMM_AVX512BW sUT_GEMM_AVX512BW;+#endif++class UT_GEMM_KBLOCK {+ public:+ UT_GEMM_KBLOCK() {+ UT_START();+ std::vector<std::array<int, 3>> cases = {{96, 36, 36}, {144, 128, 32}, {144, 128, 128}, {144, 256, 128}};+ GetCPUDevice();+ if (_cd->AVX2()) {+ using Core0 = gemm::KblockGemmAvx2vnni<24, 2>;+ using Core1 = gemm::KblockGemmAvx2vnni<24, 1>;+ using Core2 = gemm::KblockGemmAvx2vnniSS<24, 1>;+ for (size_t i = 0; i < cases.size(); i++) {+ ut<Core0>(cases[i][0], cases[i][1], cases[i][2]);+ ut<Core1>(cases[i][0], cases[i][1], cases[i][2]);+ ut<Core2>(cases[i][0], cases[i][1], cases[i][2]);+ }+ }+ if (_cd->AVX_VNNI()) {+ using Core0 = gemm::KblockGemmAvxvnni<24, 2>;+ using Core1 = gemm::KblockGemmAvxvnni<24, 1>;+ using Core2 = gemm::KblockGemmAvxvnniSS<24, 1>;+ for (size_t i = 0; i < cases.size(); i++) {+ ut<Core0>(cases[i][0], cases[i][1], cases[i][2]);+ ut<Core1>(cases[i][0], cases[i][1], cases[i][2]);+ ut<Core2>(cases[i][0], cases[i][1], cases[i][2]);+ }+ }+ if (_cd->AVX512BW()) {+ using Core0 = gemm::KblockGemmAvx512bw<48, 4>;+ for (size_t i = 0; i < cases.size(); i++) {+ ut<Core0>(cases[i][0], cases[i][1], cases[i][2]);+ }+ }+ if (_cd->AVX512_VNNI()) {+ using Core0 = gemm::KblockGemmAvx512vnni<48, 4>;+ using Core1 = gemm::KblockGemmAvx512vnni<48, 5>;+ for (size_t i = 0; i < cases.size(); i++) {+ ut<Core0>(cases[i][0], cases[i][1], cases[i][2]);+ ut<Core1>(cases[i][0], cases[i][1], cases[i][2]);+ }+ }+ if (_cd->AMX_INT8()) {+ using Core0 = gemm::KblockGemmAmxint8<48, 16>;+ using Core1 = gemm::KblockGemmAmxint8SS<48, 16>;+ for (size_t i = 0; i < cases.size(); i++) {+ ut<Core0>(cases[i][0], cases[i][1], cases[i][2]);+ ut<Core1>(cases[i][0], cases[i][1], cases[i][2]);+ }+ }+ }++ template <class Core>+ void ut(int n, int k, int kblock) {+ auto m = Core::MTILE;+ printf("Test Case %s: %d %d %d-%d\n", gemm::CoreAttr::to_str(Core::ID), m, n, k, kblock);+ if (n % Core::NTILE != 0) {+ return;+ }+ if (k % Core::KTILE != 0) {+ return;+ }+ if (kblock % Core::KTILE != 0) {+ return;+ }+ using AT = typename Core::AType;+ avector<AT> A(m * k);+ avector<int8_t> B(k * n);+ avector<float> C(m * n, 0), RefC(m * n, 0);+ int blk_num = utils::updiv(k, kblock);+ avector<float> scaleA(blk_num * m), scaleB(blk_num * n);+ avector<int> reduceB(blk_num * n, 0);+ avector<AT> zpA(m * blk_num);+ fill_buffer_randn(A.data(), A.size(), (AT)0, (AT)127);+ fill_buffer_randn(zpA.data(), zpA.size(), (AT)0, (AT)55);+ fill_buffer_randn(B.data(), B.size(), (int8_t)-127, (int8_t)127);+ fill_buffer_randn(scaleA.data(), scaleA.size(), 0.003f, 0.005f);+ fill_buffer_randn(scaleB.data(), scaleB.size(), 0.003f, 0.005f);+ for (size_t i = 0; i < k; i++) {+ for (size_t j = 0; j < n; j++) {+ reduceB[i / kblock * n + j] += B[i * n + j];+ }+ }+ if constexpr (std::is_same_v<AT, uint8_t>) {+ ref_kblock_int8<Core::Code::NTILE>(A.data(), B.data(), RefC.data(), zpA.data(), scaleA.data(), blk_num,+ scaleB.data(), reduceB.data(), n, m, n, k, kblock, k * sizeof(A[0]),+ k * sizeof(B[0]), n * sizeof(C[0]), 0);+ } else {+ ref_kblock_int8_ss<Core::Code::NTILE>(A.data(), B.data(), RefC.data(), scaleA.data(), blk_num, scaleB.data(), n,+ m, n, k, kblock, k * sizeof(A[0]), k * sizeof(B[0]), n * sizeof(C[0]), 0);+ }+ Core::configure(m, n, k);+ Core::forward(A.data(), B.data(), C.data(), std::is_same_v<AT, uint8_t> ? zpA.data() : nullptr, scaleA.data(),+ blk_num, scaleB.data(), reduceB.data(), n, m, n, k, kblock, k * sizeof(A[0]), k * sizeof(B[0]),+ n * sizeof(C[0]), 0, cache, CacheSize);+ ut::buffer_error(RefC.data(), C.data(), RefC.size(), 0.001f);+ }+};+#ifdef BTLA_UT_GEMM+static UT_GEMM_KBLOCK sUT_GEMM_KBLOCK;+#endif++class UT_GEMM_AVXVNNI {+ public:+ UT_GEMM_AVXVNNI() {+ UT_START();+ CheckISA(AVX_VNNI);+ ut<24>(4, 48, 12);+ ut<48>(2, 96, 12);++ ut_ss<24>(4, 48, 12);+ ut_ss<48>(2, 96, 12);+ }++ template <int NTile>+ void ut(int m, int n, int k) {+ printf("Test Case: %d %d %d\n", m, n, k);+ using Core = gemm::ICoreRowNAvxvnni<NTile>;+ static Core gemm;+ if (n % Core::Code::NTILE != 0) {+ return;+ }+ if (k % Core::Code::KTILE != 0) {+ return;+ }+ avector<uint8_t> A(m * k);+ avector<int8_t> B(k * n);+ avector<int> C(m * n, 0), RefC(m * n, 0);+ fill_buffer_randn(A.data(), A.size(), (uint8_t)0, (uint8_t)255);+ fill_buffer_randn(B.data(), B.size(), (int8_t)-127, (int8_t)127);+ ref_int8<Core::Code::NTILE>(A.data(), B.data(), RefC.data(), m, n, k, k * sizeof(A[0]), k * sizeof(B[0]),+ n * sizeof(C[0]), 0);++ gemm.forward(A.data(), B.data(), C.data(), m, n, k, k * sizeof(A[0]), k * sizeof(B[0]), n * sizeof(C[0]), 0, cache,+ CacheSize);+ ut::buffer_error(RefC.data(), C.data(), RefC.size(), 1);+ }++ template <int NTile>+ void ut_ss(int m, int n, int k) {+ printf("Test Case: %d %d %d\n", m, n, k);+ using Core = gemm::ICoreRowNAvxvnniSS<NTile>;+ static Core gemm;+ if (n % Core::Code::NTILE != 0) {+ return;+ }+ if (k % Core::Code::KTILE != 0) {+ return;+ }+ avector<int8_t> A(m * k);+ avector<int8_t> B(k * n);+ avector<int> C(m * n, 0), RefC(m * n, 0);+ fill_buffer_randn(A.data(), A.size(), (int8_t)-127, (int8_t)127);+ fill_buffer_randn(B.data(), B.size(), (int8_t)-127, (int8_t)127);+ ref_int8<Core::Code::NTILE>(A.data(), B.data(), RefC.data(), m, n, k, k * sizeof(A[0]), k * sizeof(B[0]),+ n * sizeof(C[0]), 0);++ gemm.forward(A.data(), B.data(), C.data(), m, n, k, k * sizeof(A[0]), k * sizeof(B[0]), n * sizeof(C[0]), 0, cache,+ CacheSize);+ ut::buffer_error(RefC.data(), C.data(), RefC.size(), 1);+ }+};+#ifdef BTLA_UT_GEMM+static UT_GEMM_AVXVNNI sUT_GEMM_AVXVNNI;+#endif++class UT_GEMM_AVX2VNNI {+ public:+ UT_GEMM_AVX2VNNI() {+ UT_START();+ CheckISA(AVX2);+ ut<24>(1, 48, 12);+ ut<24>(2, 48, 12);++ ut_ss<24>(1, 48, 12);+ ut_ss<24>(2, 96, 12);+ }++ template <int NTile>+ void ut(int m, int n, int k) {+ printf("Test Case: %d %d %d\n", m, n, k);+ using Core = gemm::ICoreRowNAvx2vnni<NTile, 3>;+ static Core gemm;+ if (n % Core::Code::NTILE != 0) {+ return;+ }+ if (k % Core::Code::KTILE != 0) {+ return;+ }+ avector<uint8_t> A(m * k);+ avector<int8_t> B(k * n);+ avector<int> C(m * n, 0), RefC(m * n, 0);+ fill_buffer_randn(A.data(), A.size(), (uint8_t)0, (uint8_t)127);+ fill_buffer_randn(B.data(), B.size(), (int8_t)-127, (int8_t)127);+ ref_int8<Core::Code::NTILE>(A.data(), B.data(), RefC.data(), m, n, k, k * sizeof(A[0]), k * sizeof(B[0]),+ n * sizeof(C[0]), 0);++ gemm.forward(A.data(), B.data(), C.data(), m, n, k, k * sizeof(A[0]), k * sizeof(B[0]), n * sizeof(C[0]), 0, cache,+ CacheSize);+ ut::buffer_error(RefC.data(), C.data(), RefC.size(), 1);+ }++ template <int NTile>+ void ut_ss(int m, int n, int k) {+ printf("Test Case: %d %d %d\n", m, n, k);+ using Core = gemm::ICoreRowNAvx2vnniSS<NTile, 3>;+ static Core gemm;+ if (n % Core::Code::NTILE != 0) {+ return;+ }+ if (k % Core::Code::KTILE != 0) {+ return;+ }+ avector<int8_t> A(m * k);+ avector<int8_t> B(k * n);+ avector<int> C(m * n, 0), RefC(m * n, 0);+ fill_buffer_randn(A.data(), A.size(), (int8_t)-127, (int8_t)127);+ fill_buffer_randn(B.data(), B.size(), (int8_t)-127, (int8_t)127);+ ref_int8<Core::Code::NTILE>(A.data(), B.data(), RefC.data(), m, n, k, k * sizeof(A[0]), k * sizeof(B[0]),+ n * sizeof(C[0]), 0);++ gemm.forward(A.data(), B.data(), C.data(), m, n, k, k * sizeof(A[0]), k * sizeof(B[0]), n * sizeof(C[0]), 0, cache,+ CacheSize);+ ut::buffer_error(RefC.data(), C.data(), RefC.size(), 1);+ }+};+#ifdef BTLA_UT_GEMM+static UT_GEMM_AVX2VNNI sUT_GEMM_AVX2VNNI;+#endif++class UT_GEMM_AVX512FP16 {+ public:+ UT_GEMM_AVX512FP16() {+ UT_START();+ CheckISA(AVX512_FP16);+ ut<32, 0>(4, 64, 3);+ ut<64, 0>(4, 128, 3);+ }++ template <int NTILE, int MTILE>+ void ut(int m, int n, int k) {+ printf("Test Case: %d %d %d\n", m, n, k);+ using Core = gemm::HCoreRowNAvx512fp16<NTILE, MTILE>;+ static Core gemm;+ if (n % Core::Code::NTILE != 0) {+ return;+ }+ if (k % Core::Code::KTILE != 0) {+ return;+ }++ avector<utils::fp16> matAbf16(m * k), matBbf16(k * n), matC(m * n), refC(m * n);+ fill_buffer_randn(matAbf16.data(), matAbf16.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ fill_buffer_randn(matBbf16.data(), matBbf16.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ int reordered_bstride = k * 2;+ ref_fp16<Core::NTILE>(matAbf16.data(), matBbf16.data(), refC.data(), m, n, k, k * 2, reordered_bstride, n * 2, 0);+ gemm.forward(matAbf16.data(), matBbf16.data(), matC.data(), m, n, k, k * sizeof(fp16), k * sizeof(fp16),+ n * sizeof(fp16), 0, cache, CacheSize);+ ut::buffer_error(refC.data(), matC.data(), refC.size(), fp16(FP16_ERR));+ }+};+#ifdef BTLA_UT_GEMM+static UT_GEMM_AVX512FP16 sUT_GEMM_AVX512FP16;+#endif++class UT_GEMM_AVX512BF16 {+ public:+ UT_GEMM_AVX512BF16() {+ UT_START();+ CheckISA(AVX512_BF16);+ ut<48, 0>(4, 96, 6);+ ut<48, 8>(4, 96, 6);+ ut<64, 0>(4, 128, 6);+ }++ template <int NTILE, int MTILE>+ void ut(int m, int n, int k) {+ printf("Test Case: %d %d %d\n", m, n, k);+ using Core = gemm::HCoreRowNAvx512bf16<NTILE, MTILE>;+ static Core gemm;+ if (n % Core::Code::NTILE != 0) {+ return;+ }+ if (k % Core::Code::KTILE != 0) {+ return;+ }++ avector<utils::bf16> matAbf16(m * k), matBbf16(k * n);+ avector<float> matC(m * n), refC(m * n);+ fill_buffer_randn(matAbf16.data(), matAbf16.size(), utils::bf16(-0.5f), utils::bf16(0.5f));+ fill_buffer_randn(matBbf16.data(), matBbf16.size(), utils::bf16(-0.5f), utils::bf16(0.5f));+ ref_bf16<Core::NTILE>(matAbf16.data(), matBbf16.data(), refC.data(), m, n, k, k * 2, k * 2, n * 4, 0);+ gemm.forward(matAbf16.data(), matBbf16.data(), matC.data(), m, n, k, k * sizeof(bf16), k * sizeof(bf16),+ n * sizeof(float), 0, cache, CacheSize);+ ut::buffer_error(refC.data(), matC.data(), refC.size(), 0.001f);+ }+};+#ifdef BTLA_UT_GEMM+static UT_GEMM_AVX512BF16 sUT_GEMM_AVX512BF16;+#endif++class UT_GEMM_AMXBF16 {+ public:+ UT_GEMM_AMXBF16() {+ UT_START();+ CheckISA(AMX_BF16);+ ut<32, 32>(32, 32, 64);+ ut<32, 32>(4, 96, 96);+ ut<48, 0>(4, 96, 96);+ ut<64, 16>(4, 128, 96);+ }++ template <int NTILE, int MTILE>+ void ut(int m, int n, int k) {+ printf("Test Case: %d %d %d\n", m, n, k);+ using Core = gemm::HCoreRowNAmxbf16<NTILE, MTILE>;+ static Core gemm;+ if (n % Core::Code::NTILE != 0) {+ return;+ }+ if (k % Core::Code::KTILE != 0) {+ return;+ }++ avector<utils::bf16> matAbf16(m * k), matBbf16(k * n);+ avector<float> matC(Core::Code::MTILE * n), refC(Core::Code::MTILE * n);+ fill_buffer_randn(matAbf16.data(), matAbf16.size(), utils::bf16(-0.5f), utils::bf16(0.5f));+ fill_buffer_randn(matBbf16.data(), matBbf16.size(), utils::bf16(-0.5f), utils::bf16(0.5f));+ ref_bf16<Core::NTILE>(matAbf16.data(), matBbf16.data(), refC.data(), m, n, k, k * 2, k * 2, n * 4, 0);+ gemm.configure(m, n, k);++ gemm.forward(matAbf16.data(), matBbf16.data(), matC.data(), m, n, k, k * sizeof(bf16), k * sizeof(bf16),+ n * sizeof(float), 0, cache, CacheSize);+ ut::buffer_error(refC.data(), matC.data(), m * n, 0.001f);+ }+};+#ifdef BTLA_UT_GEMM+static UT_GEMM_AMXBF16 sUT_GEMM_AMXBF16;+#endif++class UT_GEMM_AMXFP16 {+ public:+ UT_GEMM_AMXFP16() {+ UT_START();+ CheckISA(AMX_FP16);+ ut<32, 32>(32, 32, 64);+ ut<32, 32>(4, 96, 96);+ ut<48, 0>(4, 96, 96);+ ut<64, 16>(4, 128, 96);+ }++ template <int NTILE, int MTILE>+ void ut(int m, int n, int k) {+ printf("Test Case: %d %d %d\n", m, n, k);+ using Core = gemm::HCoreRowNAmxfp16<NTILE, MTILE>;+ static Core gemm;+ if (n % Core::Code::NTILE != 0) {+ return;+ }+ if (k % Core::Code::KTILE != 0) {+ return;+ }++ avector<utils::fp16> matAfp16(m * k), matBfp16(k * n);+ avector<float> matC(Core::Code::MTILE * n), refC(Core::Code::MTILE * n);+ fill_buffer_randn(matAfp16.data(), matAfp16.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ fill_buffer_randn(matBfp16.data(), matBfp16.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ ref_fp16_fp32<Core::NTILE>(matAfp16.data(), matBfp16.data(), refC.data(), m, n, k, k * 2, k * 2, n * 4, 0);+ gemm.configure(m, n, k);++ gemm.forward(matAfp16.data(), matBfp16.data(), matC.data(), m, n, k, k * sizeof(fp16), k * sizeof(fp16),+ n * sizeof(float), 0, cache, CacheSize);+ ut::buffer_error(refC.data(), matC.data(), m * n, 0.001f);+ }+};+#ifdef BTLA_UT_GEMM+static UT_GEMM_AMXFP16 sUT_GEMM_AMXFP16;+#endif++class UT_GEMM_AMXINT8 {+ public:+ UT_GEMM_AMXINT8() {+ UT_START();+ CheckISA(AMX_INT8);+ ut<32, 32>(32, 64, 64 * 3);+ ut<48, 16>(16, 96, 64 * 3);+ ut<32, 32>(4, 64, 64 * 3);+ ut<32, 32>(20, 64, 64 * 3);+ ut<32, 32>(32, 64, 64 * 3);++ ut<64, 16>(16, 128, 64 * 3);+ }++ template <int NTile, int MTile>+ void ut(int m, int n, int k) {+ printf("Test Case: %d %d %d\n", m, n, k);+ using Core = gemm::ICoreRowNAmxint8<NTile, MTile>;+ static Core gemm;+ if (n % Core::Code::NTILE != 0) {+ return;+ }+ if (k % Core::Code::KTILE != 0) {+ return;+ }+ avector<uint8_t> A(m * k);+ avector<int8_t> B(k * n);+ avector<int> C(Core::Code::MTILE * n, 0), RefC(Core::Code::MTILE * n, 0);+ fill_buffer_randn(A.data(), A.size(), (uint8_t)0, (uint8_t)255);+ fill_buffer_randn(B.data(), B.size(), (int8_t)-127, (int8_t)127);+ ref_int8<Core::Code::NTILE>(A.data(), B.data(), RefC.data(), m, n, k, k * sizeof(A[0]), k * sizeof(B[0]),+ n * sizeof(C[0]), 0);+ gemm.configure(m, n, k);+ gemm.forward(A.data(), B.data(), C.data(), m, n, k, k * sizeof(A[0]), k * sizeof(B[0]), n * sizeof(C[0]), 0, cache,+ CacheSize);+ ut::buffer_error(RefC.data(), C.data(), m * n, 1);+ }+};+#ifdef BTLA_UT_GEMM+static UT_GEMM_AMXINT8 sUT_GEMM_AMXINT8;+#endif+} // namespace ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_parallel.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_parallel.cppnew file mode 100644index 000000000..734937790--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_parallel.cpp@@ -0,0 +1,279 @@+#include "bestla_utils.h"+#include "bestla_parallel.h"+#include "bestla_device.h"+#include "bestla_gemm.h"+#include "bestla_ut.h"+#include "bestla_prologue_a.h"++namespace bestla {+using namespace utils;+namespace ut {+#ifdef _OPENMP+class UT_OMPThreading {+ public:+ UT_OMPThreading() {+ UT_START();+ GetCPUDevice();+ ut_transpose(1024, 1024, _cd->getThreads());+ ut_transpose(123, 111, _cd->getThreads());+ }++ void ut_transpose(int row, int col, int threads) {+ printf("%s %d %d %d\n", __FUNCTION__, row, col, threads);+ avector<float> src(row * col), dst(row * col), ref(row * col);+ fill_buffer_randn(src.data(), src.size(), -0.5f, 0.5f);+ int ld_src = col, ld_dst = row;+ kernel::wrapper::Transpose2D<float>::template forward<BTLA_ISA::AVX512F>(src.data(), ref.data(), row, col, col,+ row);+ parallel::Scheduler2D _para({threads, row, col, 1, 1, 0, 0});+ UT_Threading::get()->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ kernel::wrapper::Transpose2D<float>::template forward<BTLA_ISA::AVX512F>(+ src.data() + thdp.loc[0] * ld_src + thdp.loc[1], dst.data() + thdp.loc[0] + thdp.loc[1] * ld_dst,+ thdp.size[0], thdp.size[1], ld_src, ld_dst);+ }+ });+ buffer_error(ref.data(), dst.data(), ref.size());+ }+};+#ifdef BTLA_UT_PARALLEL+static UT_OMPThreading sUT_OMPThreading;+#endif+#endif++class UT_StdThreading {+ public:+ UT_StdThreading() {+ UT_START();+ GetCPUDevice();+ ut_transpose(1024, 1024, _cd->getThreads());+ ut_transpose(123, 111, _cd->getThreads());+ }++ void ut_transpose(int row, int col, int threads) {+ printf("%s %d %d %d\n", __FUNCTION__, row, col, threads);+ avector<float> src(row * col), dst(row * col), ref(row * col);+ fill_buffer_randn(src.data(), src.size(), -0.5f, 0.5f);+ int ld_src = col, ld_dst = row;+ kernel::wrapper::Transpose2D<float>::template forward<BTLA_ISA::AVX512F>(src.data(), ref.data(), row, col, col,+ row);+ parallel::Scheduler2D _para({threads, row, col, 1, 1, 0, 0});+ UT_Threading::get()->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ _para.getIndex(thdp);+ if (thdp.valid) {+ kernel::wrapper::Transpose2D<float>::template forward<BTLA_ISA::AVX512F>(+ src.data() + thdp.loc[0] * ld_src + thdp.loc[1], dst.data() + thdp.loc[0] + thdp.loc[1] * ld_dst,+ thdp.size[0], thdp.size[1], ld_src, ld_dst);+ }+ });+ buffer_error(ref.data(), dst.data(), ref.size());+ }+};+#ifdef BTLA_UT_PARALLEL+static UT_StdThreading sUT_StdThreading;+#endif++class UT_StdThreadingV2 {+ public:+ UT_StdThreadingV2() {+ UT_START();+ new_thread_group();+ gelu_test(4, 8);+ // gemm_test_hyper();+ }++ void gelu_test(int pcores, int ecores = 8) {+ size_t batch = 100;+ size_t count = 4 * 384 * 1024;+ utils::avector<float> src(batch * count, 1.f), dst(batch * count), ref(batch * count);+ utils::timer tm;+ tm.start();+ for (size_t i = 0; i < batch; i++) {+ bestla::kernel::wrapper::Memcpy2D::forward1_auto<float, float, BTLA_ELTWISEOP::GELU>(+ src.data() + i * count, ref.data() + i * count, 1, count, 0, 0, nullptr);+ }+ printf("Single Time:%f\n", tm.stop());+ parallel::StdGroupThreading th;+ th.set_threads(device::CpuRange(pcores, 0, false));+ for (size_t i = 0; i < batch; i++) {+ th.parallel_for(parallel::Config{count, 64}, [&](const parallel::ThreadRange& tr) {+ if (tr.size_) {+ bestla::kernel::wrapper::Memcpy2D::forward1_auto<float, float, BTLA_ELTWISEOP::GELU>(+ src.data() + tr.loc_ + i * count, dst.data() + tr.loc_ + i * count, 1, tr.size_, 0, 0, nullptr);+ }+ });+ }+ buffer_error(dst.data(), ref.data(), dst.size());++ tm.start();+ utils::timer_statistics_logger<5000> log;+ while (tm.stop() < 3000) {+ log.start();+ for (size_t i = 0; i < batch; i++) {+ th.parallel_for(parallel::Config{count, 64}, [&](const parallel::ThreadRange& tr) {+ if (tr.size_) {+ bestla::kernel::wrapper::Memcpy2D::forward1_auto<float, float, BTLA_ELTWISEOP::GELU>(+ src.data() + tr.loc_ + i * count, dst.data() + tr.loc_ + i * count, 1, tr.size_, 0, 0, nullptr);+ }+ });+ }+ log.stop();+ }+ log.record();+ printf("MT %s\n", log.get_log_str());++ th.set_threads(device::CpuRange(pcores, ecores, false));+ th.enable_profiling();++ for (size_t i = 0; i < batch; i++) {+ th.parallel_for(parallel::Config{count, 64, 0}, [&](const parallel::ThreadRange& tr) {+ if (tr.size_) {+ bestla::kernel::wrapper::Memcpy2D::forward1_auto<float, float, BTLA_ELTWISEOP::GELU>(+ src.data() + tr.loc_ + i * count, dst.data() + tr.loc_ + i * count, 1, tr.size_, 0, 0, nullptr);+ }+ });+ }+ th.disable_profiling();+ log.clear();+ tm.start();+ th.enable_profiling(1);+ while (tm.stop() < 3000) {+ log.start();+ for (size_t i = 0; i < batch; i++) {+ th.parallel_for(parallel::Config{count, 64, 0}, [&](const parallel::ThreadRange& tr) {+ if (tr.size_) {+ bestla::kernel::wrapper::Memcpy2D::forward1_auto<float, float, BTLA_ELTWISEOP::GELU>(+ src.data() + tr.loc_ + i * count, dst.data() + tr.loc_ + i * count, 1, tr.size_, 0, 0, nullptr);+ }+ });+ }+ log.stop();+ }+ log.record();+ printf("MT %s\n", log.get_log_str());+ }++ void new_thread_group() {+ device::CpuRange crange(8, 4, true);+ crange.print();+ parallel::StdGroupThreading thread;+ thread.enable_profiling();+ thread.set_threads(crange);+ auto wait_func = [&](const parallel::ThreadRange& tr) {+ utils::timer tm;+ int n = 0;+ printf("Thread %d-%d Start\n", tr.info_.group_idx_, tr.info_.thread_idx_);+ tm.start();+ while (tm.stop() < 3000) {+ n++;+ }+ printf("Thread %d-%d End %d\n", tr.info_.group_idx_, tr.info_.thread_idx_, n);+ };+ thread.parallel_for({}, wait_func);+ thread.set_threads(device::CpuRange(1, 8, true));+ thread.parallel_for({}, wait_func);+ thread.set_threads(device::CpuRange(8, 0, true));+ thread.parallel_for({}, wait_func);+ }+};+#ifdef BTLA_UT_PARALLEL+static UT_StdThreadingV2 sUT_StdThreadingV2;+#endif++class UT_Scheduler2D {+ public:+ UT_Scheduler2D() {+ UT_START();+ ut(4096, 4096, 24);+ ut(4096, 4096, 28);+ ut(4096, 4096, 48);+ ut(4096, 4096, 56);+ }++ void ut(int row, int col, int threads) {+ printf("%s %d %d %d\n", __FUNCTION__, row, col, threads);+ parallel::Scheduler2D sch;+ sch.update({threads, row, col, 1, 1, 0, 0});+ sch.print();+ parallel::ThreadProblem2D prb{threads - 1};+ sch.getIndex(prb);+ prb.print();+ }+};+#ifdef BTLA_UT_PARALLEL+static UT_Scheduler2D sUT_Scheduler2D;+#endif++class UT_SchedulerGemmBase {+ public:+ UT_SchedulerGemmBase() {+ UT_START();+ ut<gemm::ICoreRowNAmxint8<64, 16>>(2024, 11008, 4096, 48, 2048 * 1024, 32 * 1024);+ ut<gemm::ICoreRowNAmxint8<48, 16>>(2048, 4096, 4096, 48, 2048 * 1024, 32 * 1024);+ ut<gemm::ICoreRowNAmxint8SS<32, 32>>(2048, 4096, 4096, 24);+ ut<gemm::SCoreRowNAvx512f<48, 8>>(1, 4096, 4096, 24);+ ut<gemm::ICoreRowNAmxint8SS<32, 32>>(4, 4096, 4096, 48);+ }++ template <class GemmCore_T>+ void ut(int m, int n, int k, int threads, size_t l2cache = 0, size_t l1cache = 0) {+ printf("%s %d %d %d %d core:%s\n", __FUNCTION__, m, n, k, threads, gemm::CoreAttr::to_str(GemmCore_T::ID));+ parallel::gemm::SchedulerBase<GemmCore_T> sch;+ GetCPUDevice();+ utils::GemmProblem gp(1, m, n, k);+ sch.update({threads, gp, 0, 0, l2cache == 0 ? _cd->getL2CacheSize() : l2cache,+ l1cache == 0 ? _cd->getL1CacheSize() : l1cache});+ sch.print();+ parallel::gemm::ThreadProblemBase prb{sch.valid_theads() - 1};+ sch.getIndex(prb);+ prb.print();+ }+};+#ifdef BTLA_UT_PARALLEL+static UT_SchedulerGemmBase sUT_SchedulerGemmBase;+#endif++class UT_SchedulerGemmKBlockNew {+ public:+ UT_SchedulerGemmKBlockNew() {+ UT_START();+ ut<gemm::ICoreRowNAmxint8SS<48, 16>>(2011, 32000, 4096, 128, 32);+ ut<gemm::ICoreRowNAmxint8SS<64, 16>>(2048, 4096, 4096, 4096, 48);+ ut<gemm::ICoreRowNAmxint8SS<64, 16>>(2048, 4096, 4096, 128, 24);+ ut<gemm::ICoreRowNAmxint8SS<64, 16>>(2048, 4096, 4096, 4096, 24);+ ut<gemm::SCoreRowNAvx512f<48, 8>>(1, 4096, 4096, 32, 24);+ ut<gemm::SCoreRowNAvx512f<48, 8>>(1, 4096, 4096, 64, 22, 32 * 1024);+ ut<gemm::SCoreRowNAvx512f<48, 8>>(1, 4096, 4096, 128, 24);+ ut<gemm::SCoreRowNAvx512f<48, 8>>(1, 4096, 4096, 1024, 24);+ ut<gemm::ICoreRowNAmxint8SS<64, 16>>(1, 4096, 4096, 64, 24, 32 * 1024);+ ut<gemm::ICoreRowNAmxint8SS<64, 16>>(2048, 4096, 4096, 64, 24);+ ut<gemm::ICoreRowNAmxint8SS<64, 16>>(2048, 4096, 4096, 4096, 56);+ ut<gemm::ICoreRowNAmxint8SS<64, 16>>(2048, 4096, 4096, 64, 56);+ ut<gemm::ICoreRowNAmxint8SS<64, 16>>(4, 4096, 4096, 128, 48);+ ut<gemm::ICoreRowNAmxint8SS<64, 16>>(4, 4096, 3072, 64, 48);+ ut<gemm::ICoreRowNAmxint8SS<64, 16>>(2048, 4096, 3072, 3072, 48);+ ut<gemm::ICoreRowNAmxint8SS<64, 16>>(2048, 4096, 3072, 64, 56);+ }++ template <class GemmCore_T>+ void ut(int m, int n, int k, int kblock, int threads, size_t l1cache = 0) {+ printf("%s %d %d %d %d %d core:%s\n", __FUNCTION__, m, n, k, kblock, threads,+ gemm::CoreAttr::to_str(GemmCore_T::ID));+ parallel::gemm::SchedulerIntKBlock<GemmCore_T> sch;+ GetCPUDevice();+ utils::GemmProblem gp(1, m, n, k, kblock);+ sch.update({threads, gp, 0, 0, _cd->getL2CacheSize(), l1cache == 0 ? _cd->getL1CacheSize() : l1cache});+ sch.print();+ parallel::gemm::ThreadProblemBase prb{sch.valid_theads() - 1};+ sch.getIndex(prb);+ prb.print();+ }+};+#ifdef BTLA_UT_PARALLEL+static UT_SchedulerGemmKBlockNew sUT_SchedulerGemmKBlockNew;+#endif+} // namespace ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_prologue_a.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_prologue_a.cppnew file mode 100644index 000000000..5c2853f6e--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_prologue_a.cpp@@ -0,0 +1,301 @@+#include "bestla_prologue_a.h"+#include "bestla_ut.h"+#include "kernel_avx512f.h"++namespace bestla {+using namespace utils;+namespace ut {+class UT_ActivationBase {+ public:+ UT_ActivationBase() {+ UT_START();+ CheckISA(AVX512F);+ ut<gemm::SCoreRowNAvx512f<48, 8>>(8, 3, 128);+ ut<gemm::SCoreRowNAvx512f<48, 8>>(8, 48, 128);+ ut<gemm::SCoreRowNAvx512f<48, 8>>(8, 3, 128);+ }+ template <typename _T>+ void ut(int m, int k, int lda) {+ int kpad = padto(k, _T::KTILE);+ using BType = typename _T::BType;+ printf("Test Case %dB NTile%d: %d %d %d %d \n", int(sizeof(BType)), _T::NTILE, m, k, lda, kpad);+ std::vector<BType> src(m * lda);+ std::vector<BType> dst(m * kpad), dstref(m * kpad);+ for (int i = 0; i < src.size(); i++) {+ src[i] = static_cast<BType>(i);+ }+ using ProA = prologue_a::gemm::ActivationBase<_T>;+ auto dstrefptr = dstref.data();+ auto dstptr = dst.data();+ int dststride = 0;+ ProA::template getActivation<BTLA_ISA::NoSIMD>(&dstrefptr, &dststride, {src.data(), lda}, m, k, 0, 0, cache,+ CacheSize);+ GetCPUDevice();+ if (_cd->AVX512F()) {+ ProA::template getActivation<BTLA_ISA::AVX512F>(&dstptr, &dststride, {src.data(), lda}, m, k, 0, 0, cache,+ CacheSize);+ ut::buffer_error(dst.data(), dstref.data(), dst.size());+ }+ }+};+#ifdef BTLA_UT_PROLOGUE_A+static UT_ActivationBase sUT_ActivationBase;+#endif++class UT_ActivationConverter {+ public:+ UT_ActivationConverter() {+ UT_START();+ CheckISA(AMX_BF16);+ ut<float, gemm::HCoreRowNAmxbf16<64, 16>>(8, 3, 128);+ ut<float, gemm::HCoreRowNAmxbf16<64, 16>>(8, 48, 128);+ ut<utils::bf16, gemm::SCoreRowNAvx512f<48, 8>>(8, 3, 128);+ ut<utils::bf16, gemm::SCoreRowNAvx512f<48, 8>>(8, 48, 128);+ }++ template <typename SRC_T, typename _T>+ void ut(int m, int k, int lda) {+ using SrcType = SRC_T;+ using AType = typename _T::AType;+ int kpad = padto(k, _T::KTILE);+ printf("Test Case %dB NTile%d: %d %d %d %d \n", int(sizeof(AType)), _T::NTILE, m, k, lda, kpad);+ std::vector<SrcType> src(m * lda);+ std::vector<AType> dst(m * kpad), dstref(m * kpad);+ for (int i = 0; i < src.size(); i++) {+ src[i] = static_cast<SrcType>(float(i));+ }+ using ProA = prologue_a::gemm::ActivationConverter<_T, SRC_T>;+ auto dstptr = dstref.data();+ int dststride = 0;+ auto ret = ProA::template getActivation<BTLA_ISA::NoSIMD>(&dstptr, &dststride, {src.data(), lda}, m, k, 0, 0, cache,+ CacheSize);+ assert(ret == BTLA_CODE::Success);+ dstptr = dst.data();+ ret = ProA::template getActivation<BTLA_ISA::AVX512F>(&dstptr, &dststride, {src.data(), lda}, m, k, 0, 0, cache,+ CacheSize);+ assert(ret == BTLA_CODE::Success);+ ut::buffer_error(dst.data(), dstref.data(), dst.size(), AType{0});+ aligned_vector<SrcType> revert(dst.size());+ for (size_t i = 0; i < revert.size(); i++) {+ revert[i] = utils::cast<AType, SrcType>(dst[i]);+ }+ for (size_t i = 0; i < src.size(); i++) {+ auto tmp = utils::cast<SrcType, AType>(src[i]);+ src[i] = utils::cast<AType, SrcType>(tmp);+ }+ buffer_error_2d(src.data(), revert.data(), m, k, lda, kpad);+ }+};+#ifdef BTLA_UT_PROLOGUE_A+static UT_ActivationConverter sUT_ActivationConverter;+#endif++class UT_ActivationU8KBlockQuantize {+ public:+ UT_ActivationU8KBlockQuantize() {+ UT_START();+ CheckISA(AVX512F);+ ut<gemm::ICoreRowNAmxint8<48, 16>>(15, 63, 64, 2);+ ut<gemm::ICoreRowNAvx512vnni<48, 8>>(2, 4096, 4096, 32, true);+ ut<gemm::ICoreRowNAvx512vnni<48, 8>>(2, 4096, 4096, 4096, true);+ ut<gemm::ICoreRowNAvxvnni<48, 2>>(2, 4096, 4096, 32, true);+ ut<gemm::ICoreRowNAvxvnni<48, 2>>(2, 4096, 4096, 4096, true);+ ut<gemm::ICoreRowNAvx512vnni<48, 8>>(2, 4096, 4096, 4096);+ ut<gemm::ICoreRowNAvx512vnni<48, 8>>(2, 4096, 4096, 128);+ ut<gemm::ICoreRowNAvx512vnni<48, 8>>(2, 4096, 4096, 32);+ ut<gemm::ICoreRowNAvx512vnni<48, 8>>(2, 11040, 11040, 32);+ ut<gemm::ICoreRowNAvx512vnni<48, 8>>(1024, 4096, 4096, 32);+ ut<gemm::ICoreRowNAvx512vnni<48, 8>>(1024, 11040, 11040, 32);+ ut<gemm::ICoreRowNAvxvnni<48, 2>>(2, 4096, 4096, 4096);+ ut<gemm::ICoreRowNAvxvnni<48, 2>>(2, 4096, 4096, 128);+ ut<gemm::ICoreRowNAvxvnni<48, 2>>(2, 4096, 4096, 32);+ ut<gemm::ICoreRowNAvxvnni<48, 2>>(2, 11040, 11040, 32);+ ut<gemm::ICoreRowNAvxvnni<48, 2>>(1024, 4096, 4096, 32);+ ut<gemm::ICoreRowNAvxvnni<48, 2>>(1024, 11040, 11040, 32);+ }+ template <typename _T>+ void ut(int m, int k, int lda, int kblock, bool hasreduce = false) {+ int kpad = padto(k, _T::KTILE);+ printf("Test Case core:%s: %d %d %d %d %d reduce:%d\n", gemm::CoreAttr::to_str(_T::ID), m, k, lda, kblock, kpad,+ hasreduce);+ int kcount = updiv(kpad, kblock);+ utils::aligned_vector<float> raw(m * lda), scales(m * kcount);+ ut::fill_buffer_randn(raw.data(), raw.size(), -0.5f, 0.5f);+ utils::aligned_vector<uint8_t> q, zp;+ q.resize(m * lda);+ zp.resize(m * kcount);+ avector<float> reduce(m * kcount);++ kernel::ref::quantize_fp_u8_colblock(m, k, raw.data(), lda, q.data(), lda, scales.data(), kcount, zp.data(), kblock,+ hasreduce ? reduce.data() : nullptr);+ using ProA = prologue_a::gemm::ActivationF32KBlockQuantize<_T>;+ auto constexpr ISA = _T::ISA;+ auto quanAct = ProA::createStorage(m, k, kblock, hasreduce);+ avector<int8_t> bufA(quanAct.mSize);+ quanAct.assign(bufA.data());+ ProA::quantize({raw.data(), lda, &quanAct}, m, k, UT_Threading::get());++ ut::buffer_error(q.data(), quanAct.template APtr<uint8_t>(), q.size(), uint8_t(1));+ ut::buffer_error(zp.data(), quanAct.template ZPtr<uint8_t>(), zp.size(), uint8_t(1));+ if (hasreduce) {+ avector<float> redref(reduce.size(), 0.f), redqref(reduce.size(), 0.f);+ for (size_t i = 0; i < m; i++) {+ for (size_t j = 0; j < k; j++) {+ redref[i * kcount + j / kblock] += raw[i * k + j];+ redqref[i * kcount + j / kblock] +=+ (float(q[i * k + j]) - zp[i * kcount + j / kblock]) * scales[i * kcount + j / kblock];+ }+ }+ buffer_error(redref.data(), reduce.data(), reduce.size(), INT8_ERR);+ buffer_error(redqref.data(), reduce.data(), reduce.size(), 0.01f);+ buffer_error(reduce.data(), quanAct.template RPtr<float>(), reduce.size(), 0.01f);+ }+ }+};+#ifdef BTLA_UT_PROLOGUE_A+static UT_ActivationU8KBlockQuantize sUT_ActivationU8KBlockQuantize;+#endif++class UT_ActivationS8KBlockQuantize {+ public:+ UT_ActivationS8KBlockQuantize() {+ UT_START();+ CheckISA(AVX512F);+ ut<gemm::ICoreRowNAmxint8SS<48, 16>>(15, 63, 2, true);+ ut<gemm::ICoreRowNAmxint8SS<48, 16>>(2, 4096, 32, true);+ ut<gemm::ICoreRowNAmxint8SS<48, 16>>(2, 4096, 4096, true);+ ut<gemm::ICoreRowNAmxint8SS<48, 16>>(2, 11040, 32);+ ut<gemm::ICoreRowNAmxint8SS<48, 16>>(2, 4096, 128);+ ut<gemm::ICoreRowNAmxint8SS<48, 16>>(2, 4096, 32);+ ut<gemm::ICoreRowNAmxint8SS<48, 16>>(1024, 4096, 32);+ ut<gemm::ICoreRowNAmxint8SS<48, 16>>(1024, 11040, 32);+ }++ template <typename _T>+ void ut(int m, int k, int kblock, bool hasreduce = false) {+ int kpad = padto(k, _T::KTILE);+ int lda = kpad;+ printf("Test Case NTile%d: %d %d %d %d %d reduce:%d\n", _T::KTILE, m, k, lda, kblock, kpad, hasreduce);+ int kcount = updiv(kpad, kblock);+ utils::aligned_vector<float> raw(m * k), scales(m * kcount);+ ut::fill_buffer_randn(raw.data(), raw.size(), -0.1f, 0.1f);+ utils::aligned_vector<int8_t> q;+ avector<float> reduce(m * kcount);+ q.resize(m * lda);+ kernel::ref::quantize_fp_s8_colblock(m, k, raw.data(), k, q.data(), lda, scales.data(), kcount, kblock,+ hasreduce ? reduce.data() : nullptr);+ using ProA = prologue_a::gemm::ActivationF32KBlockQuantize<_T>;+ auto quanAct = ProA::createStorage(m, k, kblock, hasreduce);+ auto constexpr ISA = BTLA_ISA::AVX512F;+ avector<int8_t> bufA(quanAct.mSize);+ quanAct.assign(bufA.data());+ ProA::quantize({raw.data(), k, &quanAct}, m, k, UT_Threading::get());+ ut::buffer_error(q.data(), quanAct.template APtr<int8_t>(), q.size(), int8_t(1));+ if (hasreduce) {+ avector<float> redref(reduce.size(), 0.f), redqref(reduce.size(), 0.f);+ for (size_t i = 0; i < m; i++) {+ for (size_t j = 0; j < k; j++) {+ redref[i * kcount + j / kblock] += raw[i * k + j];+ redqref[i * kcount + j / kblock] += (float(q[i * lda + j])) * scales[i * kcount + j / kblock];+ }+ }+ buffer_error(redref.data(), reduce.data(), reduce.size(), INT8_ERR);+ buffer_error(redqref.data(), reduce.data(), reduce.size(), FP32_ERR);+ buffer_error(reduce.data(), quanAct.template RPtr<float>(), reduce.size(), FP32_ERR);+ }+ }+};+#ifdef BTLA_UT_PROLOGUE_A+static UT_ActivationS8KBlockQuantize sUT_ActivationS8KBlockQuantize;+#endif++class UT_ShuffleActivationKblock {+ public:+ UT_ShuffleActivationKblock() {+ UT_START();+ CheckISA(AVX2);+ ut<float, gemm::SCoreRowNAvx2<48, 2>>(15, 63);+ ut<bf16, gemm::SCoreRowNAvx2<48, 2>>(15, 63);+ CheckISA(AMX_BF16);+ ut<float, gemm::HCoreRowNAmxbf16<64, 16>>(15, 63);+ ut<bf16, gemm::HCoreRowNAmxbf16<64, 16>>(15, 63);+ dynamic_ut(15, 63, 2, true);+ }+ template <typename _SRC_T, class GC>+ void ut(int m, int k) {+ int kpad = padto(k, GC::KTILE);+ using BType = typename GC::BType;+ printf("Test Case %s %dB NTile%d: %d %d %d \n", __FUNCTION__, int(sizeof(BType)), GC::NTILE, m, k, kpad);+ std::vector<_SRC_T> src(m * k);+ std::vector<BType> dst(m * kpad), dstref(m * kpad);+ std::vector<int> indices(k);+ for (int i = 0; i < indices.size(); i++) {+ indices[i] = i % 2 == 0 ? (i + 1) == indices.size() ? i : i + 1 : i - 1;+ }+ fill_buffer_randn(src.data(), src.size(), _SRC_T(0), _SRC_T(100));+ using ProA = prologue_a::gemm::ShuffleActivationKBlockBase<GC, _SRC_T>;+ auto dstrefptr = dstref.data();+ auto dstptr = dst.data();+ int dststride = 0;+ auto reordA = ProA::createReorderStorage(m, k, 32);+ avector<int8_t> bufA(reordA.mSize);+ reordA.assign(bufA.data());+ ProA::preprocess({src.data(), k, nullptr, indices.data(), &reordA}, m, k, 32, UT_Threading::get());++ ProA::template getActivation<GC::ISA>(&dstptr, &dststride, {src.data(), k, nullptr, indices.data(), &reordA}, m,+ kpad, 0, 0, cache, CacheSize);+ for (int i = 0; i < m; i++) {+ int j = 0;+ for (; j < k; j++) dstrefptr[i * kpad + j] = static_cast<BType>(src[i * k + indices[j]]);+ for (; j < kpad; j++) dstrefptr[i * kpad + j] = static_cast<BType>(0);+ }+ buffer_error(dstrefptr, dstptr, dst.size());+ }++ void dynamic_ut(int m, int k, int kblock, bool hasreduce = false) {+ using GC = gemm::ICoreRowNAmxint8SS<48, 16>;+ int kpad = padto(k, GC::KTILE);+ int lda = kpad;+ printf("Test Case %s NTile%d: %d %d %d %d %d reduce:%d\n", __FUNCTION__, GC::KTILE, m, k, lda, kblock, kpad,+ hasreduce);+ int kcount = updiv(kpad, kblock);+ utils::aligned_vector<float> raw(m * k, 0), raw_cp(m * k, 0), scales(m * kcount);+ ut::fill_buffer_randn(raw_cp.data(), raw_cp.size(), -0.1f, 0.1f);+ utils::aligned_vector<int8_t> q;+ avector<float> reduce(m * kcount);+ q.resize(m * lda);+ std::vector<int> indices(k);+ for (int i = 0; i < indices.size(); i++) {+ indices[i] = i % 2 == 0 ? (i + 1) == indices.size() ? i : i + 1 : i - 1;+ }+ kernel::ref::shuffle_activation(raw_cp.data(), raw.data(), m, k, 0, 0, indices.data(), k, k);+ kernel::ref::quantize_fp_s8_colblock(m, k, raw.data(), k, q.data(), lda, scales.data(), kcount, kblock,+ hasreduce ? reduce.data() : nullptr);+ using ProA = prologue_a::gemm::ShuffleActivationKBlockQuantize<GC, float>;+ auto constexpr RunISA = BTLA_ISA::NoSIMD;+ auto quanAct = ProA::createQuantStorage(m, k, kblock, hasreduce);+ auto reordAct = ProA::createReorderStorage(m, k, kblock);+ avector<int8_t> bufA(quanAct.mSize + reordAct.mSize);+ quanAct.assign(bufA.data());+ reordAct.assign(bufA.data() + quanAct.mSize);+ ProA::quantize({raw_cp.data(), k, &quanAct, indices.data(), &reordAct}, m, k, UT_Threading::get());+ ut::buffer_error(quanAct.template APtr<int8_t>(), q.data(), q.size(), int8_t(1));+ if (hasreduce) {+ avector<float> redref(reduce.size(), 0.f), redqref(reduce.size(), 0.f);+ for (size_t i = 0; i < m; i++) {+ for (size_t j = 0; j < k; j++) {+ redref[i * kcount + j / kblock] += raw[i * k + j];+ redqref[i * kcount + j / kblock] += (float(q[i * lda + j])) * scales[i * kcount + j / kblock];+ }+ }+ buffer_error(redref.data(), reduce.data(), reduce.size(), INT8_ERR);+ buffer_error(redqref.data(), reduce.data(), reduce.size(), FP32_ERR);+ buffer_error(reduce.data(), quanAct.template RPtr<float>(), reduce.size(), FP32_ERR);+ }+ }+};+#ifdef BTLA_UT_PROLOGUE_A+static UT_ShuffleActivationKblock sUT_ShuffleActivationKblock;+#endif+} // namespace ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_prologue_b.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_prologue_b.cppnew file mode 100644index 000000000..8c0852d66--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_prologue_b.cpp@@ -0,0 +1,1339 @@+#include "bestla_gemm.h"+#include "bestla_gemm_kblock.h"+#include "bestla_prologue_b.h"+#include "bestla_parallel.h"+#include "bestla_device.h"+#include "bestla_wrapper.h"+#include "bestla_ut.h"++namespace bestla {+using namespace utils;+namespace ut {+class UT_BlockQunatize_INT8 {+ public:+ UT_BlockQunatize_INT8() {+ UT_START();+ CheckISA(AVX512F);+ ut(1024, 1024, 32);+ ut(1024, 1024, 32, true);+ ut(4128, 4096, 32);+ ut(4128, 4096, 32, true);+ ut(1024, 4096, 32);+ ut(4096, 1024, 32);++ ut_transpose(4096, 4096, 32);+ ut_transpose(4096, 4096, 32, true);+ ut_transpose(4128, 4096, 32);+ ut_transpose(4128, 4096, 32, true);+ ut_transpose(1024, 4096, 32);+ ut_transpose(4096, 1024, 32);+ }++ void ut(int n, int k, int blocksize, bool asym = false) {+ printf("%s: %d %d %d %s\n", __FUNCTION__, n, k, blocksize, asym ? "asym" : "sym");+ int ldb = n;+ utils::aligned_vector<float> dequanRef(n * k);+ int kblk_num = utils::updiv(k, blocksize);+ utils::aligned_vector<float> scales(kblk_num * n);+ ut::fill_buffer_randn(scales.data(), scales.size(), 0.001f, 0.003f);+ utils::aligned_vector<int8_t> zero_points(kblk_num * n);+ ut::fill_buffer_randn(zero_points.data(), zero_points.size(), (int8_t)(-5), (int8_t)(5));+ ut::UT_vector_s8 quanW;+ quanW.resize(k * n);+ quanW.fill_rand(-127, 127);+ for (int i = 0; i < k; i++) {+ for (int j = 0; j < n; j++) {+ if (i % blocksize == 0) {+ quanW.data()[i * n + j] = 127; // make sure each block has maximum value to quantize+ }+ if (i % blocksize == 1 && asym) {+ quanW.data()[i * n + j] = -128; // make sure each block has minimum value to quantize if asym+ }+ }+ }++ for (int j = 0; j < k; j++) {+ for (int i = 0; i < n; i++) {+ if (asym) {+ dequanRef[j * ldb + i] = (float(quanW.data()[j * ldb + i]) - float(zero_points[j / blocksize * n + i])) *+ scales[j / blocksize * n + i];+ } else {+ dequanRef[j * ldb + i] = float(quanW.data()[j * ldb + i]) * scales[j / blocksize * n + i];+ }+ }+ }++ auto constexpr RuntimeISA = BTLA_ISA::AVX512F;+ using PrologueB = prologue_b::gemm::WeightKBlockNInteger<gemm::SCoreRowNAvx512f<48, 8>>;+ auto ptr = PrologueB::createStorage(n, k, blocksize, BTLA_DTYPE::S8, bestla_dtype<float>, asym);+ avector<int8_t> buffer(ptr.mSize);+ ptr.assign(buffer.data());+ PrologueB::packWeight(n, k, dequanRef.data(), ldb, &ptr, UT_Threading::get());+ avector<float> dequant(n * k);+ PrologueB::unpackWeight(n, k, &ptr, dequant.data(), n, UT_Threading::get());+ ut::buffer_error(dequanRef.data(), dequant.data(), dequanRef.size(), 0.01f);+ }++ void ut_transpose(int n, int k, int blocksize, bool asym = false) {+ printf("%s: %d %d %d %s\n", __FUNCTION__, n, k, blocksize, asym ? "asym" : "sym");+ utils::aligned_vector<float> dequanRef(n * k);+ int kblk_num = utils::updiv(k, blocksize);+ utils::aligned_vector<float> scales(kblk_num * n);+ ut::fill_buffer_randn(scales.data(), scales.size(), 0.001f, 0.003f);+ utils::aligned_vector<int8_t> zero_points(kblk_num * n);+ ut::fill_buffer_randn(zero_points.data(), zero_points.size(), (int8_t)(-5), (int8_t)(5));+ ut::UT_vector_s8 quanW;+ quanW.resize(k * n);+ quanW.fill_rand(-127, 127);+ for (int i = 0; i < k; i++) {+ for (int j = 0; j < n; j++) {+ if (i % blocksize == 0) {+ quanW.data()[i * n + j] = 127; // make sure each block has maximum value to quantize+ }+ if (i % blocksize == 1 && asym) {+ quanW.data()[i * n + j] = -128; // make sure each block has minimum value to quantize if asym+ }+ }+ }++ avector<float> dequanT(k * n);+ for (int j = 0; j < k; j++) {+ for (int i = 0; i < n; i++) {+ if (asym) {+ dequanRef[j * n + i] = (float(quanW.data()[j * n + i]) - float(zero_points[j / blocksize * n + i])) *+ scales[j / blocksize * n + i];+ } else {+ dequanRef[j * n + i] = float(quanW.data()[j * n + i]) * scales[j / blocksize * n + i];+ }+ dequanT[j + i * k] = dequanRef[j * n + i];+ }+ }++ auto constexpr RuntimeISA = BTLA_ISA::AVX512F;+ using PrologueB = prologue_b::gemm::WeightKBlockNInteger<gemm::SCoreRowNAvx512f<48, 8>>;+ auto ptr = PrologueB::createStorage(n, k, blocksize, BTLA_DTYPE::S8, bestla_dtype<float>, asym);+ avector<int8_t> buffer(ptr.mSize);+ ptr.assign(buffer.data());+ PrologueB::packTransposeWeight(n, k, dequanT.data(), k, &ptr, UT_Threading::get());+ avector<float> dequant(n * k), tardequanT(k * n);+ PrologueB::unpackWeight(n, k, &ptr, dequant.data(), n, UT_Threading::get());+ PrologueB::unpackTransposeWeight(n, k, &ptr, tardequanT.data(), k, UT_Threading::get());+ ut::buffer_error(dequanT.data(), tardequanT.data(), tardequanT.size(), 0.01f);+ ut::buffer_error(dequanRef.data(), dequant.data(), dequanRef.size(), 0.01f);+ }+};+#ifdef BTLA_UT_PROLOGUE_B+static UT_BlockQunatize_INT8 sUT_BlockQunatize_INT8;+#endif++class UT_BlockQunatize_SN {+ public:+ UT_BlockQunatize_SN() {+ UT_START();+ CheckISA(AVX2);+ ut<sAVX2>(4096, 4096, 32, BTLA_DTYPE::S1_CLIP, true);+ ut<sAVX2>(4096, 4096, 32, BTLA_DTYPE::S1_CLIP);+ ut<sAVX2>(4096, 4096, 128, BTLA_DTYPE::S1_CLIP);+ ut<sAVX2>(4096, 4096, 32, BTLA_DTYPE::S7_CLIP, true);+ ut<sAVX2>(4096, 4096, 32, BTLA_DTYPE::S7_CLIP);+ ut<sAVX2>(4096, 4096, 128, BTLA_DTYPE::S7_CLIP);+ ut<sAVX2>(4096, 4096, 32, BTLA_DTYPE::S6_CLIP, true);+ ut<sAVX2>(4096, 4096, 32, BTLA_DTYPE::S6_CLIP);+ ut<sAVX2>(4096, 4096, 128, BTLA_DTYPE::S6_CLIP);+ ut<sAVX2>(4096, 4096, 32, BTLA_DTYPE::S5_CLIP, true);+ ut<sAVX2>(4096, 4096, 32, BTLA_DTYPE::S5_CLIP);+ ut<sAVX2>(4096, 4096, 128, BTLA_DTYPE::S5_CLIP);+ ut<sAVX2>(4096, 4096, 32, BTLA_DTYPE::S4_CLIP, true);+ ut<sAVX2>(4096, 4096, 32, BTLA_DTYPE::S4_CLIP);+ ut<sAVX2>(4096, 4096, 128, BTLA_DTYPE::S4_CLIP);+ ut<sAVX2>(127, 4096, 32, BTLA_DTYPE::S2_CLIP);+ ut<sAVX2>(127, 4096, 32, BTLA_DTYPE::S2_CLIP, true);+ ut<sAVX2>(4096, 4096, 32, BTLA_DTYPE::S2_CLIP);+ ut<sAVX2>(4096, 4096, 128, BTLA_DTYPE::S2_CLIP);+ ut<sAVX2>(127, 4096, 32, BTLA_DTYPE::S3_CLIP);+ ut<sAVX2>(127, 4096, 32, BTLA_DTYPE::S3_CLIP, true);+ ut<sAVX2>(4096, 4096, 32, BTLA_DTYPE::S3_CLIP);+ ut<sAVX2>(127, 4096, 32, BTLA_DTYPE::S4_CLIP);+ ut<sAVX2>(127, 4096, 32, BTLA_DTYPE::S4_CLIP, true);++ CheckISA(AVX512F);+ ut<sAVX512F>(127, 4096, 32, BTLA_DTYPE::S3_CLIP);+ ut<sAVX512F>(4096, 4096, 32, BTLA_DTYPE::S3_CLIP);+ ut<sAVX512F>(4096, 4096, 128, BTLA_DTYPE::S3_CLIP);+ ut<sAVX512F>(127, 4096, 32, BTLA_DTYPE::S4_CLIP);+ ut<sAVX512F>(4096, 4096, 32, BTLA_DTYPE::S4_CLIP);+ ut<sAVX512F>(4096, 4096, 128, BTLA_DTYPE::S4_CLIP);+ }+ template <class GemmCore>+ void ut(int n, int k, int blocksize, BTLA_DTYPE QUANT_T, bool isAsym = false) {+ auto constexpr RuntimeISA = GemmCore::ISA;+ printf("%s DType %s %d: %d %d %d Asym:%d\n", __FUNCTION__, utils::bestla_dtype_str(QUANT_T), int(RuntimeISA), n, k,+ blocksize, isAsym);+ int ldb = n;+ utils::aligned_vector<float> raw(n * k);+ ut::fill_buffer_randn(raw.data(), raw.size(), -0.5f, 0.5f);+ using PrologueB = prologue_b::gemm::WeightKBlockNInteger<GemmCore>;+ auto ptr = PrologueB::createStorage(n, k, blocksize, QUANT_T, BTLA_DTYPE::F32, isAsym);+ avector<int8_t> buffer(ptr.mSize);+ ptr.assign(buffer.data());+ PrologueB::packWeight(n, k, raw.data(), ldb, &ptr, UT_Threading::get());+ avector<float> dequant(n * k, 0);+ PrologueB::unpackWeight(n, k, &ptr, dequant.data(), n, UT_Threading::get());+ ut::buffer_error(raw.data(), dequant.data(), dequant.size(), 0.01f);+ }+};+#ifdef BTLA_UT_PROLOGUE_B+// no proper threshold for this UT+// static UT_BlockQunatize_SN sUT_BlockQunatize_SN;+#endif++class UT_TransposeBlockQuantize_F4 {+ public:+ UT_TransposeBlockQuantize_F4() {+ UT_START();+ CheckISA(AVX512F);+ ut(4096, 4096, 32, BTLA_DTYPE::F4_BNB, BTLA_DTYPE::F32);+ ut(1024, 4096, 32, BTLA_DTYPE::F4_BNB, BTLA_DTYPE::F32);+ ut(4096, 1024, 32, BTLA_DTYPE::F4_BNB, BTLA_DTYPE::F32);+ ut(48, 32, 32, BTLA_DTYPE::F4_BNB, BTLA_DTYPE::F32);+ ut(32, 32, 32, BTLA_DTYPE::F4_BNB, BTLA_DTYPE::F32);+ ut(48, 32, 32, BTLA_DTYPE::F4_BNB, BTLA_DTYPE::F32);+ ut(48, 32, 32, BTLA_DTYPE::F4_NF4, BTLA_DTYPE::F32);+ ut(48, 32, 32, BTLA_DTYPE::F4_E2M1, BTLA_DTYPE::F32);+ ut(16, 15, 8, BTLA_DTYPE::F4_NF4, BTLA_DTYPE::DQ8_BNB);+ ut(48, 32, 16, BTLA_DTYPE::F4_NF4, BTLA_DTYPE::DQ8_BNB);+ ut(1024, 4096, 32, BTLA_DTYPE::F4_NF4, BTLA_DTYPE::DQ8_BNB);+ }++ void ut(int n, int k, int blocksize, BTLA_DTYPE F4_T, BTLA_DTYPE SCA_T) {+ printf("Test Case: %d %d %d\n", n, k, blocksize);+ int ldb = n;+ utils::aligned_vector<float> dequanRef(n * k);+ int kblk_num = utils::updiv(k, blocksize);+ utils::aligned_vector<float> scales(kblk_num * n);+ ut::fill_buffer_randn(scales.data(), scales.size(), 1.f, 5.f);+ ut::UT_vector_s8 quanW;+ quanW.resize(k * n);+ quanW.fill_rand(0, 16);+ for (int i = 0; i < k; i++) {+ for (int j = 0; j < n; j++) {+ if (i % blocksize == 0) {+ switch (F4_T) {+ case BTLA_DTYPE::F4_E2M1:+ quanW.data()[i * n + j] = 7; // make sure each block has maximum fp4e2m1 value(0b111) to quantize+ break;+ case BTLA_DTYPE::F4_BNB:+ quanW.data()[i * n + j] = 3; // make sure each block has maximum fp4bnb value(0b011) to quantize+ break;+ case BTLA_DTYPE::F4_NF4:+ quanW.data()[i * n + j] = 15; // make sure each block has maximum nf4 value(0b1111) to quantize+ break;+ default:+ break;+ }+ }+ }+ }+ for (int j = 0; j < k; j++) {+ for (int i = 0; i < n; i++) {+ switch (F4_T) {+ case BTLA_DTYPE::F4_E2M1:+ dequanRef[j + i * k] = kernel::ref::f4_dequantize<BTLA_DTYPE::F4_E2M1>(quanW.data()[j * ldb + i],+ scales[j / blocksize * n + i]);+ quanW.data()[j * ldb + i] =+ kernel::ref::f4_quantize<BTLA_DTYPE::F4_E2M1>(dequanRef[j + i * k] / scales[j / blocksize * n + i]);+ break;+ case BTLA_DTYPE::F4_BNB:+ dequanRef[j + i * k] = kernel::ref::f4_dequantize<BTLA_DTYPE::F4_BNB>(quanW.data()[j * ldb + i],+ scales[j / blocksize * n + i]);+ quanW.data()[j * ldb + i] =+ kernel::ref::f4_quantize<BTLA_DTYPE::F4_BNB>(dequanRef[j + i * k] / scales[j / blocksize * n + i]);+ break;+ case BTLA_DTYPE::F4_NF4:+ dequanRef[j + i * k] = kernel::ref::f4_dequantize<BTLA_DTYPE::F4_NF4>(quanW.data()[j * ldb + i],+ scales[j / blocksize * n + i]);+ quanW.data()[j * ldb + i] =+ kernel::ref::f4_quantize<BTLA_DTYPE::F4_NF4>(dequanRef[j + i * k] / scales[j / blocksize * n + i]);+ break;+ default:+ break;+ }+ }+ }++ auto constexpr RuntimeISA = BTLA_ISA::AVX512F;+ using PrologueB = prologue_b::gemm::WeightKBlockNFloat<gemm::SCoreRowNAvx512f<48, 8>>;+ auto packedW = PrologueB::createStorage(n, k, blocksize, F4_T, SCA_T);+ auto packedW1 = PrologueB::createStorage(n, k, blocksize, F4_T, SCA_T);+ avector<int8_t> buf(packedW.mSize), buf1(packedW1.mSize);+ packedW.assign(buf.data());+ packedW1.assign(buf1.data());+ PrologueB::packTransposeWeight(n, k, dequanRef.data(), k, &packedW, UT_Threading::get());+ PrologueB::packQWeight(n, k, quanW.data(), ldb, scales.data(), nullptr, &packedW1, UT_Threading::get());+ avector<float> dequant(n * k);+ PrologueB::unpackTransposeWeight(n, k, &packedW1, dequant.data(), k, UT_Threading::get());+ if (SCA_T != BTLA_DTYPE::DQ8_BNB) {+ ut::buffer_error(packedW.SPtr<float>(), packedW1.SPtr<float>(), packedW1.CSize());+ ut::buffer_error(dequanRef.data(), dequant.data(), dequant.size());+ } else {+ ut::buffer_error(packedW.SPtr<int8_t>(), packedW1.SPtr<int8_t>(), packedW1.CSize());+ ut::buffer_error(dequanRef.data(), dequant.data(), dequant.size(), 0.1f);+ }+ ut::buffer_error(packedW.WPtr<int8_t>(), packedW1.WPtr<int8_t>(), packedW1.mQBuf.size<int8_t>());+ }+};+#ifdef BTLA_UT_PROLOGUE_B+static UT_TransposeBlockQuantize_F4 sUT_TransposeBlockQuantize_F4;+#endif++class UT_StorageMemCheck {+ public:+ UT_StorageMemCheck() {+ UT_START();+ ut_npack(1024, 768);+ ut_s4(4096, 4096, 128, BTLA_DTYPE::S4_CLIP);+ ut_f4(4096, 4096, 32, BTLA_DTYPE::F4_BNB);+ ut_f4(4096, 4096, 32, BTLA_DTYPE::F4_E2M1);+ }+ void ut_npack(int n, int k) {+ printf("Test C type Case: %d %d\n", n, k);+ int ldb = n;+ using GemmCore = gemm::SCoreRowNAvx512f<48, 8>;+ using PrologueB = prologue_b::gemm::WeightNPack<GemmCore>;++ auto stor = PrologueB::create_storage(n, k, storage::gemm::Layout::T);+ std::vector<float> src(n * k);+ fill_buffer_randn(src.data(), src.size(), -0.5f, 0.5f);+ stor.set_buffers((int8_t*)src.data());++ std::vector<int8_t> misc_mem(stor.misc_size_ + stor.buf_size_);++ stor.serialize(misc_mem.data());+ stor.copy_buffers(misc_mem.data() + stor.misc_size_);++ auto ptr = storage::CollectionParser::parse_buffer(misc_mem.data());+ if (ptr) {+ if (ptr->prologue_id_ == BTLA_PROLOGUEB_IDS::WeightPack) {+ auto wptr = (storage::gemm::StorageWeight*)ptr;+ wptr->set_buffers(misc_mem.data() + wptr->misc_size_);+ buffer_error(wptr->ptr(), (int8_t*)src.data(), wptr->buf_size_);+ }+ delete ptr;+ }+ }++ void ut_s4(int n, int k, int blocksize, BTLA_DTYPE qtype, bool asym = false) {+ printf("Test C type Case: %d %d %d %s\n", n, k, blocksize, asym ? "asym" : "sym");+ int ldb = n;+ int kblk_num = utils::updiv(k, blocksize);+ using GemmCore = gemm::SCoreRowNAvx512f<48, 8>;+ using PrologueB = prologue_b::gemm::WeightKBlockNInteger<GemmCore>;++ auto packedW = PrologueB::createStorage(n, k, blocksize, qtype, bestla_dtype<float>, asym);+ avector<int8_t> buf0(packedW.mSize), buf1(packedW.mSize);+ packedW.assign(buf0.data());+ storage::gemm::StorageWeightKBlockNInteger tmp(GemmCore::ID);+ tmp.deserialize(buf0.data());+ tmp.serialize(buf1.data());+ buffer_error(buf0.data(), buf1.data(), buf0.size());+ }++ void ut_f4(int n, int k, int blocksize, BTLA_DTYPE qtype) {+ printf("Test C type Case: %d %d %d\n", n, k, blocksize);+ int ldb = n;+ int kblk_num = utils::updiv(k, blocksize);+ using GemmCore = gemm::HCoreRowNAmxbf16<64, 16>;+ using PrologueB = prologue_b::gemm::WeightKBlockNFloat<GemmCore>;++ auto packedW = PrologueB::createStorage(n, k, blocksize, qtype, bestla_dtype<float>);+ avector<int8_t> buf0(packedW.mSize), buf1(packedW.mSize);+ packedW.assign(buf0.data());+ storage::gemm::StorageWeightKBlockNFloat tmp(GemmCore::ID);+ tmp.deserialize(buf0.data());+ tmp.serialize(buf1.data());+ buffer_error(buf0.data(), buf1.data(), buf0.size());+ }+};+#ifdef BTLA_UT_PROLOGUE_B+static UT_StorageMemCheck sUT_StorageMemCheck;+#endif++class UT_WeightNPack {+ public:+ UT_WeightNPack() {+ UT_START();+ ut_npack<gemm::SCoreRowNAvx2<24, 4>>(1024, 768);+ }+ template <class _Gc>+ void ut_npack(int n, int k) {+ printf("Test C type Case: %d %d\n", n, k);+ int ldb = n;+ using PrologueB = prologue_b::gemm::WeightNPack<_Gc>;+ using T = typename _Gc::BType;++ std::vector<T> src(n * k), vdst(k * n);+ fill_buffer_randn(src.data(), src.size(), T(-0.5f), T(0.5f));++ storage::gemm::StorageWeight raw;+ raw.set_plain_data(n, k, k, utils::bestla_dtype<T>, src.data());+ auto stor = PrologueB::create_storage(n, k, storage::gemm::Layout::NPack);+ std::vector<int8_t> tmp(stor.buf_size_);+ stor.set_buffers(tmp.data());+ PrologueB::convert(raw, stor, UT_Threading::get());++ storage::gemm::StorageWeight tar = raw;+ tar.set_buffers((int8_t*)vdst.data());+ PrologueB::convert(stor, tar, UT_Threading::get());+ buffer_error(src.data(), vdst.data(), src.size());+ }+};+#ifdef BTLA_UT_PROLOGUE_B+static UT_WeightNPack sUT_WeightNPack;+#endif++class UT_WeightNInt {+ public:+ UT_WeightNInt() {+ UT_START();+ ut_qdq<gemm::SCoreRowNAvx2<24, 4>>(1024, 768, 32, BTLA_DTYPE::S4_CLIP, false);+ ut_qdq<gemm::SCoreRowNAvx2<24, 4>>(1024, 768, 32, BTLA_DTYPE::S4_CLIP, true);+ ut_qdq<gemm::SCoreRowNAvx2<24, 4>>(1024, 768, -1, BTLA_DTYPE::S8, false);+ ut_convert<gemm::SCoreRowNAvx2<24, 4>>(1024, 768, 32, BTLA_DTYPE::S4_CLIP, false);+ }+ template <class _Gc>+ void ut_qdq(int n, int k, int block, BTLA_DTYPE qdtype, bool is_asym) {+ printf("Test C type Case: %d %d\n", n, k);+ int ldb = n;+ using PrologueB = prologue_b::gemm::WeightKBlockNInt<_Gc>;+ using T = typename _Gc::BType;++ std::vector<T> src(n * k), vdst(k * n);+ fill_buffer_randn(src.data(), src.size(), T(-0.5f), T(0.5f));++ storage::gemm::StorageWeightNInt raw, deq;+ raw.set_plain_data(n, k, k, utils::bestla_dtype<T>, src.data());+ deq.set_plain_data(n, k, k, utils::bestla_dtype<T>, vdst.data());+ auto stor = PrologueB::create_storage(n, k, block, qdtype, BTLA_DTYPE::F32, is_asym);+ std::vector<int8_t> tmp(stor.buf_size_);+ stor.set_buffers(tmp.data());+ PrologueB::convert(raw, stor, UT_Threading::get());+ PrologueB::convert(stor, deq, UT_Threading::get());+ buffer_error(src.data(), vdst.data(), src.size(), 0.1f);+ }++ template <class _Gc>+ void ut_convert(int n, int k, int block, BTLA_DTYPE qdtype, bool is_asym) {+ printf("Test C type Case: %d %d\n", n, k);+ int ldb = n;+ using PrologueB = prologue_b::gemm::WeightKBlockNInt<_Gc>;+ using T = typename _Gc::BType;++ std::vector<T> src(n * k), vdst(k * n);+ fill_buffer_randn(src.data(), src.size(), T(-0.5f), T(0.5f));+ std::vector<T> srcNT(k * n);+ prologue_b::gemm::transpose(n, k, (int8_t*)src.data(), utils::bestla_dtype<T>, k, (int8_t*)srcNT.data(), n,+ UT_Threading::get());+ storage::gemm::StorageWeightNInt raw, t;+ raw.set_plain_data(n, k, k, utils::bestla_dtype<T>, src.data());+ auto stor = PrologueB::create_storage(n, k, block, qdtype, BTLA_DTYPE::F32, is_asym);+ std::vector<int8_t> tmp(stor.buf_size_);+ stor.set_buffers(tmp.data());+ t.t_from(stor);+ std::vector<int8_t> tmp2(t.buf_size_);+ t.set_buffers(tmp2.data());+ PrologueB::convert(raw, stor, UT_Threading::get());+ PrologueB::convert(stor, t, UT_Threading::get());+ storage::gemm::StorageWeightNInt stor2 = stor;+ std::vector<int8_t> tmp3(stor2.buf_size_);+ stor2.set_buffers(tmp3.data());+ PrologueB::convert(t, stor2, UT_Threading::get());+ std::vector<int8_t> upk0(n * k), upk1(k * n);+ PrologueB::template unpack_weight<int8_t>(n, k, stor, upk0.data(), n, UT_Threading::get());+ PrologueB::decompress_weight(n, k, t, upk1.data(), n, UT_Threading::get());+ buffer_error(tmp.data(), tmp3.data(), tmp.size(), int8_t(0));+ }+};+#ifdef BTLA_UT_PROLOGUE_B+static UT_WeightNInt sUT_WeightNInt;+#endif++class UT_ShuffleIndices {+ public:+ UT_ShuffleIndices() {+ UT_START();+ CheckISA(AVX2);+ // ut_file();+ ut_s4(4096, 4096, 128, BTLA_DTYPE::S4_CLIP);+ }++ void ut_s4(int n, int k, int blocksize, BTLA_DTYPE qtype, bool asym = false) {+ printf("Test C type Case: %d %d %d %s\n", n, k, blocksize, asym ? "asym" : "sym");+ int ldb = n;+ int kblk_num = utils::updiv(k, blocksize);+ using GemmCore = gemm::SCoreRowNAvx2<24, 4>;+ using PrologueB = prologue_b::gemm::WeightKBlockNInteger<GemmCore>;+ auto packedW = PrologueB::createStorage(n, k, blocksize, qtype, bestla_dtype<float>, asym);+ PrologueB::enableShuffle(&packedW);+ avector<int> groupindices(k, 0);+ auto groupsize = utils::updiv(k, blocksize);+ avector<int> reflut(k, 0);+ for (size_t i = 0; i < k; i++) {+ groupindices[i] = i % groupsize;+ auto offset = i / groupsize;+ reflut[groupindices[i] * blocksize + offset] = static_cast<int>(i);+ }+ avector<int8_t> buf0(packedW.mSize), buf1(packedW.mSize);+ packedW.assign(buf0.data());+ PrologueB::setShuffleIndices(groupindices.data(), &packedW, UT_Threading::get());+ buffer_error(reflut.data(), packedW.ShfIndice(), reflut.size());++ storage::gemm::StorageWeightKBlockNInteger tmp(GemmCore::ID);+ tmp.deserialize(buf0.data());+ tmp.serialize(buf1.data());+ buffer_error(buf0.data(), buf1.data(), buf0.size());+ }+};+#ifdef BTLA_UT_PROLOGUE_B+static UT_ShuffleIndices sUT_ShuffleIndices;+#endif++class UT_CompFp32 {+ public:+ UT_CompFp32() {+ UT_START();+ ut_int_new<sAVX2>(1, 4096, 4096, 16, BTLA_DTYPE::S4_CLIP, BTLA_DTYPE::F16, true);+ ut_int_new<sAVX512F>(1, 4096, 4096, 16, BTLA_DTYPE::S4_CLIP, BTLA_DTYPE::F16, true);+ ut_s4_full();+ ut_new_type(BTLA_DTYPE::S1_CLIP);+ ut_new_type(BTLA_DTYPE::S2_CLIP);+ ut_new_type(BTLA_DTYPE::S3_CLIP);+ ut_new_type(BTLA_DTYPE::S5_CLIP);+ ut_new_type(BTLA_DTYPE::S6_CLIP);+ ut_new_type(BTLA_DTYPE::S7_CLIP);+ ut_new_type(BTLA_DTYPE::S8);++ ut_f4();+ ut_f8();+ }++ void ut_new_type(BTLA_DTYPE qtype) {+ GetCPUDevice();+ if (_cd->AVX2()) {+ ut_int<sAVX2, prologue_b::gemm::WeightKBlockNInteger>(1, 4096, 4096, 16, qtype, BTLA_DTYPE::F16, true);+ ut_int<sAVX2, prologue_b::gemm::WeightKBlockNInteger>(1, 4096, 4096, 16, qtype, BTLA_DTYPE::BF16, true);+ ut_int<sAVX2, prologue_b::gemm::WeightKBlockNInteger>(4, 4096, 4096, 32, qtype, BTLA_DTYPE::F32, true);+ ut_int<sAVX2, prologue_b::gemm::WeightKBlockNInteger>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F16, true);+ ut_int<sAVX2, prologue_b::gemm::WeightKBlockNInteger>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ ut_int<sAVX2, prologue_b::gemm::WeightKBlockNInteger>(8, 4096, 4096, -1, qtype, BTLA_DTYPE::BF16, false);+ }+ if (_cd->AVX512F()) {+ ut_int<sAVX512F, prologue_b::gemm::WeightKBlockNInteger>(1, 4096, 4096, 16, qtype, BTLA_DTYPE::BF16, true);+ ut_int<sAVX512F, prologue_b::gemm::WeightKBlockNInteger>(4, 4096, 4096, 32, qtype, BTLA_DTYPE::F32, true);+ ut_int<sAVX512F, prologue_b::gemm::WeightKBlockNInteger>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ ut_int<sAVX512F, prologue_b::gemm::WeightKBlockNInteger>(8, 4096, 4096, -1, qtype, BTLA_DTYPE::BF16, false);+ }+ }++ void ut_f8() {+ CheckISA(AVX2);+ ut<sAVX2, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F8_E4M3, BTLA_DTYPE::F8_E8M0);+ ut<sAVX2, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F8_E4M3, BTLA_DTYPE::F32);+ ut<sAVX2, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F8_E5M2, BTLA_DTYPE::F8_E8M0);+ ut<sAVX2, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F8_E5M2, BTLA_DTYPE::F32);+ CheckISA(AVX512F);+ ut<sAVX512F, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F8_E4M3, BTLA_DTYPE::F8_E8M0);+ ut<sAVX512F, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F8_E4M3, BTLA_DTYPE::F32);+ ut<sAVX512F, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F8_E5M2, BTLA_DTYPE::F8_E8M0);+ ut<sAVX512F, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F8_E5M2, BTLA_DTYPE::F32);+ }++ void ut_s4_full() {+ BTLA_DTYPE qtype = BTLA_DTYPE::S4_CLIP;+ CheckISA(AVX2);+ ut_int<sAVX2, prologue_b::gemm::WeightKBlockNInteger>(1, 4096, 4096, 16, qtype, BTLA_DTYPE::F16, true);+ ut_int<sAVX2, prologue_b::gemm::WeightKBlockNInteger>(1, 4096, 4096, 16, qtype, BTLA_DTYPE::BF16, true);+ ut_int<sAVX2, prologue_b::gemm::WeightKBlockNInteger>(4, 4096, 4096, 32, qtype, BTLA_DTYPE::F32, true);+ ut_int<sAVX2, prologue_b::gemm::WeightKBlockNInteger>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F16, true);+ ut_int<sAVX2, prologue_b::gemm::WeightKBlockNInteger>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ ut_int<sAVX2, prologue_b::gemm::WeightKBlockNInteger>(8, 4096, 4096, -1, qtype, BTLA_DTYPE::BF16, false);++ CheckISA(AVX512F);+ ut_int<sAVX512F, prologue_b::gemm::WeightKBlockNInteger>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F16, false);+ ut_int<sAVX512F, prologue_b::gemm::WeightKBlockNInteger>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F32, false);+ ut_int<sAVX512F, prologue_b::gemm::WeightKBlockNInteger>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F32, true);+ ut_int<sAVX512F, prologue_b::gemm::WeightKBlockNInteger>(2, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, false);+ ut_int<sAVX512F, prologue_b::gemm::WeightKBlockNInteger>(2, 4096, 4096, -1, qtype, BTLA_DTYPE::F32, false);+ ut_int<sAVX512F, prologue_b::gemm::WeightKBlockNInteger>(8, 4096, 4096, 32, qtype, BTLA_DTYPE::BF16, false);+ ut_int<sAVX512F, prologue_b::gemm::WeightKBlockNInteger>(8, 4096, 4096, 32, qtype, BTLA_DTYPE::F16, false);+ }++ void ut_f4() {+ CheckISA(AVX2);+ ut<sAVX2, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F4_BNB, BTLA_DTYPE::F32);+ ut<sAVX2, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, -1, BTLA_DTYPE::F4_BNB, BTLA_DTYPE::F32);+ ut<sAVX2, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F4_E2M1, BTLA_DTYPE::F32);+ ut<sAVX2, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, -1, BTLA_DTYPE::F4_E2M1, BTLA_DTYPE::F32);+ ut<sAVX2, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F4_NF4, BTLA_DTYPE::F32);+ ut<sAVX2, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, -1, BTLA_DTYPE::F4_NF4, BTLA_DTYPE::F32);+ ut<sAVX2, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F4_BNB, BTLA_DTYPE::BF16);+ ut<sAVX2, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F4_E2M1, BTLA_DTYPE::BF16);+ ut<sAVX2, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F4_NF4, BTLA_DTYPE::BF16);++ CheckISA(AVX512F);+ ut<sAVX512F, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F4_BNB, BTLA_DTYPE::F32);+ ut<sAVX512F, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, -1, BTLA_DTYPE::F4_BNB, BTLA_DTYPE::F32);+ ut<sAVX512F, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F4_E2M1, BTLA_DTYPE::F32);+ ut<sAVX512F, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, -1, BTLA_DTYPE::F4_E2M1, BTLA_DTYPE::F32);+ ut<sAVX512F, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F4_NF4, BTLA_DTYPE::F32);+ ut<sAVX512F, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, -1, BTLA_DTYPE::F4_NF4, BTLA_DTYPE::F32);+ ut<sAVX512F, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F4_BNB, BTLA_DTYPE::BF16);+ ut<sAVX512F, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F4_E2M1, BTLA_DTYPE::BF16);+ ut<sAVX512F, prologue_b::gemm::WeightKBlockNFloat>(2, 4096, 4096, 32, BTLA_DTYPE::F4_NF4, BTLA_DTYPE::BF16);+ }++ template <class GemmCore_T>+ void ut_int_new(int m, int n, int k, int blocksize, BTLA_DTYPE qtype, BTLA_DTYPE stype, bool isAsym) {+ printf("Test Case %s: %d %d %d-%d type:%s core:%s scaletype:%s Asym:%d\n", __FUNCTION__, m, n, k, blocksize,+ bestla_dtype_str(qtype), gemm::CoreAttr::to_str(GemmCore_T::ID), bestla_dtype_str(stype), isAsym);+ auto constexpr ISA = GemmCore_T::ISA;+ using Launcher =+ wrapper::gemm::LauncherBaseV2<GemmCore_T, prologue_a::gemm::ActivationKBlockBaseF32,+ prologue_b::gemm::WeightKBlockNInt, epilogue::gemm::AccumulatorWriteBackFp32>;+ using Parallel = parallel::gemm::SchedulerBase<GemmCore_T>;+ blocksize = blocksize == -1 ? k : blocksize;+ auto packedw = Launcher::PrologueB::create_storage(n, k, blocksize, qtype, stype, isAsym);+ utils::avector<int8_t> buffer(packedw.buf_size_);+ packedw.set_buffers(buffer.data());+ auto reduceA = Launcher::PrologueA::createStorage(m, k, blocksize);+ utils::avector<int8_t> bufferA(reduceA.mSize);+ reduceA.assign(bufferA.data());+ avector<float> matBf32(k * n), matAf32(m * k), matC(m * n), refC(m * n), refCupk(m * n);+ fill_buffer_randn(matBf32.data(), matBf32.size(), -0.5f, 0.5f);+ fill_buffer_randn(matAf32.data(), matAf32.size(), -0.5f, 0.5f);+ storage::gemm::StorageWeightNInt rawB, dstB;+ rawB.set_nt_data(n, k, k, utils::bestla_dtype<float>, matBf32.data());+ Launcher::PrologueB::convert(rawB, packedw, UT_Threading::get());+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), matBf32.data(), refC.data(), k, n, n);+ Launcher::PrologueB::convert(packedw, rawB, UT_Threading::get());+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), matBf32.data(), refCupk.data(), k, n, n);++ Launcher::PrologueA::reduce({matAf32.data(), k, &reduceA}, m, k, blocksize, UT_Threading::get());+ utils::GemmProblem gp(1, m, n, k, blocksize);+ typename Launcher::Param args{gp, {matAf32.data(), k, &reduceA}, {&packedw}, {matC.data(), n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ auto err = get_ut_err(qtype);+ auto dbits = bestla_dtype_bits(qtype);+ auto type = bestla_dtype_type(qtype);+ auto constexpr dtype_int = bestla_dtype_type(BTLA_DTYPE::TypeInt);+ buffer_error(refC.data(), matC.data(), refC.size(), err);+ buffer_error(refCupk.data(), matC.data(), refCupk.size(), 0.001f);+ }++ template <class GemmCore_T, template <class _T> class Wei>+ void ut_int(int m, int n, int k, int blocksize, BTLA_DTYPE qtype, BTLA_DTYPE stype, bool isAsym) {+ printf("Test Case %s: %d %d %d-%d type:%s core:%s scaletype:%s Asym:%d\n", __FUNCTION__, m, n, k, blocksize,+ bestla_dtype_str(qtype), gemm::CoreAttr::to_str(GemmCore_T::ID), bestla_dtype_str(stype), isAsym);+ auto constexpr ISA = GemmCore_T::ISA;+ using Launcher =+ wrapper::gemm::LauncherBase<GemmCore_T, prologue_a::gemm::ActivationKBlockBaseF32,+ prologue_b::gemm::WeightKBlockNInteger, epilogue::gemm::AccumulatorWriteBackFp32>;+ using Parallel = parallel::gemm::SchedulerBase<GemmCore_T>;++ blocksize = blocksize == -1 ? k : blocksize;+ using WType = typename Launcher::PrologueB::StorageWeight;+ WType packedw(0);+ if constexpr (std::is_same_v<Wei<GemmCore_T>, prologue_b::gemm::WeightKBlockNInteger<GemmCore_T>>) {+ packedw = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, stype, isAsym);+ } else if constexpr (std::is_same_v<Wei<GemmCore_T>, prologue_b::gemm::WeightKBlockNFloat<GemmCore_T>>) {+ packedw = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, stype);+ }++ utils::avector<int8_t> buffer(packedw.mSize);+ packedw.assign(buffer.data());+ auto reduceA = Launcher::PrologueA::createStorage(m, k, blocksize);+ utils::avector<int8_t> bufferA(packedw.mSize);+ reduceA.assign(bufferA.data());+ avector<float> matBf32(k * n), matAf32(m * k), matC(m * n), refC(m * n), refCupk(m * n);+ fill_buffer_randn(matBf32.data(), matBf32.size(), -0.5f, 0.5f);+ fill_buffer_randn(matAf32.data(), matAf32.size(), -0.5f, 0.5f);+ Launcher::PrologueB::packWeight(n, k, matBf32.data(), n, &packedw, UT_Threading::get());+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), matBf32.data(), refC.data(), k, n, n);+ Launcher::PrologueB::unpackWeight(n, k, &packedw, matBf32.data(), n, UT_Threading::get());+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), matBf32.data(), refCupk.data(), k, n, n);++ Launcher::PrologueA::reduce({matAf32.data(), k, &reduceA}, m, k, blocksize, UT_Threading::get());+ utils::GemmProblem gp(1, m, n, k, blocksize);+ typename Launcher::Param args{gp, {matAf32.data(), k, &reduceA}, {&packedw}, {matC.data(), n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ auto err = get_ut_err(qtype);+ auto dbits = bestla_dtype_bits(qtype);+ auto type = bestla_dtype_type(qtype);+ auto constexpr dtype_int = bestla_dtype_type(BTLA_DTYPE::TypeInt);+ buffer_error(refC.data(), matC.data(), refC.size(), err);+ buffer_error(refCupk.data(), matC.data(), refCupk.size(), 0.001f);+ }++ template <class GemmCore_T, template <class _T> class Wei>+ void ut(int m, int n, int k, int blocksize, BTLA_DTYPE qtype, BTLA_DTYPE stype) {+ printf("Test Case %s: %d %d %d-%d type:%s core:%s scaletype:%s\n", __FUNCTION__, m, n, k, blocksize,+ bestla_dtype_str(qtype), gemm::CoreAttr::to_str(GemmCore_T::ID), bestla_dtype_str(stype));+ auto constexpr ISA = GemmCore_T::ISA;+ using Launcher = wrapper::gemm::LauncherBase<GemmCore_T, prologue_a::gemm::ActivationBase, Wei,+ epilogue::gemm::AccumulatorWriteBackFp32>;+ using Parallel = parallel::gemm::SchedulerBase<GemmCore_T>;++ blocksize = blocksize == -1 ? k : blocksize;+ using WType = typename Wei<GemmCore_T>::StorageWeight;+ WType packedw(0);+ packedw = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, stype);+ utils::avector<int8_t> buffer(packedw.mSize);+ packedw.assign(buffer.data());+ avector<float> matBf32(k * n), matAf32(m * k), matC(m * n), refC(m * n), refCupk(m * n);+ fill_buffer_randn(matBf32.data(), matBf32.size(), -0.5f, 0.5f);+ fill_buffer_randn(matAf32.data(), matAf32.size(), -0.5f, 0.5f);+ Launcher::PrologueB::packWeight(n, k, matBf32.data(), n, &packedw, UT_Threading::get());+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), matBf32.data(), refC.data(), k, n, n);+ Launcher::PrologueB::unpackWeight(n, k, &packedw, matBf32.data(), n, UT_Threading::get());+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), matBf32.data(), refCupk.data(), k, n, n);+ GemmProblem gp(1, m, n, k, blocksize);+ typename Launcher::Param args{gp, {matAf32.data(), k}, {&packedw}, {matC.data(), n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ auto err = get_ut_err(qtype);+ buffer_error(refC.data(), matC.data(), refC.size(), err);+ buffer_error(refCupk.data(), matC.data(), refCupk.size(), 0.001f);+ }+};+#ifdef BTLA_UT_PROLOGUE_B+static UT_CompFp32 sUT_CompFp32;+#endif++class UT_CompInt8 {+ public:+ UT_CompInt8() {+ UT_START();+ ut_newkblock_v2<gemm::KblockGemmAvx512vnni<48, 4>>(1, 4096, 4096, 32, BTLA_DTYPE::S4_CLIP, BTLA_DTYPE::F16, true);+ ut_s4_full();+ ut_new_dtype(BTLA_DTYPE::S1_CLIP);+ ut_new_dtype(BTLA_DTYPE::S2_CLIP);+ ut_new_dtype(BTLA_DTYPE::S3_CLIP);+ ut_new_dtype(BTLA_DTYPE::S5_CLIP);+ ut_new_dtype(BTLA_DTYPE::S6_CLIP);+ ut_new_dtype(BTLA_DTYPE::S7_CLIP);+ }++ void ut_new_dtype(BTLA_DTYPE qtype) {+ GetCPUDevice();+ if (_cd->AVX2()) {+ ut_newkblock<gemm::KblockGemmAvx2vnni<24, 2>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAvx2vnni<24, 2>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F16, true);+ ut_newkblock<gemm::KblockGemmAvx2vnni<24, 2>>(4, 4096, 4096, 64, qtype, BTLA_DTYPE::BF16);+ ut_newkblock<gemm::KblockGemmAvx2vnni<24, 2>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ }+ if (_cd->AVX_VNNI()) {+ ut_newkblock<gemm::KblockGemmAvxvnni<24, 2>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAvxvnni<24, 2>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F16, true);+ ut_newkblock<gemm::KblockGemmAvxvnni<24, 2>>(4, 4096, 4096, 64, qtype, BTLA_DTYPE::BF16);+ ut_newkblock<gemm::KblockGemmAvxvnni<24, 2>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ }+ if (_cd->AVX512_VNNI()) {+ ut_newkblock<gemm::KblockGemmAvx512vnni<48, 4>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAvx512vnni<48, 4>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F16, true);+ ut_newkblock<gemm::KblockGemmAvx512vnni<48, 4>>(4, 4096, 4096, 64, qtype, BTLA_DTYPE::BF16);+ ut_newkblock<gemm::KblockGemmAvx512vnni<48, 4>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ }+ if (_cd->AVX512BW()) {+ ut_newkblock<gemm::KblockGemmAvx512bw<48, 4>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAvx512bw<48, 4>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F16, true);+ ut_newkblock<gemm::KblockGemmAvx512bw<48, 4>>(4, 4096, 4096, 64, qtype, BTLA_DTYPE::BF16);+ ut_newkblock<gemm::KblockGemmAvx512bw<48, 4>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ }+ if (_cd->AMX_INT8()) {+ ut_newkblock<gemm::KblockGemmAmxint8<64, 16>>(1, 4096, 4096, 64, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAmxint8<64, 16>>(1, 4096, 4096, 64, qtype, BTLA_DTYPE::F16, true);+ ut_newkblock<gemm::KblockGemmAmxint8<64, 16>>(8, 4096, 4096, 64, qtype, BTLA_DTYPE::BF16);+ ut_newkblock<gemm::KblockGemmAmxint8<64, 16>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ }+ }++ void ut_s4_full() {+ GetCPUDevice();+ auto qtype = BTLA_DTYPE::S4_CLIP;+ if (_cd->AVX2()) {+ ut_newkblock<gemm::KblockGemmAvx2vnni<24, 2>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAvx2vnni<24, 2>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F16);+ ut_newkblock<gemm::KblockGemmAvx2vnni<24, 2>>(4, 4096, 4096, 64, qtype, BTLA_DTYPE::BF16);+ ut_newkblock<gemm::KblockGemmAvx2vnni<24, 2>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F16, true);+ ut_newkblock<gemm::KblockGemmAvx2vnni<24, 2>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAvx2vnni<24, 2>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::DQ8_BNB);+ ut_newkblock_pc<gemm::ICoreRowNAvx2vnni<24, 4>>(1, 4096, 4096, 4096, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock_pc<gemm::ICoreRowNAvx2vnni<24, 4>>(8, 4096, 4096, 4096, qtype, BTLA_DTYPE::F32, true);++ ut_newkblock<gemm::KblockGemmAvx2vnniSS<24, 1>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAvx2vnniSS<24, 1>>(4, 4096, 4096, 64, qtype, BTLA_DTYPE::BF16);+ ut_newkblock<gemm::KblockGemmAvx2vnniSS<24, 1>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAvx2vnniSS<24, 1>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::DQ8_BNB);+ ut_newkblock_pc<gemm::ICoreRowNAvx2vnniSS<24, 1>>(1, 4096, 4096, 4096, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock_pc<gemm::ICoreRowNAvx2vnniSS<24, 1>>(8, 4096, 4096, 4096, qtype, BTLA_DTYPE::F32, true);+ }+ if (_cd->AVX_VNNI()) {+ ut_newkblock<gemm::KblockGemmAvxvnni<24, 2>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAvxvnni<24, 2>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F16, true);+ ut_newkblock<gemm::KblockGemmAvxvnni<24, 2>>(4, 4096, 4096, 64, qtype, BTLA_DTYPE::BF16);+ ut_newkblock<gemm::KblockGemmAvxvnni<24, 2>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAvxvnni<24, 2>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::DQ8_BNB);+ ut_newkblock<gemm::KblockGemmAvxvnni<24, 2>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, false);+ ut_newkblock<gemm::KblockGemmAvxvnni<24, 2>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock_pc<gemm::ICoreRowNAvxvnni<24, 4>>(1, 4096, 4096, 4096, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock_pc<gemm::ICoreRowNAvxvnni<24, 4>>(8, 4096, 4096, 4096, qtype, BTLA_DTYPE::F32, true);++ ut_newkblock<gemm::KblockGemmAvxvnniSS<24, 2>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAvxvnniSS<24, 2>>(4, 4096, 4096, 64, qtype, BTLA_DTYPE::BF16);+ ut_newkblock<gemm::KblockGemmAvxvnniSS<24, 2>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAvxvnniSS<24, 2>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::DQ8_BNB);+ ut_newkblock_pc<gemm::ICoreRowNAvxvnniSS<24, 4>>(1, 4096, 4096, 4096, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock_pc<gemm::ICoreRowNAvxvnniSS<24, 4>>(8, 4096, 4096, 4096, qtype, BTLA_DTYPE::F32, true);+ }++ if (_cd->AVX512_VNNI()) {+ ut_newkblock<gemm::KblockGemmAvx512vnni<48, 4>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAvx512vnni<48, 4>>(1, 4096, 4096, 64, qtype, BTLA_DTYPE::F16);+ ut_newkblock<gemm::KblockGemmAvx512vnni<48, 4>>(4, 4096, 4096, 64, qtype, BTLA_DTYPE::BF16);+ ut_newkblock<gemm::KblockGemmAvx512vnni<48, 4>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, false);+ ut_newkblock<gemm::KblockGemmAvx512vnni<48, 4>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAvx512vnni<48, 4>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, false);+ ut_newkblock<gemm::KblockGemmAvx512vnni<48, 4>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::DQ8_BNB);+ ut_newkblock_pc<gemm::ICoreRowNAvx512vnni<48, 8>>(1, 4096, 4096, 4096, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock_pc<gemm::ICoreRowNAvx512vnni<48, 8>>(8, 4096, 4096, 4096, qtype, BTLA_DTYPE::F32, true);+ }++ if (_cd->AVX512BW()) {+ ut_newkblock<gemm::KblockGemmAvx512bw<48, 4>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAvx512bw<48, 4>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::F16, true);+ ut_newkblock<gemm::KblockGemmAvx512bw<48, 4>>(4, 4096, 4096, 64, qtype, BTLA_DTYPE::BF16);+ ut_newkblock<gemm::KblockGemmAvx512bw<48, 4>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAvx512bw<48, 4>>(1, 4096, 4096, 32, qtype, BTLA_DTYPE::DQ8_BNB);+ ut_newkblock_pc<gemm::ICoreRowNAvx512bw<48, 8>>(1, 4096, 4096, 4096, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock_pc<gemm::ICoreRowNAvx512bw<48, 8>>(8, 4096, 4096, 4096, qtype, BTLA_DTYPE::F32, true);+ }++ if (_cd->AMX_INT8()) {+ ut_newkblock<gemm::KblockGemmAmxint8SS<64, 16>>(1, 4096, 4096, 64, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAmxint8SS<64, 16>>(1, 4096, 4096, 64, qtype, BTLA_DTYPE::F16);+ ut_newkblock<gemm::KblockGemmAmxint8SS<64, 16>>(8, 4096, 4096, 64, qtype, BTLA_DTYPE::BF16);+ ut_newkblock<gemm::KblockGemmAmxint8SS<64, 16>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAmxint8SS<64, 16>>(1, 4096, 4096, 128, qtype, BTLA_DTYPE::DQ8_BNB);+ ut_newkblock_pc<gemm::ICoreRowNAmxint8SS<64, 16>>(1, 4096, 4096, 4096, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock_pc<gemm::ICoreRowNAmxint8SS<64, 16>>(8, 4096, 4096, 4096, qtype, BTLA_DTYPE::F32, true);++ ut_newkblock<gemm::KblockGemmAmxint8<64, 16>>(1, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAmxint8<64, 16>>(1, 4096, 4096, 64, qtype, BTLA_DTYPE::F32);+ ut_newkblock<gemm::KblockGemmAmxint8<64, 16>>(8, 4096, 4096, 128, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock<gemm::KblockGemmAmxint8<64, 16>>(128, 4096, 4096, 128, qtype, BTLA_DTYPE::DQ8_BNB);+ ut_newkblock_pc<gemm::ICoreRowNAmxint8<64, 16>>(1, 4096, 4096, 4096, qtype, BTLA_DTYPE::F32, true);+ ut_newkblock_pc<gemm::ICoreRowNAmxint8<64, 16>>(8, 4096, 4096, 4096, qtype, BTLA_DTYPE::F32, true);+ }+ }++ using PcWriteBack = epilogue::gemm::PcKBlockCompInt8Epilogue<epilogue::gemm::AccumulatorWriteBackFp32>;++ template <class GemmCore_T>+ void ut_newkblock(int m, int n, int k, int blocksize, BTLA_DTYPE qtype, BTLA_DTYPE stype, bool isAsym = false) {+ printf("Test Case %s: %d %d %d-%d type:%s core:%s scaletype:%s Asym:%d\n", __FUNCTION__, m, n, k, blocksize,+ bestla_dtype_str(qtype), gemm::CoreAttr::to_str(GemmCore_T::ID), bestla_dtype_str(stype), isAsym);+ auto constexpr ISA = GemmCore_T::ISA;+ using Launcher = wrapper::gemm::LauncherIntKBlock<GemmCore_T, prologue_a::gemm::ActivationF32KBlockQuantize,+ prologue_b::gemm::WeightKBlockNInteger,+ epilogue::gemm::AccumulatorWriteBackFp32>;+ using Parallel = parallel::gemm::SchedulerIntKBlock<GemmCore_T>;++ blocksize = blocksize == -1 ? k : blocksize;+ int kblks = updiv(k, blocksize);+ using WType = typename Launcher::PrologueB::StorageWeight;+ WType packedw = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, stype, isAsym);+ utils::avector<int8_t> buffer(packedw.mSize);+ packedw.assign(buffer.data());+ avector<float> matBf32(k * n), matAf32(m * k), matC(m * n), refC(m * n), refCupk(m * n);+ fill_buffer_randn(matBf32.data(), matBf32.size(), -0.5f, 0.5f);+ avector<uint8_t> matAu8(m * k), zpAu8(m * kblks);+ avector<float> scaleAf32(m * kblks);+ fill_buffer_randn(matAu8.data(), matAu8.size(), uint8_t(0), uint8_t(127));+ fill_buffer_randn(zpAu8.data(), zpAu8.size(), uint8_t(60), uint8_t(64));+ fill_buffer_randn(scaleAf32.data(), scaleAf32.size(), 0.001f, 0.005f);+ ut::fill_buffer_randn(matBf32.data(), matBf32.size(), -0.5f, 0.5f);+ for (size_t i = 0; i < m; i++) {+ for (size_t j = 0; j < k; j++) {+ matAf32[i * k + j] =+ (float(matAu8[i * k + j]) - zpAu8[i * kblks + j / blocksize]) * scaleAf32[i * kblks + j / blocksize];+ }+ }++ Launcher::PrologueB::packWeight(n, k, matBf32.data(), n, &packedw, UT_Threading::get());+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), matBf32.data(), refC.data(), k, n, n);+ Launcher::PrologueB::unpackWeight(n, k, &packedw, matBf32.data(), n, UT_Threading::get());+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), matBf32.data(), refCupk.data(), k, n, n);+ auto quanA = Launcher::PrologueA::createStorage(m, k, blocksize, isAsym);+ utils::avector<int8_t> bufferA(quanA.mSize);+ quanA.assign(bufferA.data());+ GemmProblem gp(1, m, n, k, blocksize);+ typename Launcher::Param args{gp, {matAf32.data(), k, &quanA}, {&packedw}, {matC.data(), n}};+ parallel::GemmRunWithA<Parallel, Launcher>(args, UT_Threading::get());+ auto err = get_ut_err(qtype);+ buffer_error(refC.data(), matC.data(), refC.size(), err);+ if (stype != BTLA_DTYPE::DQ8_BNB) {+ buffer_error(refCupk.data(), matC.data(), refCupk.size(), INT8_ERR); // dynamic quant error+ } else {+ auto DQ_INT8_ERR = 0.8f;+ buffer_error(refCupk.data(), matC.data(), refCupk.size(), DQ_INT8_ERR); // dynamic quant error+ }+ }++ template <class GemmCore_T>+ void ut_newkblock_v2(int m, int n, int k, int blocksize, BTLA_DTYPE qtype, BTLA_DTYPE stype, bool isAsym = false) {+ printf("Test Case %s: %d %d %d-%d type:%s core:%s scaletype:%s Asym:%d\n", __FUNCTION__, m, n, k, blocksize,+ bestla_dtype_str(qtype), gemm::CoreAttr::to_str(GemmCore_T::ID), bestla_dtype_str(stype), isAsym);+ auto constexpr ISA = GemmCore_T::ISA;+ using Launcher = wrapper::gemm::LauncherIntKBlockV2<GemmCore_T, prologue_a::gemm::ActivationF32KBlockQuantize,+ prologue_b::gemm::WeightKBlockNInt,+ epilogue::gemm::AccumulatorWriteBackFp32>;+ using Parallel = parallel::gemm::SchedulerIntKBlock<GemmCore_T>;++ blocksize = blocksize == -1 ? k : blocksize;+ int kblks = updiv(k, blocksize);+ auto packedw = Launcher::PrologueB::create_storage(n, k, blocksize, qtype, stype, isAsym);+ utils::avector<int8_t> buffer(packedw.buf_size_);+ packedw.set_buffers(buffer.data());+ avector<float> matBf32(k * n), matAf32(m * k), matC(m * n), refC(m * n), refCupk(m * n);+ fill_buffer_randn(matBf32.data(), matBf32.size(), -0.5f, 0.5f);+ avector<uint8_t> matAu8(m * k), zpAu8(m * kblks);+ avector<float> scaleAf32(m * kblks);+ fill_buffer_randn(matAu8.data(), matAu8.size(), uint8_t(0), uint8_t(127));+ fill_buffer_randn(zpAu8.data(), zpAu8.size(), uint8_t(60), uint8_t(64));+ fill_buffer_randn(scaleAf32.data(), scaleAf32.size(), 0.001f, 0.005f);+ ut::fill_buffer_randn(matBf32.data(), matBf32.size(), -0.5f, 0.5f);+ for (size_t i = 0; i < m; i++) {+ for (size_t j = 0; j < k; j++) {+ matAf32[i * k + j] =+ (float(matAu8[i * k + j]) - zpAu8[i * kblks + j / blocksize]) * scaleAf32[i * kblks + j / blocksize];+ }+ }+ storage::gemm::StorageWeightNInt rawB, dstB;+ rawB.set_nt_data(n, k, k, utils::bestla_dtype<float>, matBf32.data());+ Launcher::PrologueB::convert(rawB, packedw, UT_Threading::get());+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), matBf32.data(), refC.data(), k, n, n);+ Launcher::PrologueB::convert(packedw, rawB, UT_Threading::get());+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), matBf32.data(), refCupk.data(), k, n, n);+ auto quanA = Launcher::PrologueA::createStorage(m, k, blocksize, isAsym);+ utils::avector<int8_t> bufferA(quanA.mSize);+ quanA.assign(bufferA.data());+ GemmProblem gp(1, m, n, k, blocksize);+ typename Launcher::Param args{gp, {matAf32.data(), k, &quanA}, {&packedw}, {matC.data(), n}};+ parallel::GemmRunWithA<Parallel, Launcher>(args, UT_Threading::get());+ auto err = get_ut_err(qtype);+ buffer_error(refC.data(), matC.data(), refC.size(), err);+ if (stype != BTLA_DTYPE::DQ8_BNB) {+ buffer_error(refCupk.data(), matC.data(), refCupk.size(), INT8_ERR); // dynamic quant error+ } else {+ auto DQ_INT8_ERR = 0.8f;+ buffer_error(refCupk.data(), matC.data(), refCupk.size(), DQ_INT8_ERR); // dynamic quant error+ }+ }++ template <class GemmCore_T>+ void ut_newkblock_pc(int m, int n, int k, int blocksize, BTLA_DTYPE qtype, BTLA_DTYPE stype, bool isAsym = false) {+ assert(blocksize >= k);+ printf("Test Case %s: %d %d %d-%d type:%s core:%s scaletype:%s Asym:%d\n", __FUNCTION__, m, n, k, blocksize,+ bestla_dtype_str(qtype), gemm::CoreAttr::to_str(GemmCore_T::ID), bestla_dtype_str(stype), isAsym);+ auto constexpr ISA = GemmCore_T::ISA;+ using Parallel = parallel::gemm::SchedulerBase<GemmCore_T>;+ using Launcher = wrapper::gemm::LauncherBase<GemmCore_T, prologue_a::gemm::ActivationF32KBlockQuantize,+ prologue_b::gemm::WeightKBlockNInteger, PcWriteBack>;++ blocksize = blocksize == -1 ? k : blocksize;+ int kblks = updiv(k, blocksize);+ using WType = typename Launcher::PrologueB::StorageWeight;+ WType packedw = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, stype, isAsym);+ utils::avector<int8_t> buffer(packedw.mSize);+ packedw.assign(buffer.data());+ avector<float> matBf32(k * n), matAf32(m * k), matC(m * n), refC(m * n), refCupk(m * n);+ fill_buffer_randn(matBf32.data(), matBf32.size(), -0.5f, 0.5f);+ avector<uint8_t> matAu8(m * k), zpAu8(m * kblks);+ avector<float> scaleAf32(m * kblks);+ fill_buffer_randn(matAu8.data(), matAu8.size(), uint8_t(0), uint8_t(127));+ fill_buffer_randn(zpAu8.data(), zpAu8.size(), uint8_t(60), uint8_t(64));+ fill_buffer_randn(scaleAf32.data(), scaleAf32.size(), 0.001f, 0.005f);+ ut::fill_buffer_randn(matBf32.data(), matBf32.size(), -0.5f, 0.5f);+ avector<float> reduceAf32(m * kblks, 0.f);+ for (size_t i = 0; i < m; i++) {+ for (size_t j = 0; j < k; j++) {+ matAf32[i * k + j] =+ (float(matAu8[i * k + j]) - zpAu8[i * kblks + j / blocksize]) * scaleAf32[i * kblks + j / blocksize];+ reduceAf32[i * kblks + j / blocksize] += matAf32[i * k + j];+ }+ }+ Launcher::PrologueB::packWeight(n, k, matBf32.data(), n, &packedw, UT_Threading::get());+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), matBf32.data(), refC.data(), k, n, n);+ Launcher::PrologueB::unpackWeight(n, k, &packedw, matBf32.data(), n, UT_Threading::get());+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), matBf32.data(), refCupk.data(), k, n, n);+ auto quanA = Launcher::PrologueA::createStorage(m, k, blocksize, isAsym);+ utils::avector<int8_t> bufferA(quanA.mSize);+ quanA.assign(bufferA.data());+ GemmProblem gp(1, m, n, k, blocksize);+ typename Launcher::Param args{+ gp,+ {matAf32.data(), k, &quanA},+ {&packedw},+ {{packedw.template SPtr<char>(), packedw.SDtype(), quanA.template SPtr<float>(), quanA.template ZPtr<uint8_t>(),+ packedw.template RPtr<int32_t>(), nullptr, nullptr, k},+ {matC.data(), n}}};+ parallel::GemmRunWithA<Parallel, Launcher>(args, UT_Threading::get());+ auto err = get_ut_err(qtype);+ buffer_error(refC.data(), matC.data(), refC.size(), err);++ if (stype != BTLA_DTYPE::DQ8_BNB) {+ buffer_error(refCupk.data(), matC.data(), refCupk.size(), INT8_ERR); // dynamic quant error+ } else {+ auto DQ_INT8_ERR = 0.8f;+ buffer_error(refCupk.data(), matC.data(), refCupk.size(), DQ_INT8_ERR); // dynamic quant error+ }+ }+};+#ifdef BTLA_UT_PROLOGUE_B+static UT_CompInt8 sUT_CompInt8;+#endif++class UT_CompBf16 {+ public:+ UT_CompBf16() {+ UT_START();+ CheckISA(AMX_BF16);+ request_perm_xtile_data();+ ut_s4();+ ut_s8();+ ut_f4();+ ut_f8();+ }++ void ut_f8() {+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNFloat, f8>(2, 4096, 4096, 32, BTLA_DTYPE::F8_E4M3);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNFloat, float>(2, 4096, 4096, 32, BTLA_DTYPE::F8_E4M3);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNFloat, f8>(2, 4096, 4096, 32, BTLA_DTYPE::F8_E5M2);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNFloat, float>(2, 4096, 4096, 32, BTLA_DTYPE::F8_E5M2);+ }++ void ut_s4() {+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNInteger, float>(2, 4096, 4096, 32, BTLA_DTYPE::S4_CLIP);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNInteger, float>(2, 4096, 4096, 128, BTLA_DTYPE::S4_CLIP);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNInteger, float>(2, 4096, 4096, -1, BTLA_DTYPE::S4_CLIP);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(2, 4096, 4096, 32, BTLA_DTYPE::S4_CLIP);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNInteger, utils::fp16>(2, 4096, 4096, 32, BTLA_DTYPE::S4_CLIP);+ }++ void ut_s8() {+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNInteger, float>(2, 4096, 4096, 32, BTLA_DTYPE::S8);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNInteger, float>(2, 4096, 4096, 128, BTLA_DTYPE::S8);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNInteger, float>(2, 4096, 4096, -1, BTLA_DTYPE::S8);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(2, 4096, 4096, 32, BTLA_DTYPE::S8);+ }++ void ut_f4() {+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNFloat, float>(2, 4096, 4096, 32, BTLA_DTYPE::F4_E2M1);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNFloat, float>(2, 4096, 4096, 32, BTLA_DTYPE::F4_BNB);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNFloat, float>(2, 4096, 4096, -1, BTLA_DTYPE::F4_BNB);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNFloat, float>(2, 4096, 4096, 32, BTLA_DTYPE::F4_E2M1);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNFloat, float>(2, 4096, 4096, -1, BTLA_DTYPE::F4_E2M1);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNFloat, float>(2, 4096, 4096, 32, BTLA_DTYPE::F4_NF4);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNFloat, float>(2, 4096, 4096, -1, BTLA_DTYPE::F4_NF4);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNFloat, utils::bf16>(2, 4096, 4096, 32, BTLA_DTYPE::F4_BNB);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNFloat, utils::bf16>(2, 4096, 4096, 32, BTLA_DTYPE::F4_E2M1);+ ut<sAMX_BF16, prologue_b::gemm::WeightKBlockNFloat, utils::bf16>(2, 4096, 4096, 32, BTLA_DTYPE::F4_NF4);+ }++ template <class GemmCore_T, template <class _T> class Wei, typename Scale_T>+ void ut(int m, int n, int k, int blocksize, BTLA_DTYPE qtype) {+ printf("Test Case %s: %d %d %d-%d type:%s core:%s scaletype:%s\n", __FUNCTION__, m, n, k, blocksize,+ bestla_dtype_str(qtype), gemm::CoreAttr::to_str(GemmCore_T::ID), type_str<Scale_T>);+ auto constexpr ISA = GemmCore_T::ISA;+ using Launcher = wrapper::gemm::LauncherBase<GemmCore_T, prologue_a::gemm::ActivationBase, Wei,+ epilogue::gemm::AccumulatorWriteBackFp32>;+ using Parallel = parallel::gemm::SchedulerBase<GemmCore_T>;++ blocksize = blocksize == -1 ? k : blocksize;+ using WType = typename Wei<GemmCore_T>::StorageWeight;+ WType packedw(0);+ if constexpr (std::is_same_v<Wei<GemmCore_T>, prologue_b::gemm::WeightKBlockNInteger<GemmCore_T>>) {+ packedw = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, bestla_dtype<Scale_T>, false);+ } else if constexpr (std::is_same_v<Wei<GemmCore_T>, prologue_b::gemm::WeightKBlockNFloat<GemmCore_T>>) {+ packedw = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, bestla_dtype<Scale_T>);+ }++ utils::avector<int8_t> buffer(packedw.mSize);+ packedw.assign(buffer.data());+ avector<utils::bf16> matAbf16(m * k), matBbf16(k * n);+ fill_buffer_randn(matAbf16.data(), matAbf16.size(), utils::bf16(-0.5f), utils::bf16(0.5f));+ fill_buffer_randn(matBbf16.data(), matBbf16.size(), utils::bf16(-0.5f), utils::bf16(0.5f));+ avector<float> matBf32(k * n), matAf32(m * k), matC(m * n), refC(m * n), refCupk(m * n);+ for (size_t i = 0; i < matBf32.size(); i++) {+ matBf32[i] = matBbf16[i];+ }+ Launcher::PrologueB::packWeight(n, k, matBf32.data(), n, &packedw, UT_Threading::get());+ gemmref_bf16bf16fp32(m, n, k, matAbf16.data(), matBbf16.data(), refC.data(), k, n, n);+ Launcher::PrologueB::unpackWeight(n, k, &packedw, matBf32.data(), n, UT_Threading::get());+ for (size_t i = 0; i < matBf32.size(); i++) {+ matBbf16[i] = static_cast<utils::bf16>(matBf32[i]);+ }+ gemmref_bf16bf16fp32(m, n, k, matAbf16.data(), matBbf16.data(), refCupk.data(), k, n, n);+ GemmProblem gp(1, m, n, k, blocksize);+ typename Launcher::Param args{gp, {matAbf16.data(), k}, {&packedw}, {matC.data(), n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ auto err = get_ut_err(qtype);+ buffer_error(refC.data(), matC.data(), refC.size(), err);+ buffer_error(refCupk.data(), matC.data(), refCupk.size(), 0.05f);+ }+};+#ifdef BTLA_UT_PROLOGUE_B+static UT_CompBf16 sUT_CompBf16;+#endif++class UT_ORT_NBits {+ public:+ UT_ORT_NBits() {+ UT_START();+ ut_s4();+ }++ void ut_s4() {+ CheckISA(AVX2);+ ut<sAVX2>(1, 14336, 4096, 32, BTLA_DTYPE::S4_CLIP, true);+ ut<sAVX2>(1, 1, 32, 32, BTLA_DTYPE::S4_CLIP, false);+ ut<sAVX2>(1, 2, 32, 32, BTLA_DTYPE::S4_CLIP, true);+ ut<sAVX2>(2, 4096, 4096, 32, BTLA_DTYPE::S4_CLIP, true);+ ut<sAVX2>(1, 11008, 4096, 32, BTLA_DTYPE::S4_CLIP, false);+ ut<sAVX2>(2, 4096, 4096, 32, BTLA_DTYPE::S4_CLIP, false);+ ut<sAVX2>(2, 4096, 4096, 128, BTLA_DTYPE::S4_CLIP, false);+ ut<sAVX2>(2, 4096, 4096, -1, BTLA_DTYPE::S4_CLIP, false);+ CheckISA(AVX512F);+ ut<sAVX512F>(2, 4096, 4096, 32, BTLA_DTYPE::S4_CLIP, true);+ ut<sAVX512F>(2, 4096, 4096, 32, BTLA_DTYPE::S4_CLIP, false);+ ut<sAVX512F>(2, 4096, 4096, 128, BTLA_DTYPE::S4_CLIP, false);+ ut<sAVX512F>(2, 4096, 4096, -1, BTLA_DTYPE::S4_CLIP, false);+ }++ template <class GemmCore_T>+ void ut(int m, int n, int k, int blocksize, BTLA_DTYPE qtype, bool isasym) {+ printf("Test Case %s: %d %d %d-%d type:%s core:%s asym:%d \n", __FUNCTION__, m, n, k, blocksize,+ bestla_dtype_str(qtype), gemm::CoreAttr::to_str(GemmCore_T::ID), isasym);+ auto constexpr ISA = GemmCore_T::ISA;+ using Launcher =+ wrapper::gemm::LauncherBase<GemmCore_T, prologue_a::gemm::ActivationKBlockBaseF32,+ prologue_b::gemm::WeightKBlockNInteger, epilogue::gemm::AccumulatorWriteBackFp32>;+ using Parallel = parallel::gemm::SchedulerBase<GemmCore_T>;++ blocksize = blocksize == -1 ? k : blocksize;+ using WType = storage::gemm::StorageWeightKBlockNInteger;+ WType packedw(0);+ packedw = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, bestla_dtype<float>, isasym);++ utils::avector<int8_t> buffer(packedw.mSize);+ packedw.assign(buffer.data());+ avector<float> matBf32(k * n), matAf32(m * k), matC(m * n), refC(m * n), refCupk(m * n);+ fill_buffer_randn(matAf32.data(), matAf32.size(), -0.5f, 0.5f);+ avector<uint8_t> matBs8(n * k);+ avector<int4x2> matBs4(n * updiv(k, 2));+ int blks = updiv(k, blocksize);+ avector<float> scalesB(n * blks);+ avector<uint8_t> zpBs8(n * blks, 8);+ auto blk_padding = updiv(blks, 2);+ avector<int4x2> zpBs4(n * blk_padding, uint8_t(0x88));+ fill_buffer_randn(matBs8.data(), matBs8.size(), uint8_t(0), uint8_t(15));+ if (isasym) {+ fill_buffer_randn(zpBs8.data(), zpBs8.size(), uint8_t(0), uint8_t(15));+ }+ fill_buffer_randn(scalesB.data(), scalesB.size(), 0.001f, 0.005f);+ avector<float> reduceA(m * blks, 0.f);++ auto rA = Launcher::PrologueA::createStorage(m, k, blocksize);+ avector<int8_t> tmpA(rA.mSize);+ if (isasym) {+ for (size_t i = 0; i < m; i++) {+ for (size_t j = 0; j < k; j++) {+ reduceA[i * blks + j / blocksize] += matAf32[i * k + j];+ }+ }+ rA.assign(tmpA.data());+ Launcher::PrologueA::reduce({matAf32.data(), k, &rA}, m, k, blocksize,+ UT_Threading::get()); // for reduce UT+ buffer_error(reduceA.data(), rA.template RPtr<float>(), reduceA.size(), FP32_ERR);+ memset(tmpA.data(), 0, tmpA.size()); // clear+ }+ for (size_t i = 0; i < n; i++) {+ for (size_t j = 0; j < k; j += 2) {+ *(uint8_t*)&matBs4[i * k / 2 + j / 2] = matBs8[i * k + j] | matBs8[i * k + j + 1] << 4;+ auto koff = j / blocksize + i * blks;+ auto koff1 = (j + 1) / blocksize + i * blks;+ matBf32[j * n + i] = (float(matBs8[i * k + j]) - zpBs8[koff]) * scalesB[koff];+ matBf32[(j + 1) * n + i] = (float(matBs8[i * k + j + 1]) - zpBs8[koff1]) * scalesB[koff1];+ }+ for (size_t j = 0; j < k; j += blocksize * 2) {+ *(uint8_t*)&zpBs4[i * blk_padding + j / blocksize / 2] =+ zpBs8[i * blks + j / blocksize] | zpBs8[i * blks + j / blocksize + 1] << 4;+ }+ }+ Launcher::PrologueB::packNbitsWeightQ4(n, k, isasym, (uint8_t*)matBs4.data(), k, scalesB.data(),+ isasym ? (uint8_t*)zpBs4.data() : nullptr, &packedw, UT_Threading::get());+ Launcher::PrologueB::reduceWeight(&packedw, UT_Threading::get());+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), matBf32.data(), refC.data(), k, n, n);+ avector<float> revB(matBf32.size());+ Launcher::PrologueB::unpackWeight(n, k, &packedw, revB.data(), n, UT_Threading::get());+ buffer_error(matBf32.data(), revB.data(), revB.size(), FP32_ERR);+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), revB.data(), refCupk.data(), k, n, n);+ GemmProblem gp(1, m, n, k, blocksize);+ typename Launcher::Param args{gp, {matAf32.data(), k, &rA}, {&packedw}, {matC.data(), n}};+ if (isasym) {+ parallel::GemmRunWithA<Parallel, Launcher>(args, UT_Threading::get());+ } else {+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ }+ auto err = INT4_ERR;+ buffer_error(refC.data(), matC.data(), refC.size(), err);+ buffer_error(refCupk.data(), matC.data(), refCupk.size(), 0.001f);+ }++ template <class GemmCore_T>+ void ut_file(int m) {+ int n = 14336, k = 4096, blocksize = 32;+ BTLA_DTYPE qtype = BTLA_DTYPE::S4_CLIP;+ bool isasym = true;+ printf("Test Case %s: %d %d %d-%d type:%s core:%s asym:%d \n", __FUNCTION__, m, n, k, blocksize,+ bestla_dtype_str(qtype), gemm::CoreAttr::to_str(GemmCore_T::ID), isasym);+ auto constexpr ISA = GemmCore_T::ISA;+ using Launcher =+ wrapper::gemm::LauncherBase<GemmCore_T, prologue_a::gemm::ActivationKBlockBaseF32,+ prologue_b::gemm::WeightKBlockNInteger, epilogue::gemm::AccumulatorWriteBackFp32>;+ using Parallel = parallel::gemm::SchedulerIntKBlock<GemmCore_T>;++ const char *qfile = "int_weight.bin", *sfile = "scales.bin", *zfile = "zeros.bin";+ auto qdata = ut::readFile2Buffer<int8_t>(qfile);+ auto sdata = readFile2Buffer<float>(sfile);+ auto zdata = readFile2Buffer<int8_t>(zfile);+ using WType = storage::gemm::StorageWeightKBlockNInteger;+ WType packedw = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, bestla_dtype<float>,+ bestla_dtype<utils::bf16>, isasym);++ utils::avector<int8_t> buffer(packedw.mSize);+ packedw.assign(buffer.data());+ avector<float> matBf32(k * n), matAf32(m * k), matC(m * n), refC(m * n), refCupk(m * n);+ fill_buffer_randn(matAf32.data(), matAf32.size(), -0.5f, 0.5f);+ int blks = updiv(k, blocksize);+ avector<float> reduceA(m * blks, 0.f);++ auto rA = Launcher::PrologueA::createStorage(m, k, blocksize);+ avector<int8_t> tmpA(rA.mSize);+ if (isasym) {+ for (size_t i = 0; i < m; i++) {+ for (size_t j = 0; j < k; j++) {+ reduceA[i * blks + j / blocksize] += matAf32[i * k + j];+ }+ }+ rA.assign(tmpA.data());+ Launcher::PrologueA::template reduce<ISA>({matAf32.data(), k, &rA}, m, k, blocksize,+ UT_Threading::get()); // for reduce UT+ buffer_error(reduceA.data(), rA.template RPtr<float>(), reduceA.size(), FP32_ERR);+ memset(tmpA.data(), 0, tmpA.size()); // clear+ }+ for (size_t i = 0; i < k; i++) {+ for (size_t j = 0; j < n; j++) {+ matBf32[i * n + j] = (float(qdata[i * n + j]) - zdata[i / blocksize * n + j]) * sdata[i / blocksize * n + j];+ }+ }++ Launcher::PrologueB::packQWeight(n, k, qdata.data(), n, sdata.data(), zdata.data(), &packedw, UT_Threading::get());++ auto bfile = readFile2Buffer<int8_t>("bestla_w3.weight.bin");+ WType packedfile(0);+ packedfile.deserialize(bfile.data());+ buffer_error(packedw.WPtr<int8_t>(), packedfile.WPtr<int8_t>(), packedw.mQBuf.size<int8_t>());+ buffer_error(packedw.SPtr<float>(), packedfile.SPtr<float>(), packedw.CSize());+ buffer_error(packedw.ZPtr<int8_t>(), packedfile.ZPtr<int8_t>(), packedw.CSize());+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), matBf32.data(), refC.data(), k, n, n);+ avector<float> revB(matBf32.size());+ Launcher::PrologueB::unpackWeight(n, k, &packedw, revB.data(), n, UT_Threading::get());+ buffer_error(matBf32.data(), revB.data(), revB.size(), FP32_ERR);+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), revB.data(), refCupk.data(), k, n, n);+ GemmProblem gp(1, m, n, k, blocksize);+ typename Launcher::Param args{gp, {matAf32.data(), k, &rA}, {&packedw}, {matC.data(), n}};+ if (isasym) {+ parallel::GemmRunWithA<Parallel, Launcher>(args, UT_Threading::get());+ } else {+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ }+ auto err = INT4_ERR;+ buffer_error(refC.data(), matC.data(), refC.size(), err);+ buffer_error(refCupk.data(), matC.data(), refCupk.size(), 0.001f);+ }+};+#ifdef BTLA_UT_PROLOGUE_B+static UT_ORT_NBits sUT_ORT_NBits;+#endif++class UT_CompFp16 {+ public:+ UT_CompFp16() {+ UT_START();+ CheckISA(AMX_FP16);+ ut_s4();+ }++ void ut_s4() {+ ut<sAMX_FP16, prologue_b::gemm::WeightKBlockNInteger, float>(2, 4096, 4096, 32, BTLA_DTYPE::S4_CLIP);+ ut<sAMX_FP16, prologue_b::gemm::WeightKBlockNInteger, float>(2, 4096, 4096, 128, BTLA_DTYPE::S4_CLIP);+ ut<sAMX_FP16, prologue_b::gemm::WeightKBlockNInteger, float>(2, 4096, 4096, -1, BTLA_DTYPE::S4_CLIP);+ ut<sAMX_FP16, prologue_b::gemm::WeightKBlockNInteger, utils::bf16>(2, 4096, 4096, 32, BTLA_DTYPE::S4_CLIP);+ ut<sAMX_FP16, prologue_b::gemm::WeightKBlockNInteger, utils::fp16>(2, 4096, 4096, 32, BTLA_DTYPE::S4_CLIP);+ ut<sAMX_FP16, prologue_b::gemm::WeightKBlockNInteger, float>(16, 4096, 4096, 128, BTLA_DTYPE::S4_CLIP);+ }++ template <class GemmCore_T, template <class _T> class Wei, typename Scale_T>+ void ut(int m, int n, int k, int blocksize, BTLA_DTYPE qtype) {+ printf("Test Case %s: %d %d %d-%d type:%s core:%s scaletype:%s\n", __FUNCTION__, m, n, k, blocksize,+ bestla_dtype_str(qtype), gemm::CoreAttr::to_str(GemmCore_T::ID), type_str<Scale_T>);+ auto constexpr ISA = GemmCore_T::ISA;+ using Launcher = wrapper::gemm::LauncherBase<GemmCore_T, prologue_a::gemm::ActivationBase, Wei,+ epilogue::gemm::AccumulatorWriteBackFp32>;+ using Parallel = parallel::gemm::SchedulerBase<GemmCore_T>;++ blocksize = blocksize == -1 ? k : blocksize;+ using WType = typename Wei<GemmCore_T>::StorageWeight;+ WType packedw(0);+ if constexpr (std::is_same_v<Wei<GemmCore_T>, prologue_b::gemm::WeightKBlockNInteger<GemmCore_T>>) {+ packedw = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, bestla_dtype<Scale_T>, false);+ } else if constexpr (std::is_same_v<Wei<GemmCore_T>, prologue_b::gemm::WeightKBlockNFloat<GemmCore_T>>) {+ packedw = Launcher::PrologueB::createStorage(n, k, blocksize, qtype, bestla_dtype<Scale_T>);+ }++ utils::avector<int8_t> buffer(packedw.mSize);+ packedw.assign(buffer.data());+ avector<utils::fp16> matAfp16(m * k), matBfp16(k * n);+ fill_buffer_randn(matAfp16.data(), matAfp16.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ fill_buffer_randn(matBfp16.data(), matBfp16.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ avector<float> matBf32(k * n), matAf32(m * k), matC(m * n), refC(m * n), refCupk(m * n);+ kernel::wrapper::Memcpy2DFp16CvtFp32::forward<ISA>(matBfp16.data(), matBf32.data(), k, n, n * sizeof(matBfp16[0]),+ n * sizeof(matBf32[0]), false);+ Launcher::PrologueB::packWeight(n, k, matBf32.data(), n, &packedw, UT_Threading::get());+ gemmref_fp16fp16fp32(m, n, k, matAfp16.data(), matBfp16.data(), refC.data(), k, n, n);+ Launcher::PrologueB::unpackWeight(n, k, &packedw, matBf32.data(), n, UT_Threading::get());+ kernel::wrapper::Memcpy2DFp32CvtFp16::forward<ISA>(matBf32.data(), matBfp16.data(), k, n, n * sizeof(matBf32[0]),+ n * sizeof(matBfp16[0]), false);+ gemmref_fp16fp16fp32(m, n, k, matAfp16.data(), matBfp16.data(), refCupk.data(), k, n, n);+ GemmProblem gp(1, m, n, k, blocksize);+ typename Launcher::Param args{gp, {matAfp16.data(), k}, {&packedw}, {matC.data(), n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ auto err = get_ut_err(qtype);+ buffer_error(refC.data(), matC.data(), refC.size(), err);+ buffer_error(refCupk.data(), matC.data(), refCupk.size(), 0.05f);+ }+};+#ifdef BTLA_UT_PROLOGUE_B+static UT_CompFp16 sUT_CompFp16;+#endif+} // namespace ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_ut.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_ut.cppnew file mode 100644index 000000000..a2ac93714--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_ut.cpp@@ -0,0 +1,7 @@+#include <stdio.h>+#include <bestla_parallel.h>++int main() {+ printf("BesTLA UT done\n");+ return 0;+}diff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_ut.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_ut.hnew file mode 100644index 000000000..49e9f54e3--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_ut.h@@ -0,0 +1,803 @@+#include <random>+#include <vector>++#include "bestla/bestla_device.h"+#pragma once++#include <random>+#include <stdexcept>+#include "bestla/bestla_utils.h"+#include "bestla/bestla_gemm.h"+#include "bestla/bestla_device.h"+#include "bestla/bestla_parallel.h"++#define CheckISA(ISA) \+ { \+ GetCPUDevice(); \+ if (!_cd->ISA()) { \+ printf("Wrong Device ISA: " #ISA "\n"); \+ return; \+ } \+ }++namespace bestla {+namespace ut {+using sAVX512F = gemm::SCoreRowNAvx512f<48, 8>;+using sAMX_BF16 = gemm::HCoreRowNAmxbf16<64, 16>;+using sAMX_FP16 = gemm::HCoreRowNAmxfp16<64, 16>;+using sAVX512_FP16 = gemm::HCoreRowNAvx512fp16<96, 8>;+using sAVX_VNNI = gemm::ICoreRowNAvxvnni<24, 4>;+using sAVX_VNNI_SS = gemm::ICoreRowNAvxvnniSS<24, 4>;+using sAVX512BW = gemm::ICoreRowNAvx512bw<48, 8>;+using sAVX512_VNNI = gemm::ICoreRowNAvx512vnni<48, 8>;+using sAMX_INT8_US = gemm::ICoreRowNAmxint8<64, 16>;+using sAMX_INT8_SS = gemm::ICoreRowNAmxint8SS<64, 16>;+using sAVX2 = gemm::SCoreRowNAvx2<24, 4>;++class UT_Threading {+ public:+ static bestla::parallel::IThreading* get() {+#if BTLA_OPENMP+ static bestla::parallel::OMPThreading DefaultThreading;+#else+ static bestla::parallel::StdThreading DefaultThreading;+#endif // _OPNEMP+ return &DefaultThreading;+ }++ static void set_threads(int n_thread) { get()->set_threads(n_thread); }++ static std::vector<int> get_threads_config() {+ GetCPUDevice();+ if (_cd->isClient()) {+ if (_cd->isHybrid()) {+ return std::vector<int>{_cd->getThreads(), _cd->getCores(), int(_cd->getPcoreNum())};+ }+ return std::vector<int>{_cd->getCores() * 2, _cd->getCores()};+ }++ if (_cd->getThreads() == 56) {+ return std::vector<int>{48, 56};+ }+ if (_cd->getThreads() == 128) {+ return std::vector<int>{32};+ }+ return std::vector<int>{_cd->getThreads()};+ }+};+static inline size_t gemm_memsize(int m, int n, int k, BTLA_DTYPE dtA, BTLA_DTYPE dtB, BTLA_DTYPE dtC) {+ size_t total = 0;+ total += size_t(m) * k * utils::bestla_dtype_bits(dtA);+ total += size_t(n) * k * utils::bestla_dtype_bits(dtB);+ total += size_t(m) * n * utils::bestla_dtype_bits(dtC);+ return total / 8;+}++static inline int auto_batch(size_t memsize) {+ GetCPUDevice();+ auto L3 = _cd->getL3CacheSize();+ size_t constexpr Enlarge = 4;+ size_t constexpr TargetMem = 1LL << 30;+ auto batch = static_cast<int>(std::max(L3 * Enlarge, TargetMem) / memsize);+ return batch > 1 ? batch : 2;+}++constexpr size_t CacheSize = size_t(100) << 10;+static int8_t cache[CacheSize];++// UT Error definitions+// Activation uniform distribution range [-0.5f,0.5f]+// Weight uniform distribution range [-0.5f,0.5f]+// reduce dim: 4096+#define FP32_ERR 0.0001f+#define FP16_ERR 0.001f+#define BF16_ERR 0.02f+#define INT8_ERR 0.2f+#define F8_ERR 1.5f+#define INT4_ERR 3.5f+#define INT3_ERR 7.f+#define INT2_ERR 18.f+#define INT1_ERR 1e3f+#define FP4_ERR 3.5f++static inline float get_ut_err(BTLA_DTYPE qtype) {+ auto dbits = utils::bestla_dtype_bits(qtype);+ auto type = utils::bestla_dtype_type(qtype);+ auto err = FP32_ERR;+ auto constexpr dtype_int = utils::bestla_dtype_type(BTLA_DTYPE::TypeInt);+ if (type == dtype_int) {+ if (dbits == 8) {+ err = INT8_ERR;+ } else if (dbits == 3) {+ err = INT3_ERR;+ } else if (dbits == 2) {+ err = INT2_ERR;+ } else if (dbits == 1) {+ err = INT1_ERR;+ } else {+ err = INT4_ERR;+ }+ } else {+ if (dbits == 4) {+ err = FP4_ERR;+ } else if (dbits == 8) {+ err = F8_ERR;+ } else if (dbits == 16) {+ if (qtype == BTLA_DTYPE::F16) {+ err = FP16_ERR;+ } else {+ err = BF16_ERR;+ }+ }+ }+ return err;+}++template <typename _T>+inline _T randn(_T minval, _T maxval) {+ auto normval = (rand() + 0.5f) / (RAND_MAX + 1.f);+ auto _gap = maxval - minval;+ return static_cast<_T>(_gap * normval + minval);+}++template <>+inline utils::bf16 randn(utils::bf16 minval, utils::bf16 maxval) {+ auto normval = (rand() + 0.5f) / (RAND_MAX + 1.f);+ auto _gap = maxval.tofloat() - minval.tofloat();+ utils::bf16 tmp;+ tmp.fromfloat(_gap * normval + minval.tofloat());+ return tmp;+}++template <>+inline utils::fp16 randn(utils::fp16 minval, utils::fp16 maxval) {+ auto normval = (rand() + 0.5f) / (RAND_MAX + 1.f);+ auto _gap = float(maxval) - float(minval);+ return utils::fp16(_gap * normval + float(minval));+}++inline float remove_bf16_err(float raw) {+ return utils::cast<utils::bf16, float>(utils::cast<float, utils::bf16>(raw));+}++inline void buffer_remove_bf16_err(float* buf, size_t size) {+ for (size_t i = 0; i < size; i++) {+ buf[i] = remove_bf16_err(buf[i]);+ }+}++template <typename _T>+static void fill_buffer_randn(_T* buf, size_t size, _T minval, _T maxval) {+ if (size <= 1024) {+ for (size_t i = 0; i < size; i++) {+ buf[i] = randn(minval, maxval);+ }+ return;+ }+ int constexpr chunks = 128;+ auto sizeP = bestla::utils::padto_le(size, chunks);+ auto size_per_run = sizeP / chunks;+#pragma omp parallel for+ for (size_t i = 0; i < chunks; i++) {+ for (size_t j = 0; j < size_per_run; j++) {+ buf[i * size_per_run + j] = randn(minval, maxval);+ }+ }+ for (size_t i = sizeP; i < size; i++) {+ buf[i] = randn(minval, maxval);+ }+}++template <typename _T>+static void fill_buffer_faken(_T* buf, size_t size, _T minval, _T maxval) {+ int constexpr seed = 997;+ auto _gap = float(maxval) - float(minval);+#pragma omp parallel for+ for (size_t i = 0; i < size; i++) {+ auto val = i % seed;+ auto normval = (val + 0.5f) / (seed + 1.f);+ buf[i] = _gap * normval + (float)minval;+ }+}++template <typename _T>+utils::aligned_vector<_T> readFile2Buffer(const char* filepath) {+ auto w1fp = fopen(filepath, "rb");+ if (w1fp == nullptr) {+ return utils::aligned_vector<_T>();+ }+ fseek(w1fp, 0, SEEK_END);+ auto size = ftell(w1fp);+ fseek(w1fp, 0, SEEK_SET);+ size_t memsize = utils::downdiv(size_t(size), sizeof(_T));+ utils::aligned_vector<_T> buf(memsize);+ fread(buf.data(), size, 1, w1fp);+ fclose(w1fp);+ return buf;+}++#define UT_START() \+ { \+ UT_Threading::get()->set_threads(0); \+ GetCPUDevice(); \+ printf("Test Class: %s\n", __FUNCTION__); \+ }+template <typename _T>+static double buffer_error(_T* ref, _T* tar, size_t size, _T athres = _T(0), _T rthres = _T(0)) {+ double err = 0;+ int cnt = 0;+ float max_err = 0, max_a = 0.f, max_b = 0.f;+ int constexpr MAX_PRINT = 10;+ int max_i = 0;+ float _rthres = (float)rthres;+ float _athres = (float)athres;+ for (size_t i = 0; i < size; i++) {+ auto vtar = float(tar[i]);+ auto vref = float(ref[i]);+ auto diff = abs(vtar - vref);+ err += diff;+ bool _err = (diff > _rthres * abs(vref)) && (diff > _athres);+ if (_err && cnt < MAX_PRINT) {+ printf("%6i Ref %12.5f\tTar %12.5f\n", int(i), vref, vtar);+ }+ if (_err) cnt++;++ if (diff > max_err) {+ max_err = diff;+ max_a = vref;+ max_b = vtar;+ max_i = int(i);+ }+ }+ if (cnt == 0) {+ printf("Case Passed!\n");+ } else {+ printf("Case Failed!\n");+ }+ printf("Max Error @%d:%.3f=%.3f-%.3f\n", max_i, max_err, max_a, max_b);+ err /= size;+ printf("Average Error: %.3f\n", err);+ return err;+}++template <typename T>+static inline void print_buffer(const T* buf, size_t size) {+ if (size <= 16) {+ std::string str = "";+ for (size_t i = 0; i < size; i++) {+ str += std::to_string(float(buf[i])) + " ";+ }+ printf("%s\n", str.c_str());+ return;+ }+ float total_value = 0.;+ for (size_t i = 0; i < size; i++) {+ total_value += float(buf[i]);+ }+ printf("Total value is: %f\n", float(total_value));+ constexpr size_t print_len = 6;+ std::string str = "";+ for (size_t i = 0; i < print_len; i++) {+ str += std::to_string(float(buf[i])) + " ";+ }+ str += "..., ";+ for (size_t i = 0; i < print_len; i++) {+ str += std::to_string(float(buf[size - print_len + i])) + " ";+ }+ printf("%s\n", str.c_str());+}++static void compare_topk(float* ref, float* tar, size_t size, int k = 8) {+ std::vector<int> indice(size);+ for (size_t i = 0; i < indice.size(); i++) {+ indice[i] = int(i);+ }+ auto comp = [&](const int& a, const int& b) { return abs(ref[a]) > abs(ref[b]); };+ if (k <= 128) {+ std::partial_sort(indice.begin(), indice.begin() + k, indice.end(), comp);+ printf("Top %d elements\n", k);+ for (size_t i = 0; i < k; i++) {+ printf("%6i Ref %12.5f\tTar %12.5f\n", int(indice[i]), float(ref[indice[i]]), float(tar[indice[i]]));+ }+ printf("\n");+ }+}++template <typename _T>+static double buffer_error_2d(_T* ref, _T* tar, size_t row, size_t col, size_t refstep, size_t tarstep,+ _T thres = _T(0)) {+ double err = 0;+ int cnt = 0;+ int constexpr MAX_PRINT = 10;+ for (size_t i = 0; i < row; i++) {+ for (size_t j = 0; j < col; j++) {+ auto refv = ref[i * refstep + j];+ auto tarv = tar[i * tarstep + j];+ auto diff = std::abs(refv - tarv);+ err += diff;+ if (diff > thres && cnt < MAX_PRINT) {+ cnt++;+ printf("%i %d Ref %12.5f\tTar %12.5f\n", int(i), int(j), utils::cast<_T, float>(refv),+ utils::cast<_T, float>(tarv));+ }+ }+ }+ if (cnt == 0) {+ printf("Case Passed!\n");+ } else {+ printf("Case Failed!\n");+ }+ auto size = (size_t)row * col;+ err /= size;+ printf("Average Error: %.3f\n", err);+ return err;+}++struct UT_vector_s8 {+ utils::aligned_vector<int8_t> data_;+ void resize(size_t _size) { data_.resize(_size); }+ size_t size() { return data_.size(); }+ int8_t* data() { return data_.data(); }+ void fill_rand(int8_t minval, int8_t maxval) {+ for (size_t i = 0; i < data_.size(); i++) {+ data_[i] = randn(minval, maxval);+ }+ }+ void rand_scale(size_t n, float minval, float maxval) {+ scales.resize(n);+ for (size_t i = 0; i < n; i++) {+ scales[i] = randn(minval, maxval);+ }+ }+ utils::aligned_vector<float> scales;+};++struct UT_vector_u8 {+ utils::aligned_vector<uint8_t> data_;+ void resize(size_t _size) { data_.resize(_size); }+ size_t size() { return data_.size(); }+ uint8_t* data() { return data_.data(); }+ void fill_rand(uint8_t minval, uint8_t maxval) {+ for (size_t i = 0; i < data_.size(); i++) {+ data_[i] = ut::randn(minval, maxval);+ }+ }+ void rand_scale(size_t n, float minval, float maxval) {+ scales.resize(n);+ zeropoints.resize(n);+ for (size_t i = 0; i < n; i++) {+ scales[i] = ut::randn(minval, maxval);+ zeropoints[i] = 0;+ }+ }+ utils::aligned_vector<float> scales;+ utils::aligned_vector<int> zeropoints;+};++struct UT_GEMMData_Row_u8s8 {+ UT_GEMMData_Row_u8s8(int m, int n, int k, int lda, int ldb, int ldc, int ldd, int nscale = 1,+ bool interleaved_ = false)+ : M(m), N(n), K(k), LDA(lda), LDB(ldb), LDC(ldc), LDD(ldd), interleaved(interleaved_) {+ matA.resize(m * lda);+ matB.resize((interleaved ? n : k) * ldb);+ matC.resize(m * ldc);+ if (ldd == 0) {+ matD.resize(n);+ } else {+ matD.resize(m * ldd);+ }+ matA.fill_rand(0, 255);+ matB.fill_rand(-127, 127);+ matD.fill_rand(0, 255);+ matA.rand_scale(nscale, 0.f, 0.01f);+ matB.rand_scale(nscale, 0.f, 0.01f);+ matD.rand_scale(nscale, 0.f, 1.f);+ }++ void calc_ref(float alpha, float beta) {+ if (interleaved) throw std::runtime_error("Only support plain data format!");+ float _cmin = std::numeric_limits<float>::max();+ float _cmax = std::numeric_limits<float>::min();+ matCRef.resize(M * LDC);+ auto tmpsrcscale = alpha * matA.scales[0] * matB.scales[0];+#pragma omp parallel for+ for (int j = 0; j < M; j++) {+ for (int i = 0; i < N; i += 1) {+ int tmp = 0;+ for (int ik = 0; ik < K; ik++) {+ tmp += matA.data()[ik + j * LDA] * matB.data()[ik * LDB + i];+ }+ auto ftmp = tmp * tmpsrcscale;+ ftmp = ftmp + (matD.data()[i + j * LDD] - matD.zeropoints[0]) * matD.scales[0] * beta;+ matCRef[i + j * LDC] = ftmp;+ _cmin = ftmp < _cmin ? ftmp : _cmin;+ _cmax = ftmp > _cmax ? ftmp : _cmax;+ }+ }+ matC.scales.resize(1);+ matC.zeropoints.resize(1);+ matC.scales[0] = (_cmax - _cmin) / (255.f);+ matC.zeropoints[0] = int((0 - _cmin) / matC.scales[0]);+ auto tmpscale = 1.f / matC.scales[0];+#pragma omp parallel for+ for (int j = 0; j < M; j++) {+ for (int i = 0; i < N; i += 1) {+ matC.data()[j * LDC + i] = utils::cast<float, uint8_t>(matCRef[j * LDC + i] * tmpscale + matC.zeropoints[0]);+ }+ }+ matCDequan.resize(matCRef.size());+#pragma omp parallel for+ for (int j = 0; j < M; j++) {+ for (int i = 0; i < N; i += 1) {+ matCDequan.data()[j * LDC + i] = ((int)matC.data()[j * LDC + i] - matC.zeropoints[0]) * matC.scales[0];+ }+ }++ /*utils::ut::buffer_error(matCRef.data(), matCDequan.data(),+ matCDequan.size(),+ matC.scales[0]);*/+ }++ UT_vector_u8 matA, matC, matD;+ utils::aligned_vector<float> matCRef, matCDequan;+ UT_vector_s8 matB;+ int M, N, K, LDA, LDB, LDC, LDD;+ bool interleaved;+};++static inline void gemmref_u8s8s32(int m, int n, int k, uint8_t* A, int8_t* B, int32_t* C, int lda, int ldb, int ldc) {+#pragma omp parallel for+ for (int j = 0; j < m; j++) {+ for (int i = 0; i < n; i += 1) {+ int tmp = 0;+ for (int ik = 0; ik < k; ik++) {+ tmp += int(A[ik + j * lda]) * int(B[ik * ldb + i]);+ }+ C[i + j * ldc] = tmp;+ }+ }+}++static inline void gemmref_s8s8s32(int m, int n, int k, int8_t* A, int8_t* B, int32_t* C, int lda, int ldb, int ldc) {+#pragma omp parallel for+ for (int j = 0; j < m; j++) {+ for (int i = 0; i < n; i += 1) {+ int tmp = 0;+ for (int ik = 0; ik < k; ik++) {+ tmp += int(A[ik + j * lda]) * int(B[ik * ldb + i]);+ }+ C[i + j * ldc] = tmp;+ }+ }+}++static inline void kblockgemmref_u8zp_s8_f32(int m, int n, int k, int kblock, uint8_t* A, uint8_t* zpA, float* scaleA,+ int8_t* B, float* scaleB, float* C, int lda, int ldsa, int ldb, int ldsb,+ int ldc) {+ int kblk = utils::padto_le(k, kblock);+#pragma omp parallel for+ for (int j = 0; j < m; j++) {+ for (int i = 0; i < n; i += 1) {+ float tmp = 0.f;+ int ik = 0;+ for (; ik < kblk; ik += kblock) {+ int stmp = 0;+ for (int ikk = 0; ikk < kblock; ikk++) {+ stmp += (int(A[(ik + ikk) + j * lda]) - int(zpA[j * ldsa + ik / kblock])) * int(B[(ik + ikk) * ldb + i]);+ }+ tmp += stmp * scaleA[j * ldsa + ik / kblock] * scaleB[ik / kblock * ldsb + i];+ }+ if (ik < k) {+ int stmp = 0;+ for (; ik < k; ik++) {+ stmp += (int(A[ik + j * lda]) - int(zpA[j * ldsa + ik / kblock])) * int(B[ik * ldb + i]);+ }+ tmp += stmp * scaleA[j * ldsa + ik / kblock] * scaleB[ik / kblock * ldsb + i];+ }+ C[i + j * ldc] = tmp;+ }+ }+}++static inline void kblockgemmref_u8zp_s8_f32(int m, int n, int k, int kblock, uint8_t* A, uint8_t* zpA, float* scaleA,+ int8_t* B, utils::bf16* scaleB, float* C, int lda, int ldsa, int ldb,+ int ldsb, int ldc) {+#pragma omp parallel for+ for (int j = 0; j < m; j++) {+ for (int i = 0; i < n; i += 1) {+ float tmp = 0.f;+ for (int ik = 0; ik < k; ik += kblock) {+ int stmp = 0;+ for (int ikk = 0; ikk < kblock; ikk++) {+ stmp += (int(A[(ik + ikk) + j * lda]) - int(zpA[j * ldsa + ik / kblock])) * int(B[(ik + ikk) * ldb + i]);+ }+ tmp += stmp * scaleA[j * ldsa + ik / kblock] * scaleB[ik / kblock * ldsb + i].tofloat();+ }+ C[i + j * ldc] = tmp;+ }+ }+}++struct UT_GEMMData_Row_bf16 {+ UT_GEMMData_Row_bf16(int m, int n, int k, int lda, int ldb, int ldc, int ldd)+ : M(m), N(n), K(k), LDA(lda), LDB(ldb), LDC(ldc), LDD(ldd) {+ matA.resize(m * lda);+ matB.resize(k * ldb);+ matC.resize(m * ldc);+ if (ldd == 0) {+ matD.resize(n);+ } else {+ matD.resize(m * ldd);+ }+ for (size_t i = 0; i < matA.size(); i++) {+ matA[i] = utils::cast<float, utils::bf16>(ut::randn(-0.5f, 0.5f));+ }+ for (size_t i = 0; i < matB.size(); i++) {+ matB[i] = utils::cast<float, utils::bf16>(ut::randn(-0.5f, 0.5f));+ }+ for (size_t i = 0; i < matD.size(); i++) {+ matD[i] = utils::cast<float, utils::bf16>(ut::randn(1.f, 10.f));+ }+ }++ void calc_ref(float alpha, float beta) {+#pragma omp parallel for+ for (int i = 0; i < M; i++) {+ for (int j = 0; j < N; j++) {+ auto tmp = 0.f;+ for (int ik = 0; ik < K; ik++) {+ auto tmpA = utils::cast<utils::bf16, float>(utils::bf16{matA.data()[i * LDA + ik]});+ auto tmpB = utils::cast<utils::bf16, float>(utils::bf16{matB.data()[ik * LDB + j]});+ tmp += tmpA * tmpB;+ }+ tmp = tmp * alpha + utils::cast<utils::bf16, float>({matD.data()[i * LDD + j]}) * beta;+ matC.data()[i * LDC + j] = utils::cast<float, utils::bf16>(tmp);+ }+ }+ }+ utils::aligned_vector<utils::bf16> matA, matC, matD;+ utils::aligned_vector<utils::bf16> matB;+ int M, N, K, LDA, LDB, LDC, LDD;+};++static inline void gemmref_fp32fp32fp32(int m, int n, int k, float* A, float* B, float* C, int lda, int ldb, int ldc,+ float* bias = nullptr, bool BT = false) {+#pragma omp parallel for+ for (int j = 0; j < m; j++) {+ for (int i = 0; i < n; i += 1) {+ float tmp = bias ? bias[i] : 0.f;+ if (BT) {+ for (int ik = 0; ik < k; ik++) {+ tmp += A[ik + j * lda] * B[ik + i * ldb];+ }+ } else {+ for (int ik = 0; ik < k; ik++) {+ tmp += A[ik + j * lda] * B[ik * ldb + i];+ }+ }++ C[i + j * ldc] = tmp;+ }+ }+}++static inline void gemmref_bf16bf16bf16(int m, int n, int k, utils::bf16* A, utils::bf16* B, utils::bf16* C, int lda,+ int ldb, int ldc, utils::bf16* bias = nullptr, bool BT = false) {+#pragma omp parallel for+ for (int j = 0; j < m; j++) {+ for (int i = 0; i < n; i += 1) {+ float tmp = bias ? float(bias[i]) : 0.f;+ if (BT) {+ for (int ik = 0; ik < k; ik++) {+ tmp += float(A[ik + j * lda]) * float(B[ik + i * ldb]);+ }+ } else {+ for (int ik = 0; ik < k; ik++) {+ tmp += float(A[ik + j * lda]) * float(B[ik * ldb + i]);+ }+ }+ C[i + j * ldc] = tmp;+ }+ }+}+static inline void gemmref_bf16bf16fp32(int m, int n, int k, utils::bf16* A, utils::bf16* B, float* C, int lda, int ldb,+ int ldc) {+#pragma omp parallel for+ for (int j = 0; j < m; j++) {+ for (int i = 0; i < n; i += 1) {+ float tmp = 0;+ for (int ik = 0; ik < k; ik++) {+ tmp += float(A[ik + j * lda]) * float(B[ik * ldb + i]);+ }+ C[i + j * ldc] = tmp;+ }+ }+}++static inline void gemmref_fp16fp16fp16(int m, int n, int k, utils::fp16* A, utils::fp16* B, utils::fp16* C, int lda,+ int ldb, int ldc, utils::fp16* bias = nullptr, bool BT = false) {+#pragma omp parallel for+ for (int j = 0; j < m; j++) {+ for (int i = 0; i < n; i += 1) {+ float tmp = bias ? float(bias[i]) : 0.f;+ if (BT) {+ for (int ik = 0; ik < k; ik++) {+ tmp += float(A[ik + j * lda]) * float(B[ik + i * ldb]);+ }+ } else {+ for (int ik = 0; ik < k; ik++) {+ tmp += float(A[ik + j * lda]) * float(B[ik * ldb + i]);+ }+ }++ C[i + j * ldc] = tmp;+ }+ }+}++static inline void gemmref_fp16fp16fp32(int m, int n, int k, utils::fp16* A, utils::fp16* B, float* C, int lda, int ldb,+ int ldc) {+#pragma omp parallel for+ for (int j = 0; j < m; j++) {+ for (int i = 0; i < n; i += 1) {+ float tmp = 0;+ for (int ik = 0; ik < k; ik++) {+ tmp += float(A[ik + j * lda]) * float(B[ik * ldb + i]);+ }+ C[i + j * ldc] = tmp;+ }+ }+}++struct UT_GEMMData_Row_fp16 {+ utils::aligned_vector<utils::fp16> matA, matB, matC, matD;+ int M, N, K, LDA, LDB, LDC, LDD;+ UT_GEMMData_Row_fp16(int m, int n, int k, int lda, int ldb, int ldc, int ldd)+ : M(m), N(n), K(k), LDA(lda), LDB(ldb), LDC(ldc), LDD(ldd) {+ matA.resize(m * lda);+ matB.resize(k * ldb);+ matC.resize(m * ldc);+ if (ldd == 0) {+ matD.resize(n);+ } else {+ matD.resize(m * ldd);+ }+ ut::fill_buffer_randn(matA.data(), m * lda, utils::fp16(-0.5f), utils::fp16(0.5f));+ ut::fill_buffer_randn(matB.data(), k * ldb, utils::fp16(-0.5f), utils::fp16(0.5f));+ ut::fill_buffer_randn(matD.data(), matD.size(), utils::fp16(0.f), utils::fp16(1.f));+ }++ void calc_ref(float alpha, float beta) {+#pragma omp parallel for+ for (int i = 0; i < M; i++) {+ for (int j = 0; j < N; j++) {+ utils::fp16 tmp = utils::fp16(0.f);+ for (int ik = 0; ik < K; ik++) {+ auto tmpA = utils::cast<utils::fp16, float>(matA.data()[i * LDA + ik]);+ auto tmpB = utils::cast<utils::fp16, float>(matB.data()[ik * LDB + j]);+ auto ret = utils::fp16(tmpA * tmpB);+ tmp = utils::fp16(float(tmp) + float(ret));+ }+ float ftmp = float(tmp);+ ftmp = ftmp * alpha + utils::cast<utils::fp16, float>(matD.data()[i * LDD + j]) * beta;+ matC.data()[i * LDC + j] = utils::cast<float, utils::fp16>(ftmp);+ }+ }+ }+};++struct UT_GEMMData_Row_f32 {+ utils::aligned_vector<float> matA, matB, matC, matD, matRef;+ int M, N, K, LDA, LDB, LDC, LDD;+ UT_GEMMData_Row_f32(int m, int n, int k, int lda, int ldb, int ldc, int ldd)+ : M(m), N(n), K(k), LDA(lda), LDB(ldb), LDC(ldc), LDD(ldd) {+ matA.resize(m * lda);+ matB.resize(k * ldb);+ matC.resize(m * ldc);+ if (ldd == 0) {+ matD.resize(n);+ } else {+ matD.resize(m * ldd);+ }+ ut::fill_buffer_randn(matA.data(), m * lda, -0.5f, 0.5f);+ ut::fill_buffer_randn(matB.data(), k * ldb, -0.5f, 0.5f);+ ut::fill_buffer_randn(matD.data(), matD.size(), 0.f, 1.f);+ }++ void calc_ref(float alpha, float beta) {+ matRef.resize(matC.size());+ ref_NN_f32(matA.data(), matB.data(), matRef.data(), matD.data(), M, N, K, LDA, LDB, LDC, LDD, alpha, beta);+ }+ static void ref_NN_f32(float* matA, float* matB, float* matC, float* matD, int m, int n, int k, int lda, int ldb,+ int ldc, int ldd, float alpha, float beta) {+ int NBlock = 128;+#if 1+#pragma omp parallel for+ for (int i = 0; i < n; i += NBlock) {+ for (int j = 0; j < m; j++) {+ int remainn = i + NBlock <= n ? NBlock : n - i;+ for (int ii = 0; ii < remainn; ii++) {+ auto tmp = 0.f;+ for (int ik = 0; ik < k; ik++) {+ tmp += matA[ik + j * lda] * matB[ik * ldb + i + ii];+ }+ tmp = tmp * alpha + matD[(i + ii) + j * ldd] * beta;+ matC[(i + ii) + j * ldc] = tmp;+ }+ }+ }+#else+#pragma omp parallel for+ for (int i = 0; i < n; i += 1) {+ for (int j = 0; j < m; j++) {+ auto tmp = 0.f;+ for (int ik = 0; ik < k; ik++) {+ tmp += matA[ik + j * lda] * matB[ik * ldb + i];+ }+ tmp = tmp * alpha + matD[i + j * ldd] * beta;+ matC[i + j * ldc] = tmp;+ }+ }+#endif+ }+};++static inline void sdpa_ref(int head_num, int head_kv_num, int seq_cur, int seq_kv, int head_idim, int head_odim,+ float scale, float* q_p, float* k_p, float* v_p, float* out, float* mask,+ bool debug = false) {+ auto n_repeat = head_num / head_kv_num;+ // batch matmul host_Q and trans_K+ std::vector<float> qk_tmp(head_num * seq_cur * seq_kv);+ for (size_t i = 0; i < head_num; i++) {+ int i_kv = i / n_repeat;+ for (size_t j = 0; j < seq_cur; j++) {+ for (size_t k = 0; k < seq_kv; k++) {+ float sum = 0;+ for (size_t l = 0; l < head_idim; l++) {+ sum += q_p[i * head_idim + j * head_num * head_idim + l] *+ k_p[i_kv * head_idim + k * head_kv_num * head_idim + l];+ }+ qk_tmp[i * seq_cur * seq_kv + j * seq_kv + k] =+ debug ? sum : sum * scale + mask[j * seq_kv + k]; // mul sqrt_dk+ }+ }+ }+ if (!debug) {+ // batch softmax host_tmp+ for (size_t i = 0; i < head_num; i++) {+ for (size_t j = 0; j < seq_cur; j++) {+ float sum = 0;+ float max = std::numeric_limits<float>::min();+ for (size_t k = 0; k < seq_kv; k++)+ max =+ qk_tmp[i * seq_cur * seq_kv + j * seq_kv + k] > max ? qk_tmp[i * seq_cur * seq_kv + j * seq_kv + k] : max;+ for (size_t k = 0; k < seq_kv; k++) {+ qk_tmp[i * seq_cur * seq_kv + j * seq_kv + k] = std::exp(qk_tmp[i * seq_cur * seq_kv + j * seq_kv + k] - max);+ sum += qk_tmp[i * seq_cur * seq_kv + j * seq_kv + k];+ }+ sum = 1.f / sum;+ for (size_t k = 0; k < seq_kv; k++) {+ qk_tmp[i * seq_cur * seq_kv + j * seq_kv + k] = qk_tmp[i * seq_cur * seq_kv + j * seq_kv + k] * sum;+ }+ }+ }+ }++ // batch matmul host_tmp and host_v+ for (size_t i = 0; i < head_num; i++) {+ auto i_kv = i / n_repeat;+ for (size_t j = 0; j < seq_cur; j++) {+ for (size_t k = 0; k < head_odim; k++) {+ float sum = 0;+ for (size_t l = 0; l < seq_kv; l++) {+ sum +=+ qk_tmp[i * seq_cur * seq_kv + j * seq_kv + l] * v_p[i_kv * head_odim + l * head_odim * head_kv_num + k];+ }+ out[i * head_odim + j * head_num * head_odim + k] = sum;+ }+ }+ }+}++}; // namespace ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_utils.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_utils.cppnew file mode 100644index 000000000..bf668b374--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_utils.cpp@@ -0,0 +1,4 @@++namespace bestla {+namespace ut {} // namespace ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_wrapper.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_wrapper.cppnew file mode 100644index 000000000..472956558--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/bestla_wrapper.cpp@@ -0,0 +1,369 @@+#include "bestla_wrapper.h"+#include "bestla_ut.h"+namespace bestla {+using namespace utils;+namespace ut {+class UT_Fp32Fp32 {+ public:+ UT_Fp32Fp32() {+ UT_START();+ CheckISA(AVX2);+ ut<sAVX2>(1, 1, 1);+ ut<sAVX2>(8, 48, 2);+ ut<sAVX2>(8, 4096, 4096);+ ut<sAVX2>(384, 768, 768);+ ut<sAVX2>(1024, 1024, 1024);+ ut<sAVX2>(1024, 1536, 1536);++ CheckISA(AVX512F);+ ut<gemm::SCoreRowNAvx512f<48, 8>>(384, 768, 768);+ ut<gemm::SCoreRowNAvx512f<48, 8>>(1024, 1024, 1024);+ ut<sAVX512F>(1, 1, 1);+ ut<sAVX512F>(8, 48, 2);+ ut<sAVX512F>(8, 4096, 4096);+ ut<sAVX512F>(384, 768, 768);+ ut<sAVX512F>(1024, 1024, 1024);+ ut<sAVX512F>(1024, 1536, 1536);+ }++ template <class GemmCore_T>+ void ut_file() {+ int m = 9;+ int n = 4096;+ int k = 4096;+ printf("Test Case: %d %d %d Core:%s\n", m, n, k, gemm::CoreAttr::to_str(sAVX2::ID));+ avector<float> matB(k * n), matBT2(k * n), matC(m * n), ref(m * n);+ auto matA2 = readFile2Buffer<float>("act.bin2");+ auto matA = readFile2Buffer<float>("act.bin");+ ut::buffer_error(matA.data(), matA2.data(), matA2.size(), 0.001f);++ auto matBT = readFile2Buffer<utils::fp16>("wei.bin");+ avector<float> matBf32(n * k);+ for (size_t i = 0; i < matBT.size(); i++) {+ matBf32[i] = float(matBT[i]);+ }+ auto bBuf = readFile2Buffer<int8_t>("wei.bin2");+ using Launcher =+ wrapper::gemm::LauncherBaseV2<sAVX2, prologue_a::gemm::ActivationBase, prologue_b::gemm::WeightNPack,+ epilogue::gemm::AccumulatorWriteBackFp32>;++ using Parallel = parallel::gemm::SchedulerBase<sAVX2>;++ auto packw = Launcher::PrologueB::create_storage(n, k);+ auto dbuf = packw.deserialize(bBuf.data());+ packw.set_buffers(dbuf);++ storage::gemm::StorageWeight weiNT, weiT;+ weiNT.set_nt_data(n, k, n, BTLA_DTYPE::F32, matB.data());+ weiT.set_plain_data(n, k, k, BTLA_DTYPE::F32, matBT2.data());+ Launcher::PrologueB::convert(packw, weiNT, UT_Threading::get());+ Launcher::PrologueB::convert(weiNT, weiT, UT_Threading::get());+ ut::buffer_error(matBf32.data(), matBT2.data(), matBT2.size(), 0.001f);+ gemmref_fp32fp32fp32(m, n, k, matA.data(), matB.data(), ref.data(), k, n, n);+ utils::GemmProblem gp(1, m, n, k);+ typename Launcher::Param args{gp, {matA.data(), k}, {matB.data(), n, &packw}, {matC.data(), n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ ut::buffer_error(ref.data(), matC.data(), ref.size(), 0.001f);+ }++ template <class GemmCore_T>+ void ut(int m, int n, int k) {+ printf("Test Case: %d %d %d Core:%s\n", m, n, k, gemm::CoreAttr::to_str(GemmCore_T::ID));+ avector<float> matA(m * k), matB(k * n), matC(m * n), ref(m * n);+ fill_buffer_randn(matA.data(), matA.size(), -0.5f, 0.5f);+ fill_buffer_randn(matB.data(), matB.size(), -0.5f, 0.5f);+ gemmref_fp32fp32fp32(m, n, k, matA.data(), matB.data(), ref.data(), k, n, n);+ using Launcher =+ wrapper::gemm::LauncherBase<GemmCore_T, prologue_a::gemm::ActivationBase, prologue_b::gemm::WeightPack,+ epilogue::gemm::AccumulatorWriteBackFp32>;++ using Parallel = parallel::gemm::SchedulerBase<GemmCore_T>;++ auto packw = Launcher::PrologueB::createStorage(n, k);+ avector<int8_t> buffer(packw.mSize);+ packw.assign(buffer.data());+ Launcher::PrologueB::packWeight(n, k, {matB.data(), n, &packw}, UT_Threading::get());+ utils::GemmProblem gp(1, m, n, k);+ typename Launcher::Param args{gp, {matA.data(), k}, {matB.data(), n, &packw}, {matC.data(), n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ ut::buffer_error(ref.data(), matC.data(), ref.size(), 0.001f);+ }+};+#ifdef BTLA_UT_WRAPPER+static UT_Fp32Fp32 sUT_Fp32Fp32;+#endif++class UT_U8S8S32 {+ public:+ UT_U8S8S32() {+ UT_START();+ GetCPUDevice();+ if (_cd->AVX512_VNNI()) {+ ut<sAVX512_VNNI>(4, 48, 4);+ ut<sAVX512_VNNI>(1, 1, 1);+ ut<sAVX512_VNNI>(8, 48, 2);+ ut<sAVX512_VNNI>(8, 4096, 4096);+ ut<sAVX512_VNNI>(384, 768, 768);+ ut<sAVX512_VNNI>(1024, 1024, 1024);+ ut<sAVX512_VNNI>(1024, 1536, 1536);+ }+ if (_cd->AVX512BW()) {+ ut<sAVX512BW>(4, 48, 4);+ ut<sAVX512BW>(1, 1, 1);+ ut<sAVX512BW>(8, 48, 2);+ ut<sAVX512BW>(8, 4096, 4096);+ ut<sAVX512BW>(384, 768, 768);+ ut<sAVX512BW>(1024, 1024, 1024);+ ut<sAVX512BW>(1024, 1536, 1536);+ }+ if (_cd->AVX_VNNI()) {+ ut<sAVX_VNNI>(1, 1, 1);+ ut<sAVX_VNNI>(8, 48, 2);+ ut<sAVX_VNNI>(8, 4096, 4096);+ ut<sAVX_VNNI>(384, 768, 768);+ ut<sAVX_VNNI>(1024, 1024, 1024);+ ut<sAVX_VNNI>(1024, 1536, 1536);+ }+ if (_cd->AMX_INT8()) {+ request_perm_xtile_data();+ ut<sAMX_INT8_US>(1, 1, 1);+ ut<sAMX_INT8_US>(8, 48, 2);+ ut<sAMX_INT8_US>(8, 4096, 4096);+ ut<sAMX_INT8_US>(384, 768, 768);+ ut<sAMX_INT8_US>(1024, 1024, 1024);+ ut<sAMX_INT8_US>(1024, 1536, 1536);+ }+ }++ template <class GemmCore_T>+ void ut(int m, int n, int k) {+ printf("Test Case: %d %d %d Core:%s\n", m, n, k, gemm::CoreAttr::to_str(GemmCore_T::ID));+ avector<float> matBf32(k * n), matAf32(m * k), matC(m * n), refC(m * n);+ avector<uint8_t> matAu8(m * k), zpAu8(m);+ avector<int8_t> matBs8(k * n);+ avector<float> scaleAf32(m), scaleBf32(n);+ fill_buffer_randn(matAu8.data(), matAu8.size(), uint8_t(0), uint8_t(127));+ fill_buffer_randn(zpAu8.data(), zpAu8.size(), uint8_t(60), uint8_t(64));+ fill_buffer_randn(matBs8.data(), matBs8.size(), int8_t(-127), int8_t(127));+ fill_buffer_randn(scaleAf32.data(), scaleAf32.size(), 0.001f, 0.005f);+ fill_buffer_randn(scaleBf32.data(), scaleBf32.size(), 0.001f, 0.005f);+ for (size_t i = 0; i < m; i++) {+ for (size_t j = 0; j < k; j++) {+ matAf32[i * k + j] = (float(matAu8[i * k + j]) - zpAu8[i]) * scaleAf32[i];+ }+ }+ avector<int32_t> reduceB(n, 0);+ for (size_t i = 0; i < k; i++) {+ for (size_t j = 0; j < n; j++) {+ matBf32[i * n + j] = (float(matBs8[i * n + j])) * scaleBf32[j];+ reduceB[j] += matBs8[i * n + j];+ }+ }+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), matBf32.data(), refC.data(), k, n, n);+ using Launcher = wrapper::gemm::LauncherBase<GemmCore_T, prologue_a::gemm::ActivationBase,+ prologue_b::gemm::WeightPack, epilogue::gemm::ZpDequantInt32ToFp32>;++ using Parallel = parallel::gemm::SchedulerBase<GemmCore_T>;++ auto packw = Launcher::PrologueB::createStorage(n, k);+ avector<int8_t> buffer(packw.mSize);+ packw.assign(buffer.data());+ Launcher::PrologueB::packWeight(n, k, {matBs8.data(), n, &packw}, UT_Threading::get());+ utils::GemmProblem gp(1, m, n, k);+ typename Launcher::Param args{+ gp,+ {matAu8.data(), k},+ {matBs8.data(), n, &packw},+ {matC.data(), n, 1, scaleAf32.data(), scaleBf32.data(), zpAu8.data(), reduceB.data()}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ ut::buffer_error(refC.data(), matC.data(), refC.size(), 0.001f);+ }+};+#ifdef BTLA_UT_WRAPPER+static UT_U8S8S32 sUT_U8S8S32;+#endif++class UT_S8S8S32 {+ public:+ UT_S8S8S32() {+ UT_START();+ GetCPUDevice();+ if (_cd->AVX_VNNI()) {+ ut<sAVX_VNNI_SS>(1, 1, 1);+ ut<sAVX_VNNI_SS>(8, 48, 2);+ ut<sAVX_VNNI_SS>(8, 4096, 4096);+ ut<sAVX_VNNI_SS>(384, 768, 768);+ }+ if (_cd->AMX_INT8()) {+ ut<sAMX_INT8_SS>(1, 1, 1);+ ut<sAMX_INT8_SS>(8, 48, 2);+ ut<sAMX_INT8_SS>(8, 4096, 4096);+ ut<sAMX_INT8_SS>(384, 768, 768);+ ut<sAMX_INT8_SS>(1024, 1024, 1024);+ ut<sAMX_INT8_SS>(1024, 1536, 1536);+ }+ }+ template <class GemmCore_T>+ void ut(int m, int n, int k) {+ printf("Test Case: %d %d %d Core:%s\n", m, n, k, gemm::CoreAttr::to_str(GemmCore_T::ID));+ avector<float> matBf32(k * n), matAf32(m * k), matC(m * n), refC(m * n);+ avector<int8_t> matAu8(m * k);+ avector<int8_t> matBs8(k * n);+ avector<float> scaleAf32(m), scaleBf32(n);+ fill_buffer_randn(matAu8.data(), matAu8.size(), int8_t(-127), int8_t(127));+ fill_buffer_randn(matBs8.data(), matBs8.size(), int8_t(-127), int8_t(127));+ fill_buffer_randn(scaleAf32.data(), scaleAf32.size(), 0.001f, 0.005f);+ fill_buffer_randn(scaleBf32.data(), scaleBf32.size(), 0.001f, 0.005f);+ for (size_t i = 0; i < m; i++) {+ for (size_t j = 0; j < k; j++) {+ matAf32[i * k + j] = (float(matAu8[i * k + j])) * scaleAf32[i];+ }+ }+ avector<float> reduceB(n, 0);+ for (size_t i = 0; i < k; i++) {+ for (size_t j = 0; j < n; j++) {+ matBf32[i * n + j] = (float(matBs8[i * n + j])) * scaleBf32[j];+ reduceB[j] += matBf32[i * n + j];+ }+ }+ gemmref_fp32fp32fp32(m, n, k, matAf32.data(), matBf32.data(), refC.data(), k, n, n);+ using Launcher = wrapper::gemm::LauncherBase<GemmCore_T, prologue_a::gemm::ActivationBase,+ prologue_b::gemm::WeightPack, epilogue::gemm::DequantInt32ToFp32>;++ using Parallel = parallel::gemm::SchedulerBase<GemmCore_T>;++ auto packw = Launcher::PrologueB::createStorage(n, k);+ avector<int8_t> buffer(packw.mSize);+ packw.assign(buffer.data());+ Launcher::PrologueB::packWeight(n, k, {matBs8.data(), n, &packw}, UT_Threading::get());+ utils::GemmProblem gp(1, m, n, k);+ typename Launcher::Param args{+ gp, {matAu8.data(), k}, {matBs8.data(), n, &packw}, {matC.data(), n, 1, scaleAf32.data(), scaleBf32.data()}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ ut::buffer_error(refC.data(), matC.data(), refC.size(), 0.001f);+ }+};+#ifdef BTLA_UT_WRAPPER+static UT_S8S8S32 sUT_S8S8S32;+#endif++class UT_Bf16Bf16Fp32 {+ public:+ UT_Bf16Bf16Fp32() {+ UT_START();+ CheckISA(AMX_BF16);+ ut<sAMX_BF16>(1, 1, 1);+ ut<sAMX_BF16>(8, 48, 2);+ ut<sAMX_BF16>(8, 4096, 4096);+ ut<sAMX_BF16>(384, 768, 768);+ ut<sAMX_BF16>(1024, 1024, 1024);+ ut<sAMX_BF16>(1024, 1536, 1536);+ }++ template <class GemmCore_T>+ void ut(int m, int n, int k) {+ printf("Test Case %s: %d %d %d core:%s\n", __FUNCTION__, m, n, k, gemm::CoreAttr::to_str(GemmCore_T::ID));+ using Launcher =+ wrapper::gemm::LauncherBase<GemmCore_T, prologue_a::gemm::ActivationBase, prologue_b::gemm::WeightPack,+ epilogue::gemm::AccumulatorWriteBackFp32>;++ using Parallel = parallel::gemm::SchedulerBase<GemmCore_T>;+ auto packw = Launcher::PrologueB::createStorage(n, k);+ avector<int8_t> buffer(packw.mSize);+ packw.assign(buffer.data());+ avector<utils::bf16> matAbf16(m * k), matBbf16(k * n);+ fill_buffer_randn(matAbf16.data(), matAbf16.size(), utils::bf16(-0.5f), utils::bf16(0.5f));+ fill_buffer_randn(matBbf16.data(), matBbf16.size(), utils::bf16(-0.5f), utils::bf16(0.5f));+ avector<float> matC(m * n), refC(m * n);+ Launcher::PrologueB::packWeight(n, k, {matBbf16.data(), n, &packw}, UT_Threading::get());+ gemmref_bf16bf16fp32(m, n, k, matAbf16.data(), matBbf16.data(), refC.data(), k, n, n);+ utils::GemmProblem gp(1, m, n, k);+ typename Launcher::Param args{gp, {matAbf16.data(), k}, {matBbf16.data(), n, &packw}, {matC.data(), n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ buffer_error(refC.data(), matC.data(), refC.size(), 0.05f);+ }+};+#ifdef BTLA_UT_WRAPPER+static UT_Bf16Bf16Fp32 sUT_Bf16Bf16Fp32;+#endif++class UT_Fp16Fp16Fp16 {+ public:+ UT_Fp16Fp16Fp16() {+ UT_START();+ CheckISA(AVX512_FP16);+ ut<sAVX512_FP16>(1, 1, 1);+ ut<sAVX512_FP16>(8, 48, 2);+ ut<sAVX512_FP16>(8, 4096, 4096);+ ut<sAVX512_FP16>(384, 768, 768);+ ut<sAVX512_FP16>(1024, 1024, 1024);+ ut<sAVX512_FP16>(1024, 1536, 1536);+ }++ template <class GemmCore_T>+ void ut(int m, int n, int k) {+ printf("Test Case %s: %d %d %d core:%s\n", __FUNCTION__, m, n, k, gemm::CoreAttr::to_str(GemmCore_T::ID));+ using Launcher =+ wrapper::gemm::LauncherBase<GemmCore_T, prologue_a::gemm::ActivationBase, prologue_b::gemm::WeightPack,+ epilogue::gemm::AccumulatorWriteBackFp16>;++ using Parallel = parallel::gemm::SchedulerBase<GemmCore_T>;+ auto packw = Launcher::PrologueB::createStorage(n, k);+ avector<int8_t> buffer(packw.mSize);+ packw.assign(buffer.data());+ avector<utils::fp16> matAbf16(m * k), matBbf16(k * n), matC(m * n), refC(m * n);+ fill_buffer_randn(matAbf16.data(), matAbf16.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ fill_buffer_randn(matBbf16.data(), matBbf16.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ Launcher::PrologueB::packWeight(n, k, {matBbf16.data(), n, &packw}, UT_Threading::get());+ gemmref_fp16fp16fp16(m, n, k, matAbf16.data(), matBbf16.data(), refC.data(), k, n, n);+ GemmProblem gp(1, m, n, k);+ typename Launcher::Param args{gp, {matAbf16.data(), k}, {matBbf16.data(), n, &packw}, {matC.data(), n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ buffer_error(refC.data(), matC.data(), refC.size(), utils::fp16(0.0002f * k));+ }+};+#ifdef BTLA_UT_WRAPPER+static UT_Fp16Fp16Fp16 sUT_Fp16Fp16Fp16;+#endif++class UT_Fp16Fp16Fp32 {+ public:+ UT_Fp16Fp16Fp32() {+ UT_START();+ CheckISA(AMX_FP16);+ ut<sAMX_FP16>(1, 1, 1);+ ut<sAMX_FP16>(8, 48, 2);+ ut<sAMX_FP16>(8, 4096, 4096);+ ut<sAMX_FP16>(384, 768, 768);+ ut<sAMX_FP16>(1024, 1024, 1024);+ ut<sAMX_FP16>(1024, 1536, 1536);+ }++ template <class GemmCore_T>+ void ut(int m, int n, int k) {+ printf("Test Case %s: %d %d %d core:%s\n", __FUNCTION__, m, n, k, gemm::CoreAttr::to_str(GemmCore_T::ID));+ using Launcher =+ wrapper::gemm::LauncherBase<GemmCore_T, prologue_a::gemm::ActivationBase, prologue_b::gemm::WeightPack,+ epilogue::gemm::AccumulatorWriteBackFp32>;++ using Parallel = parallel::gemm::SchedulerBase<GemmCore_T>;+ auto packw = Launcher::PrologueB::createStorage(n, k);+ avector<int8_t> buffer(packw.mSize);+ packw.assign(buffer.data());+ avector<utils::fp16> matAbf16(m * k), matBbf16(k * n);+ avector<float> matC(m * n), refC(m * n);+ fill_buffer_randn(matAbf16.data(), matAbf16.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ fill_buffer_randn(matBbf16.data(), matBbf16.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ Launcher::PrologueB::packWeight(n, k, {matBbf16.data(), n, &packw}, UT_Threading::get());+ gemmref_fp16fp16fp32(m, n, k, matAbf16.data(), matBbf16.data(), refC.data(), k, n, n);+ GemmProblem gp(1, m, n, k);+ typename Launcher::Param args{gp, {matAbf16.data(), k}, {matBbf16.data(), n, &packw}, {matC.data(), n}};+ parallel::GemmRun<Parallel, Launcher>(args, UT_Threading::get());+ buffer_error(refC.data(), matC.data(), refC.size(), 0.0002f * k);+ }+};+#ifdef BTLA_UT_WRAPPER+static UT_Fp16Fp16Fp32 sUT_Fp16Fp16Fp32;+#endif+} // namespace ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/kernel_intrin.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/kernel_intrin.cppnew file mode 100644index 000000000..adf68463c--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/kernel_intrin.cpp@@ -0,0 +1,3329 @@+#include "kernel_ut.h"+#include "kernel_avx2.h"+#include "kernel_avx_vnni.h"+#include "kernel_avx512f.h"+#include "kernel_avx512_vnni.h"++namespace bestla {+using namespace utils;+namespace ut {++#if CompileAVX512F()+class UT_avx512_decompress_s1_s8 {+ public:+ UT_avx512_decompress_s1_s8() {+ UT_START();+ CheckISA(AVX512F);+ ut<1, 48>(32);+ ut<4, 48>(32);+ ut<1, 48>(32, true);+ ut<2, 48>(32, true);+ ut<4, 48>(32, true);+ }++ template <int PackRow, int NTILE>+ void ut(int blocksize, bool isasym = false) {+ int row = blocksize * 2;+ int constexpr col = NTILE;+ printf("Test Case %s: %d %d %d\n", __FUNCTION__, row, col, blocksize);+ avector<utils::bit1x8> s1_wei(row * col / 8);++ std::vector<int8_t> s8_wei(col * row);+ std::vector<int8_t> s8_ref(col * row);+ int blks = row / blocksize;+ int row_offset = 8;+ assert(blocksize % 8 == 0);+ std::vector<int8_t> zp(col * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-1), int8_t(0));+ std::vector<int8_t> rev(col * row);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-1), int8_t(0));++ for (int i = 0; i < col * row; i += 8) {+ memcpy(&s8_ref[i], &s8_wei[i], 8 * sizeof(int8_t));+ s1_wei[i / 8].a = s8_wei[i + 0] + 1;+ s1_wei[i / 8].b = s8_wei[i + 1] + 1;+ s1_wei[i / 8].c = s8_wei[i + 2] + 1;+ s1_wei[i / 8].d = s8_wei[i + 3] + 1;+ s1_wei[i / 8].e = s8_wei[i + 4] + 1;+ s1_wei[i / 8].f = s8_wei[i + 5] + 1;+ s1_wei[i / 8].g = s8_wei[i + 6] + 1;+ s1_wei[i / 8].h = s8_wei[i + 7] + 1;+ }+ if (isasym) {+ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] -= zp[i / blocksize * NTILE + j];+ }+ }+ }+ }++ kernel::avx512f::decompress_kblock_s1_s8<PackRow, NTILE>(s1_wei.data(), isasym ? zp.data() : nullptr, rev.data(),+ blocksize, NTILE, 0, 0, row_offset, NTILE, cache,+ CacheSize);+ kernel::avx512f::decompress_kblock_s1_s8<PackRow, NTILE>(+ s1_wei.data() + row_offset * NTILE / 8, isasym ? zp.data() : nullptr, rev.data() + row_offset * NTILE,+ blocksize, NTILE, 0, row_offset, row - row_offset, NTILE, cache, CacheSize);+ ut::buffer_error(s8_ref.data(), rev.data(), rev.size(), int8_t(0));+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+static UT_avx512_decompress_s1_s8 sUT_avx512_decompress_s1_s8;+#endif++class UT_avx512_decompress_s4_s8 {+ public:+ UT_avx512_decompress_s4_s8() {+ UT_START();+ CheckISA(AVX512F);+ ut<1, 48>(32);+ ut<4, 48>(32);+ ut<1, 48>(32, true);+ ut<2, 48>(32, true);+ ut<4, 48>(32, true);+ }++ template <int PackRow, int NTILE>+ void ut(int blocksize, bool isasym = false) {+ int row = blocksize * 2;+ int constexpr col = NTILE;+ printf("Test Case %s: %d %d %d\n", __FUNCTION__, row, col, blocksize);+ std::vector<utils::int4x2> s4_wei(row * col / 2);+ std::vector<int8_t> s8_wei(col * row);+ std::vector<int8_t> s8_ref(col * row);+ int blks = row / blocksize;+ int row_offset = PackRow;+ std::vector<int8_t> zp(col * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-8), int8_t(7));+ std::vector<int8_t> rev(col * row);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-8), int8_t(7));++ for (int i = 0; i < col * row; i += 2) {+ s8_ref[i] = s8_wei[i];+ s8_ref[i + 1] = s8_wei[i + 1];+ s4_wei[i / 2].x = utils::int4x2::convert(s8_wei[i]) + 8;+ s4_wei[i / 2].y = utils::int4x2::convert(s8_wei[i + 1]) + 8;+ }+ if (isasym) {+ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] -= zp[i / blocksize * NTILE + j];+ }+ }+ }+ }++ kernel::avx512f::decompress_kblock_s4_s8<PackRow, NTILE>(s4_wei.data(), isasym ? zp.data() : nullptr, rev.data(),+ blocksize, NTILE, 0, 0, row_offset, NTILE, cache,+ CacheSize);+ kernel::avx512f::decompress_kblock_s4_s8<PackRow, NTILE>(+ s4_wei.data() + row_offset * NTILE / 2, isasym ? zp.data() : nullptr, rev.data() + row_offset * NTILE,+ blocksize, NTILE, 0, row_offset, row - row_offset, NTILE, cache, CacheSize);+ ut::buffer_error(s8_ref.data(), rev.data(), rev.size(), int8_t(0));+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+static UT_avx512_decompress_s4_s8 sUT_avx512_decompress_s4_s8;+#endif++class UT_avx512_decompress_s3_s8 {+ public:+ UT_avx512_decompress_s3_s8() {+ UT_START();+ CheckISA(AVX512F);+ ut<1, 48>(32);+ ut<4, 48>(32);+ ut<1, 48>(32, true);+ ut<2, 48>(32, true);+ ut<4, 48>(32, true);+ }++ template <int PackRow, int NTILE>+ void ut(int blocksize, bool isasym = false) {+ int row = blocksize * 2;+ int constexpr col = NTILE;+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, PackRow, row, col, blocksize, isasym);+ std::vector<utils::bit2x4> s2_wei(row * col / 4);+ avector<utils::bit1x8> s1_wei(row * col / 8);++ std::vector<int8_t> s8_wei(col * row);+ std::vector<int8_t> s8_ref(col * row);+ int blks = row / blocksize;+ int row_offset = 8;+ assert(blocksize % 8 == 0);+ std::vector<int8_t> zp(col * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-4), int8_t(3));+ std::vector<int8_t> rev(col * row);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-4), int8_t(3));++ for (int i = 0; i < col * row; i += 8) {+ memcpy(&s8_ref[i], &s8_wei[i], 8 * sizeof(int8_t));+ s2_wei[i / 4].a = (s8_wei[i + 0] + 4) & 0x3;+ s2_wei[i / 4].b = (s8_wei[i + 1] + 4) & 0x3;+ s2_wei[i / 4].c = (s8_wei[i + 2] + 4) & 0x3;+ s2_wei[i / 4].d = (s8_wei[i + 3] + 4) & 0x3;++ s2_wei[i / 4 + 1].a = (s8_wei[i + 4] + 4) & 0x3;+ s2_wei[i / 4 + 1].b = (s8_wei[i + 5] + 4) & 0x3;+ s2_wei[i / 4 + 1].c = (s8_wei[i + 6] + 4) & 0x3;+ s2_wei[i / 4 + 1].d = (s8_wei[i + 7] + 4) & 0x3;++ s1_wei[i / 8].a = ((s8_wei[i + 0] + 4) & 0x4) >> 2;+ s1_wei[i / 8].b = ((s8_wei[i + 1] + 4) & 0x4) >> 2;+ s1_wei[i / 8].c = ((s8_wei[i + 2] + 4) & 0x4) >> 2;+ s1_wei[i / 8].d = ((s8_wei[i + 3] + 4) & 0x4) >> 2;+ s1_wei[i / 8].e = ((s8_wei[i + 4] + 4) & 0x4) >> 2;+ s1_wei[i / 8].f = ((s8_wei[i + 5] + 4) & 0x4) >> 2;+ s1_wei[i / 8].g = ((s8_wei[i + 6] + 4) & 0x4) >> 2;+ s1_wei[i / 8].h = ((s8_wei[i + 7] + 4) & 0x4) >> 2;+ }+ if (isasym) {+ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] -= zp[i / blocksize * NTILE + j];+ }+ }+ }+ }++ kernel::avx512f::decompress_kblock_s3_s8<PackRow, NTILE>(s2_wei.data(), s1_wei.data(), isasym ? zp.data() : nullptr,+ rev.data(), blocksize, NTILE, 0, 0, row_offset, NTILE,+ cache, CacheSize);+ kernel::avx512f::decompress_kblock_s3_s8<PackRow, NTILE>(+ s2_wei.data() + row_offset * NTILE / 4, s1_wei.data() + row_offset * NTILE / 8, isasym ? zp.data() : nullptr,+ rev.data() + row_offset * NTILE, blocksize, NTILE, 0, row_offset, row - row_offset, NTILE, cache, CacheSize);+ ut::buffer_error(s8_ref.data(), rev.data(), rev.size(), int8_t(0));+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+static UT_avx512_decompress_s3_s8 sUT_avx512_decompress_s3_s8;+#endif++class UT_avx512_decompress_s5_s8 {+ public:+ UT_avx512_decompress_s5_s8() {+ UT_START();+ CheckISA(AVX512F);+ ut<1, 48>(32);+ ut<4, 48>(32);+ ut<1, 48>(32, true);+ ut<2, 48>(32, true);+ ut<4, 48>(32, true);+ }++ template <int PackRow, int NTILE>+ void ut(int blocksize, bool isasym = false) {+ int row = blocksize * 2;+ int constexpr col = NTILE;+ printf("Test Case %s: %d %d %d\n", __FUNCTION__, row, col, blocksize);+ std::vector<utils::bit4x2> s4_wei(row * col / 2);+ avector<utils::bit1x8> s1_wei(row * col / 8);++ std::vector<int8_t> s8_wei(col * row);+ std::vector<int8_t> s8_ref(col * row);+ int blks = row / blocksize;+ int row_offset = 8;+ assert(blocksize % 8 == 0);+ std::vector<int8_t> zp(col * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-16), int8_t(15));+ std::vector<int8_t> rev(col * row);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-16), int8_t(15));++ for (int i = 0; i < col * row; i += 8) {+ memcpy(&s8_ref[i], &s8_wei[i], 8 * sizeof(int8_t));+ s4_wei[i / 2].x = (s8_wei[i + 0] + 16) & 0xf;+ s4_wei[i / 2].y = (s8_wei[i + 1] + 16) & 0xf;+ s4_wei[i / 2 + 1].x = (s8_wei[i + 2] + 16) & 0xf;+ s4_wei[i / 2 + 1].y = (s8_wei[i + 3] + 16) & 0xf;+ s4_wei[i / 2 + 2].x = (s8_wei[i + 4] + 16) & 0xf;+ s4_wei[i / 2 + 2].y = (s8_wei[i + 5] + 16) & 0xf;+ s4_wei[i / 2 + 3].x = (s8_wei[i + 6] + 16) & 0xf;+ s4_wei[i / 2 + 3].y = (s8_wei[i + 7] + 16) & 0xf;++ s1_wei[i / 8].a = ((s8_wei[i + 0] + 16) & 0x10) >> 4;+ s1_wei[i / 8].b = ((s8_wei[i + 1] + 16) & 0x10) >> 4;+ s1_wei[i / 8].c = ((s8_wei[i + 2] + 16) & 0x10) >> 4;+ s1_wei[i / 8].d = ((s8_wei[i + 3] + 16) & 0x10) >> 4;+ s1_wei[i / 8].e = ((s8_wei[i + 4] + 16) & 0x10) >> 4;+ s1_wei[i / 8].f = ((s8_wei[i + 5] + 16) & 0x10) >> 4;+ s1_wei[i / 8].g = ((s8_wei[i + 6] + 16) & 0x10) >> 4;+ s1_wei[i / 8].h = ((s8_wei[i + 7] + 16) & 0x10) >> 4;+ }+ if (isasym) {+ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] -= zp[i / blocksize * NTILE + j];+ }+ }+ }+ }++ kernel::avx512f::decompress_kblock_s5_s8<PackRow, NTILE>(s4_wei.data(), s1_wei.data(), isasym ? zp.data() : nullptr,+ rev.data(), blocksize, NTILE, 0, 0, row_offset, NTILE,+ cache, CacheSize);+ kernel::avx512f::decompress_kblock_s5_s8<PackRow, NTILE>(+ s4_wei.data() + row_offset * NTILE / 2, s1_wei.data() + row_offset * NTILE / 8, isasym ? zp.data() : nullptr,+ rev.data() + row_offset * NTILE, blocksize, NTILE, 0, row_offset, row - row_offset, NTILE, cache, CacheSize);+ ut::buffer_error(s8_ref.data(), rev.data(), rev.size(), int8_t(0));+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+static UT_avx512_decompress_s5_s8 sUT_avx512_decompress_s5_s8;+#endif++class UT_avx512_decompress_s6_s8 {+ public:+ UT_avx512_decompress_s6_s8() {+ UT_START();+ CheckISA(AVX512F);+ ut<1, 48>(32);+ ut<4, 48>(32);+ ut<1, 48>(32, true);+ ut<2, 48>(32, true);+ ut<4, 48>(32, true);+ }++ template <int PackRow, int NTILE>+ void ut(int blocksize, bool isasym = false) {+ int row = blocksize * 2;+ int constexpr FullRange = 1 << (6 - 1);+ int constexpr col = NTILE;+ printf("Test Case %s: %d %d %d\n", __FUNCTION__, row, col, blocksize);+ std::vector<utils::bit4x2> s4_wei(row * col / 2);+ avector<utils::bit2x4> s2_wei(row * col / 4);++ std::vector<int8_t> s8_wei(col * row);+ std::vector<int8_t> s8_ref(col * row);+ int blks = row / blocksize;+ int row_offset = 8;+ assert(blocksize % 8 == 0);+ std::vector<int8_t> zp(col * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-FullRange), int8_t(FullRange - 1));+ std::vector<int8_t> rev(col * row);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-FullRange), int8_t(FullRange - 1));++ for (int i = 0; i < col * row; i += 4) {+ memcpy(&s8_ref[i], &s8_wei[i], 4 * sizeof(int8_t));+ s4_wei[i / 2].x = (s8_wei[i + 0] + FullRange) & 0xf;+ s4_wei[i / 2].y = (s8_wei[i + 1] + FullRange) & 0xf;+ s4_wei[i / 2 + 1].x = (s8_wei[i + 2] + FullRange) & 0xf;+ s4_wei[i / 2 + 1].y = (s8_wei[i + 3] + FullRange) & 0xf;++ s2_wei[i / 4].a = ((s8_wei[i + 0] + FullRange) & 0x30) >> 4;+ s2_wei[i / 4].b = ((s8_wei[i + 1] + FullRange) & 0x30) >> 4;+ s2_wei[i / 4].c = ((s8_wei[i + 2] + FullRange) & 0x30) >> 4;+ s2_wei[i / 4].d = ((s8_wei[i + 3] + FullRange) & 0x30) >> 4;+ }+ if (isasym) {+ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] -= zp[i / blocksize * NTILE + j];+ }+ }+ }+ }++ kernel::avx512f::decompress_kblock_s6_s8<PackRow, NTILE>(s4_wei.data(), s2_wei.data(), isasym ? zp.data() : nullptr,+ rev.data(), blocksize, NTILE, 0, 0, row_offset, NTILE,+ cache, CacheSize);+ kernel::avx512f::decompress_kblock_s6_s8<PackRow, NTILE>(+ s4_wei.data() + row_offset * NTILE / 2, s2_wei.data() + row_offset * NTILE / 4, isasym ? zp.data() : nullptr,+ rev.data() + row_offset * NTILE, blocksize, NTILE, 0, row_offset, row - row_offset, NTILE, cache, CacheSize);+ ut::buffer_error(s8_ref.data(), rev.data(), rev.size(), int8_t(0));+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+static UT_avx512_decompress_s6_s8 sUT_avx512_decompress_s6_s8;+#endif++class UT_avx512_decompress_s7_s8 {+ public:+ UT_avx512_decompress_s7_s8() {+ UT_START();+ CheckISA(AVX512F);+ ut<1, 48>(32);+ ut<4, 48>(32);+ ut<1, 48>(32, true);+ ut<2, 48>(32, true);+ ut<4, 48>(32, true);+ }++ template <int PackRow, int NTILE>+ void ut(int blocksize, bool isasym = false) {+ int row = blocksize * 2;+ int constexpr col = NTILE;+ printf("Test Case %s: %d %d %d\n", __FUNCTION__, row, col, blocksize);+ std::vector<utils::bit4x2> s4_wei(row * col / 2);+ avector<utils::bit1x8> s1_wei(row * col / 8);+ avector<utils::bit2x4> s2_wei(row * col / 4);++ std::vector<int8_t> s8_wei(col * row);+ std::vector<int8_t> s8_ref(col * row);+ int blks = row / blocksize;+ int row_offset = 8;+ assert(blocksize % 8 == 0);+ std::vector<int8_t> zp(col * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-64), int8_t(63));+ std::vector<int8_t> rev(col * row);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-64), int8_t(63));++ for (int i = 0; i < col * row; i += 8) {+ memcpy(&s8_ref[i], &s8_wei[i], 8 * sizeof(int8_t));+ s4_wei[i / 2].x = (s8_wei[i + 0] + 64) & 0xf;+ s4_wei[i / 2].y = (s8_wei[i + 1] + 64) & 0xf;+ s4_wei[i / 2 + 1].x = (s8_wei[i + 2] + 64) & 0xf;+ s4_wei[i / 2 + 1].y = (s8_wei[i + 3] + 64) & 0xf;+ s4_wei[i / 2 + 2].x = (s8_wei[i + 4] + 64) & 0xf;+ s4_wei[i / 2 + 2].y = (s8_wei[i + 5] + 64) & 0xf;+ s4_wei[i / 2 + 3].x = (s8_wei[i + 6] + 64) & 0xf;+ s4_wei[i / 2 + 3].y = (s8_wei[i + 7] + 64) & 0xf;++ s2_wei[i / 4].a = ((s8_wei[i + 0] + 64) & 0x30) >> 4;+ s2_wei[i / 4].b = ((s8_wei[i + 1] + 64) & 0x30) >> 4;+ s2_wei[i / 4].c = ((s8_wei[i + 2] + 64) & 0x30) >> 4;+ s2_wei[i / 4].d = ((s8_wei[i + 3] + 64) & 0x30) >> 4;+ s2_wei[i / 4 + 1].a = ((s8_wei[i + 4] + 64) & 0x30) >> 4;+ s2_wei[i / 4 + 1].b = ((s8_wei[i + 5] + 64) & 0x30) >> 4;+ s2_wei[i / 4 + 1].c = ((s8_wei[i + 6] + 64) & 0x30) >> 4;+ s2_wei[i / 4 + 1].d = ((s8_wei[i + 7] + 64) & 0x30) >> 4;++ s1_wei[i / 8].a = ((s8_wei[i + 0] + 64) & 0x40) >> 6;+ s1_wei[i / 8].b = ((s8_wei[i + 1] + 64) & 0x40) >> 6;+ s1_wei[i / 8].c = ((s8_wei[i + 2] + 64) & 0x40) >> 6;+ s1_wei[i / 8].d = ((s8_wei[i + 3] + 64) & 0x40) >> 6;+ s1_wei[i / 8].e = ((s8_wei[i + 4] + 64) & 0x40) >> 6;+ s1_wei[i / 8].f = ((s8_wei[i + 5] + 64) & 0x40) >> 6;+ s1_wei[i / 8].g = ((s8_wei[i + 6] + 64) & 0x40) >> 6;+ s1_wei[i / 8].h = ((s8_wei[i + 7] + 64) & 0x40) >> 6;+ }+ if (isasym) {+ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] -= zp[i / blocksize * NTILE + j];+ }+ }+ }+ }++ kernel::avx512f::decompress_kblock_s7_s8<PackRow, NTILE>(s4_wei.data(), s2_wei.data(), s1_wei.data(),+ isasym ? zp.data() : nullptr, rev.data(), blocksize, NTILE,+ 0, 0, row_offset, NTILE, cache, CacheSize);+ kernel::avx512f::decompress_kblock_s7_s8<PackRow, NTILE>(+ s4_wei.data() + row_offset * NTILE / 2, s2_wei.data() + row_offset * NTILE / 4,+ s1_wei.data() + row_offset * NTILE / 8, isasym ? zp.data() : nullptr, rev.data() + row_offset * NTILE,+ blocksize, NTILE, 0, row_offset, row - row_offset, NTILE, cache, CacheSize);+ ut::buffer_error(s8_ref.data(), rev.data(), rev.size(), int8_t(0));+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+static UT_avx512_decompress_s7_s8 sUT_avx512_decompress_s7_s8;+#endif++class UT_avx512_decompress_s2_s8 {+ public:+ UT_avx512_decompress_s2_s8() {+ UT_START();+ CheckISA(AVX512F);+ ut<1, 48>(32);+ ut<4, 48>(32);+ ut<1, 48>(32, true);+ ut<2, 48>(32, true);+ ut<4, 48>(32, true);+ }++ template <int PackRow, int NTILE>+ void ut(int blocksize, bool isasym = false) {+ int row = blocksize * 2;+ int constexpr col = NTILE;+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, PackRow, row, col, blocksize, isasym);+ std::vector<utils::bit2x4> s2_wei(row * col / 4);++ std::vector<int8_t> s8_wei(col * row);+ std::vector<int8_t> s8_ref(col * row);+ int blks = row / blocksize;+ int row_offset = blocksize;+ assert(blocksize % 8 == 0);+ std::vector<int8_t> zp(col * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-2), int8_t(1));+ std::vector<int8_t> rev(col * row);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-2), int8_t(1));++ for (int i = 0; i < col * row; i += 4) {+ memcpy(&s8_ref[i], &s8_wei[i], 4 * sizeof(int8_t));+ s2_wei[i / 4].a = (s8_wei[i + 0] + 2) & 0x3;+ s2_wei[i / 4].b = (s8_wei[i + 1] + 2) & 0x3;+ s2_wei[i / 4].c = (s8_wei[i + 2] + 2) & 0x3;+ s2_wei[i / 4].d = (s8_wei[i + 3] + 2) & 0x3;+ }+ if (isasym) {+ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] -= zp[i / blocksize * NTILE + j];+ }+ }+ }+ }++ kernel::avx512f::decompress_kblock_s2_s8<PackRow, NTILE>(s2_wei.data(), isasym ? zp.data() : nullptr, rev.data(),+ blocksize, NTILE, 0, 0, row_offset, NTILE, cache,+ CacheSize);+ kernel::avx512f::decompress_kblock_s2_s8<PackRow, NTILE>(+ s2_wei.data() + row_offset * NTILE / 4, isasym ? zp.data() : nullptr, rev.data() + row_offset * NTILE,+ blocksize, NTILE, 0, row_offset, row - row_offset, NTILE, cache, CacheSize);+ ut::buffer_error(s8_ref.data(), rev.data(), rev.size(), int8_t(0));+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+static UT_avx512_decompress_s2_s8 sUT_avx512_decompress_s2_s8;+#endif++class UT_avx512_decompress_s4_fp {+ public:+ UT_avx512_decompress_s4_fp() {+ UT_START();+ CheckISA(AVX512F);+ ut<1, 48, float>(32);+ ut<2, 48, float>(32);+ ut<4, 48, float>(32);+ ut<4, 48, utils::bf16>(32);+ ut<4, 48, utils::bf16, utils::bf16>(32);+ ut<1, 48, float>(32, true);+ ut<2, 48, float>(32, true);+ ut<4, 48, float>(32, true);+ ut<4, 48, utils::bf16>(32, true);+ ut<4, 48, utils::bf16, utils::bf16>(32, true);+ }++ template <int PackRow, int NTILE, typename T, typename ScaleT = float>+ void ut(int blocksize, bool isasym = false) {+ auto dst_dtype = bestla_dtype<T>;+ auto scale_dtype = bestla_dtype<ScaleT>;+ int row = blocksize * 2;+ int constexpr col = NTILE;+ printf("Test Case %s: %d %d %d Asym:%d Pack:%d %s %s\n", __FUNCTION__, row, col, blocksize, isasym, PackRow,+ utils::bestla_dtype_str(dst_dtype), bestla_dtype_str(scale_dtype));+ std::vector<utils::int4x2> s4_wei(row * col / 2);+ std::vector<int8_t> s8_wei(col * row);+ std::vector<T> s8_ref(col * row);+ int blks = row / blocksize;+ int row_offset = PackRow;+ std::vector<int8_t> zp(col * blks);+ avector<ScaleT> scale(col * blks);+ fill_buffer_randn(scale.data(), scale.size(), ScaleT(0.01f), ScaleT(0.03f));+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-8), int8_t(7));+ std::vector<T> rev(col * row);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-8), int8_t(7));++ for (int i = 0; i < col * row; i += 2) {+ s8_ref[i] = float(s8_wei[i]);+ s8_ref[i + 1] = float(s8_wei[i + 1]);+ s4_wei[i / 2].x = utils::int4x2::convert(s8_wei[i]) + 8;+ s4_wei[i / 2].y = utils::int4x2::convert(s8_wei[i + 1]) + 8;+ }+ if (isasym) {+ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ int corr_offset = i / blocksize * NTILE + j;+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] = float(s8_ref[i * NTILE + j * PackRow + ip]) - float(zp[corr_offset]);+ }+ }+ }+ }++ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ int corr_offset = i / blocksize * NTILE + j;+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] =+ float(s8_ref[i * NTILE + j * PackRow + ip]) * float(scale[corr_offset]);+ }+ }+ }+ kernel::avx512f::decompress_kblock_s4_fp<PackRow, NTILE>(s4_wei.data(), rev.data(), row_offset, NTILE, scale.data(),+ scale_dtype, isasym ? zp.data() : nullptr, 0, 0, blocksize,+ NTILE, cache, CacheSize);+ kernel::avx512f::decompress_kblock_s4_fp<PackRow, NTILE>(+ s4_wei.data() + row_offset * NTILE / 2, rev.data() + row_offset * NTILE, row - row_offset, NTILE, scale.data(),+ scale_dtype, isasym ? zp.data() : nullptr, row_offset, 0, blocksize, NTILE, cache, CacheSize);+ float err = get_ut_err(dst_dtype);+ ut::buffer_error(s8_ref.data(), rev.data(), rev.size(), T(err));+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+static UT_avx512_decompress_s4_fp sUT_avx512_decompress_s4_fp;+#endif+#endif++#if CompileAVX512VNNI()+class UT_avx512_gemv {+ public:+ UT_avx512_gemv() {+ UT_START();+ CheckISA(AVX512F);+ ut_1bit_fp32<1>(48, 128, 32, true);+ ut_1bit_fp32<1>(48, 128, 32, false);+ ut_1bit_fp32<4>(48, 128, 32, true);+ ut_1bit_fp32<4>(48, 128, 32, false);++ ut_7bit_fp32<1>(48, 128, 32, true);+ ut_7bit_fp32<1>(48, 128, 32, false);+ ut_7bit_fp32<4>(48, 128, 32, true);+ ut_7bit_fp32<4>(48, 128, 32, false);++ ut_6bit_fp32<1>(48, 128, 32, true);+ ut_6bit_fp32<1>(48, 128, 32, false);+ ut_6bit_fp32<4>(48, 128, 32, true);+ ut_6bit_fp32<4>(48, 128, 32, false);++ ut_5bit_fp32<1>(48, 128, 32, true);+ ut_5bit_fp32<1>(48, 128, 32, false);+ ut_5bit_fp32<4>(48, 128, 32, true);+ ut_5bit_fp32<4>(48, 128, 32, false);++ ut_4bit_fp32<1>(48, 128, 32, true);+ ut_4bit_fp32<1>(48, 128, 32, false);+ ut_4bit_fp32<4>(48, 128, 32, true);+ ut_4bit_fp32<4>(48, 128, 32, false);++ ut_3bit_fp32<1>(48, 128, 32, true);+ ut_3bit_fp32<1>(48, 128, 32, false);+ ut_3bit_fp32<4>(48, 128, 32, true);+ ut_3bit_fp32<4>(48, 128, 32, false);++ ut_2bit_fp32<1>(48, 128, 32, true);+ ut_2bit_fp32<1>(48, 128, 32, false);+ ut_2bit_fp32<4>(48, 128, 32, true);+ ut_2bit_fp32<4>(48, 128, 32, false);++ CheckISA(AVX512_VNNI);+ ut_1bit_u8s8<1>(48, 128, 32, true);+ ut_1bit_u8s8<1>(48, 128, 32, false);+ ut_1bit_u8s8<4>(48, 128, 32, true);+ ut_1bit_u8s8<4>(48, 128, 32, false);++ ut_1bit_s8s8<1>(48, 128, 32, true);+ ut_1bit_s8s8<1>(48, 128, 32, false);+ ut_1bit_s8s8<4>(48, 128, 32, true);+ ut_1bit_s8s8<4>(48, 128, 32, false);++ ut_4bit<1>(48, 128, 32, true);+ ut_4bit<1>(48, 128, 32, false);+ ut_4bit<4>(48, 128, 32, false);+ ut_4bit<4>(48, 128, 32, true);++ ut_4bit_s8s8<1>(48, 128, 32, true);+ ut_4bit_s8s8<1>(48, 128, 32, false);+ ut_4bit_s8s8<4>(48, 128, 32, true);+ ut_4bit_s8s8<4>(48, 128, 32, false);++ ut_2bit<1>(48, 128, 32, true);+ ut_2bit<1>(48, 128, 32, false);+ ut_2bit<4>(48, 128, 32, true);+ ut_2bit<4>(48, 128, 32, false);++ ut_2bit_s8s8<1>(48, 128, 32, true);+ ut_2bit_s8s8<1>(48, 128, 32, false);+ ut_2bit_s8s8<4>(48, 128, 32, true);+ ut_2bit_s8s8<4>(48, 128, 32, false);++ ut_3bit_u8s8<1>(48, 128, 32, true);+ ut_3bit_u8s8<1>(48, 128, 32, false);+ ut_3bit_u8s8<4>(48, 128, 32, true);+ ut_3bit_u8s8<4>(48, 128, 32, false);++ ut_3bit_s8s8<1>(48, 128, 32, true);+ ut_3bit_s8s8<1>(48, 128, 32, false);+ ut_3bit_s8s8<4>(48, 128, 32, true);+ ut_3bit_s8s8<4>(48, 128, 32, false);++ ut_6bit_u8s8<1>(48, 128, 32, true);+ ut_6bit_u8s8<1>(48, 128, 32, false);+ ut_6bit_u8s8<4>(48, 128, 32, true);+ ut_6bit_u8s8<4>(48, 128, 32, false);++ ut_6bit_s8s8<1>(48, 128, 32, true);+ ut_6bit_s8s8<1>(48, 128, 32, false);+ ut_6bit_s8s8<4>(48, 128, 32, true);+ ut_6bit_s8s8<4>(48, 128, 32, false);++ ut_5bit_u8s8<1>(48, 128, 32, true);+ ut_5bit_u8s8<1>(48, 128, 32, false);+ ut_5bit_u8s8<4>(48, 128, 32, true);+ ut_5bit_u8s8<4>(48, 128, 32, false);++ ut_5bit_s8s8<1>(48, 128, 32, true);+ ut_5bit_s8s8<1>(48, 128, 32, false);+ ut_5bit_s8s8<4>(48, 128, 32, true);+ ut_5bit_s8s8<4>(48, 128, 32, false);++ ut_7bit_s8s8<1>(48, 128, 32, true);+ ut_7bit_s8s8<1>(48, 128, 32, false);+ ut_7bit_s8s8<4>(48, 128, 32, true);+ ut_7bit_s8s8<4>(48, 128, 32, false);++ ut_7bit_u8s8<1>(48, 128, 32, true);+ ut_7bit_u8s8<1>(48, 128, 32, false);+ ut_7bit_u8s8<4>(48, 128, 32, true);+ ut_7bit_u8s8<4>(48, 128, 32, false);+ }++ template <int MTILE>+ void ut_1bit_fp32(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(Af32.data(), Af32.size(), -0.5f, 0.5f);+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-1), int8_t(0));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s1_s8(b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 1) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 4) {+ if (iasym) {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0] - bzp[bid * n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1] - bzp[bid * n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2] - bzp[bid * n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3] - bzp[bid * n + j + 3]) * scaleb[bid * n + j + 3];+ } else {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3]) * scaleb[bid * n + j + 3];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{nullptr, nullptr, (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr,+ 1, n};+ kernel::avx512f::gemv_1bit_fp32_fp32<float, 48, MTILE>(Af32.data(), k, B, Cf32.data(), n, k, kblock, cache,+ CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_1bit_u8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-1), int8_t(0));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<uint8_t> A(MTILE * k), azp(MTILE * blks);+ fill_buffer_randn(A.data(), A.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(azp.data(), azp.size(), uint8_t(100), uint8_t(150));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j]) - azp[bid]) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s1_s8(b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{nullptr, nullptr, (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr,+ 2, n};+ kernel::avx512f::vnni::gemv_1bit_u8s8_fp32<float, 48, MTILE>({A.data(), scalea.data(), azp.data(), k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_1bit_s8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-1), int8_t(0));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> A(MTILE * k);+ fill_buffer_randn(A.data(), A.size(), int8_t(0), int8_t(127));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j])) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s1_s8(b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{nullptr, nullptr, (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr,+ 2, n};+ kernel::avx512f::vnni::gemv_1bit_s8s8_fp32<float, 48, MTILE>({(uint8_t*)A.data(), scalea.data(), nullptr, k, blks},+ B, Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_7bit_fp32(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit1x8> b1(n * k / 8);+ avector<bit2x4> b2(n * k / 4);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(Af32.data(), Af32.size(), -0.5f, 0.5f);+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-64), int8_t(63));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s7_s8(b4.data(), b2.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 1) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 4) {+ if (iasym) {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0] - bzp[bid * n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1] - bzp[bid * n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2] - bzp[bid * n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3] - bzp[bid * n + j + 3]) * scaleb[bid * n + j + 3];+ } else {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3]) * scaleb[bid * n + j + 3];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{(uint8_t*)b4.data(),+ (uint8_t*)b2.data(),+ (uint8_t*)b1.data(),+ scaleb.data(),+ iasym ? bzp.data() : nullptr,+ 7,+ n};+ kernel::avx512f::gemv_7bit_fp32_fp32<float, 48, MTILE>(Af32.data(), k, B, Cf32.data(), n, k, kblock, cache,+ CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_7bit_s8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit2x4> b2(n * k / 4);+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-64), int8_t(63));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> A(MTILE * k);+ fill_buffer_randn(A.data(), A.size(), int8_t(0), int8_t(127));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j])) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s7_s8(b4.data(), b2.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{(uint8_t*)b4.data(),+ (uint8_t*)b2.data(),+ (uint8_t*)b1.data(),+ scaleb.data(),+ iasym ? bzp.data() : nullptr,+ 7,+ n};+ kernel::avx512f::vnni::gemv_7bit_s8s8_fp32<float, 48, MTILE>({(uint8_t*)A.data(), scalea.data(), nullptr, k, blks},+ B, Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_7bit_u8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit2x4> b2(n * k / 4);+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-64), int8_t(63));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<uint8_t> A(MTILE * k), azp(MTILE * blks);+ fill_buffer_randn(A.data(), A.size(), uint8_t(0), uint8_t(128));+ fill_buffer_randn(azp.data(), azp.size(), uint8_t(50), uint8_t(64));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j]) - azp[bid]) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s7_s8(b4.data(), b2.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{(uint8_t*)b4.data(),+ (uint8_t*)b2.data(),+ (uint8_t*)b1.data(),+ scaleb.data(),+ iasym ? bzp.data() : nullptr,+ 7,+ n};+ kernel::avx512f::vnni::gemv_7bit_u8s8_fp32<float, 48, MTILE>({A.data(), scalea.data(), azp.data(), k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_4bit(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b2(n * k / 2);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<uint8_t> A(MTILE * k), azp(MTILE * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(A.data(), A.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(azp.data(), azp.size(), uint8_t(100), uint8_t(150));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j]) - azp[bid]) * scalea[bid];+ }+ }+ }++ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ auto b24 = b2[(i * n + j * 4) / 2];+ auto b42 = b2[(i * n + j * 4 + 2) / 2];+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b24.x - 8) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b24.y - 8) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b42.x - 8) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b42.y - 8) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b24.x - 8)) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b24.y - 8)) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b42.x - 8)) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b42.y - 8)) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{(uint8_t*)b2.data(), nullptr, nullptr, scaleb.data(),+ iasym ? bzp.data() : nullptr, 2, n};+ kernel::avx512f::vnni::gemv_4bit_u8s8_fp32<float, 48, MTILE>({A.data(), scalea.data(), azp.data(), k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_4bit_fp32(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b2(n * k / 2);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(Af32.data(), Af32.size(), -0.5f, 0.5f);+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);++ for (int i = 0; i < k; i += 1) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 2) {+ auto b24 = b2[(i * n + j) / 2];+ if (iasym) {+ Bf32[(i)*n + j + 0] = (int(b24.x - 8) - bzp[bid * n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (int(b24.y - 8) - bzp[bid * n + j + 1]) * scaleb[bid * n + j + 1];+ } else {+ Bf32[(i)*n + j + 0] = (int(b24.x - 8)) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (int(b24.y - 8)) * scaleb[bid * n + j + 1];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{(uint8_t*)b2.data(), nullptr, nullptr, scaleb.data(),+ iasym ? bzp.data() : nullptr, 4, n};+ kernel::avx512f::gemv_4bit_fp32_fp32<float, 48, MTILE>(Af32.data(), k, B, Cf32.data(), n, k, kblock, cache,+ CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_4bit_s8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b2(n * k / 2);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<int8_t> A(MTILE * k);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(A.data(), A.size(), int8_t(-127), int8_t(127));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j])) * scalea[bid];+ }+ }+ }++ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ auto b24 = b2[(i * n + j * 4) / 2];+ auto b42 = b2[(i * n + j * 4 + 2) / 2];+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b24.x - 8) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b24.y - 8) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b42.x - 8) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b42.y - 8) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b24.x - 8)) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b24.y - 8)) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b42.x - 8)) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b42.y - 8)) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{(uint8_t*)b2.data(), nullptr, nullptr, scaleb.data(),+ iasym ? bzp.data() : nullptr, 2, n};+ kernel::avx512f::vnni::gemv_4bit_s8s8_fp32<float, 48, MTILE>({(uint8_t*)A.data(), scalea.data(), nullptr, k, blks},+ B, Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_2bit(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit2x4> b2(n * k / 4);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<uint8_t> A(MTILE * k), azp(MTILE * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(A.data(), A.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-2), int8_t(1));+ fill_buffer_randn(azp.data(), azp.size(), uint8_t(100), uint8_t(150));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int i = 0; i < MTILE; i++) {+ for (int j = 0; j < k; j++) {+ Af32[i * k + j] = (int(A[i * k + j]) - azp[i * blks + j / kblock]) * scalea[i * blks + j / kblock];+ }+ }+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ auto b24 = b2[(i * n + j * 4) / 4];+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b24.a - 2) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b24.b - 2) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b24.c - 2) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b24.d - 2) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b24.a - 2)) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b24.b - 2)) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b24.c - 2)) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b24.d - 2)) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ kernel::avx512f::vnni::gemv_2bit_u8s8_fp32<float, 48, MTILE>(+ {A.data(), scalea.data(), azp.data(), k, blks},+ {nullptr, (uint8_t*)b2.data(), nullptr, scaleb.data(), iasym ? bzp.data() : nullptr, 2, n}, Cf32.data(), n, k,+ kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_2bit_s8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit2x4> b2(n * k / 4);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<int8_t> A(MTILE * k);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(A.data(), A.size(), int8_t(0), int8_t(127));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-2), int8_t(1));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int i = 0; i < MTILE; i++) {+ for (int j = 0; j < k; j++) {+ Af32[i * k + j] = (int(A[i * k + j])) * scalea[i * blks + j / kblock];+ }+ }+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ auto b24 = b2[(i * n + j * 4) / 4];+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b24.a - 2) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b24.b - 2) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b24.c - 2) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b24.d - 2) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b24.a - 2)) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b24.b - 2)) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b24.c - 2)) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b24.d - 2)) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ kernel::avx512f::vnni::gemv_2bit_s8s8_fp32<float, 48, MTILE>(+ {(uint8_t*)A.data(), scalea.data(), nullptr, k, blks},+ {nullptr, (uint8_t*)b2.data(), nullptr, scaleb.data(), iasym ? bzp.data() : nullptr, 2, n}, Cf32.data(), n, k,+ kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_2bit_fp32(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit2x4> b2(n * k / 4);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(Af32.data(), Af32.size(), -0.5f, 0.5f);+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);++ for (int i = 0; i < k; i += 1) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 4) {+ auto b24 = b2[(i * n + j) / 4];+ if (iasym) {+ Bf32[(i)*n + j + 0] = (int(b24.a - 2) - bzp[bid * n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (int(b24.b - 2) - bzp[bid * n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (int(b24.c - 2) - bzp[bid * n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (int(b24.d - 2) - bzp[bid * n + j + 3]) * scaleb[bid * n + j + 3];+ } else {+ Bf32[(i)*n + j + 0] = (int(b24.a - 2)) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (int(b24.b - 2)) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (int(b24.c - 2)) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (int(b24.d - 2)) * scaleb[bid * n + j + 3];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{nullptr, (uint8_t*)b2.data(), nullptr, scaleb.data(), iasym ? bzp.data() : nullptr, 2,+ n};+ kernel::avx512f::gemv_2bit_fp32_fp32<float, 48, MTILE>(Af32.data(), k, B, Cf32.data(), n, k, kblock, cache,+ CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_3bit_fp32(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit2x4> b2(n * k / 4);+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(Af32.data(), Af32.size(), -0.5f, 0.5f);+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s3_s8(b2.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 1) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 4) {+ if (iasym) {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0] - bzp[bid * n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1] - bzp[bid * n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2] - bzp[bid * n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3] - bzp[bid * n + j + 3]) * scaleb[bid * n + j + 3];+ } else {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3]) * scaleb[bid * n + j + 3];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ nullptr, (uint8_t*)b2.data(), (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr, 2, n};+ kernel::avx512f::gemv_3bit_fp32_fp32<float, 48, MTILE>(Af32.data(), k, B, Cf32.data(), n, k, kblock, cache,+ CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_3bit_u8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit2x4> b2(n * k / 4);+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<uint8_t> A(MTILE * k), azp(MTILE * blks);+ fill_buffer_randn(A.data(), A.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(azp.data(), azp.size(), uint8_t(100), uint8_t(150));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j]) - azp[bid]) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s3_s8(b2.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ nullptr, (uint8_t*)b2.data(), (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr, 2, n};+ kernel::avx512f::vnni::gemv_3bit_u8s8_fp32<float, 48, MTILE>({A.data(), scalea.data(), azp.data(), k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_3bit_s8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit2x4> b2(n * k / 4);+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> A(MTILE * k);+ fill_buffer_randn(A.data(), A.size(), int8_t(0), int8_t(127));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j])) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s3_s8(b2.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ nullptr, (uint8_t*)b2.data(), (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr, 2, n};+ kernel::avx512f::vnni::gemv_3bit_s8s8_fp32<float, 48, MTILE>({(uint8_t*)A.data(), scalea.data(), nullptr, k, blks},+ B, Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_5bit_fp32(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(Af32.data(), Af32.size(), -0.5f, 0.5f);+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-16), int8_t(15));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s5_s8(b4.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 1) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 4) {+ if (iasym) {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0] - bzp[bid * n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1] - bzp[bid * n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2] - bzp[bid * n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3] - bzp[bid * n + j + 3]) * scaleb[bid * n + j + 3];+ } else {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3]) * scaleb[bid * n + j + 3];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ (uint8_t*)b4.data(), nullptr, (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr, 5, n};+ kernel::avx512f::gemv_5bit_fp32_fp32<float, 48, MTILE>(Af32.data(), k, B, Cf32.data(), n, k, kblock, cache,+ CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_5bit_u8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-16), int8_t(15));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<uint8_t> A(MTILE * k), azp(MTILE * blks);+ fill_buffer_randn(A.data(), A.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(azp.data(), azp.size(), uint8_t(100), uint8_t(150));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j]) - azp[bid]) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s5_s8(b4.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ (uint8_t*)b4.data(), nullptr, (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr, 2, n};+ kernel::avx512f::vnni::gemv_5bit_u8s8_fp32<float, 48, MTILE>({A.data(), scalea.data(), azp.data(), k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_5bit_s8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-16), int8_t(15));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> A(MTILE * k);+ fill_buffer_randn(A.data(), A.size(), int8_t(0), int8_t(127));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j])) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s5_s8(b4.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ (uint8_t*)b4.data(), nullptr, (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr, 5, n};+ kernel::avx512f::vnni::gemv_5bit_s8s8_fp32<float, 48, MTILE>({(uint8_t*)A.data(), scalea.data(), nullptr, k, blks},+ B, Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_6bit_fp32(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit2x4> b2(n * k / 4);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(Af32.data(), Af32.size(), -0.5f, 0.5f);+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-32), int8_t(31));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s6_s8(b4.data(), b2.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 1) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 4) {+ if (iasym) {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0] - bzp[bid * n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1] - bzp[bid * n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2] - bzp[bid * n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3] - bzp[bid * n + j + 3]) * scaleb[bid * n + j + 3];+ } else {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3]) * scaleb[bid * n + j + 3];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ (uint8_t*)b4.data(), (uint8_t*)b2.data(), nullptr, scaleb.data(), iasym ? bzp.data() : nullptr, 6, n};+ kernel::avx512f::gemv_6bit_fp32_fp32<float, 48, MTILE>(Af32.data(), k, B, Cf32.data(), n, k, kblock, cache,+ CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_6bit_u8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit2x4> b2(n * k / 4);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-32), int8_t(31));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<uint8_t> A(MTILE * k), azp(MTILE * blks);+ fill_buffer_randn(A.data(), A.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(azp.data(), azp.size(), uint8_t(100), uint8_t(150));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j]) - azp[bid]) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s6_s8(b4.data(), b2.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ (uint8_t*)b4.data(), (uint8_t*)b2.data(), nullptr, scaleb.data(), iasym ? bzp.data() : nullptr, 6, n};+ kernel::avx512f::vnni::gemv_6bit_u8s8_fp32<float, 48, MTILE>({A.data(), scalea.data(), azp.data(), k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_6bit_s8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit2x4> b2(n * k / 4);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-32), int8_t(31));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> A(MTILE * k);+ fill_buffer_randn(A.data(), A.size(), int8_t(0), int8_t(127));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j])) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s6_s8(b4.data(), b2.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ (uint8_t*)b4.data(), (uint8_t*)b2.data(), nullptr, scaleb.data(), iasym ? bzp.data() : nullptr, 5, n};+ kernel::avx512f::vnni::gemv_6bit_s8s8_fp32<float, 48, MTILE>({(uint8_t*)A.data(), scalea.data(), nullptr, k, blks},+ B, Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+UT_avx512_gemv sUT_avx512_gemv;+#endif+#endif++#if CompileAVX2()+class UT_avx2_decompress_s6_s8 {+ public:+ UT_avx2_decompress_s6_s8() {+ UT_START();+ CheckISA(AVX2);+ ut<1, 24>(32);+ ut<4, 24>(32);+ ut<1, 24>(32, true);+ ut<2, 24>(32, true);+ ut<4, 24>(32, true);+ }++ template <int PackRow, int NTILE>+ void ut(int blocksize, bool isasym = false) {+ int row = blocksize * 2;+ int constexpr FullRange = 1 << (6 - 1);+ int constexpr col = NTILE;+ printf("Test Case %s: %d %d %d\n", __FUNCTION__, row, col, blocksize);+ std::vector<utils::bit4x2> s4_wei(row * col / 2);+ avector<utils::bit2x4> s2_wei(row * col / 4);++ std::vector<int8_t> s8_wei(col * row);+ std::vector<int8_t> s8_ref(col * row);+ int blks = row / blocksize;+ int row_offset = 8;+ assert(blocksize % 8 == 0);+ std::vector<int8_t> zp(col * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-FullRange), int8_t(FullRange - 1));+ std::vector<int8_t> rev(col * row);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-FullRange), int8_t(FullRange - 1));++ for (int i = 0; i < col * row; i += 4) {+ memcpy(&s8_ref[i], &s8_wei[i], 4 * sizeof(int8_t));+ s4_wei[i / 2].x = (s8_wei[i + 0] + FullRange) & 0xf;+ s4_wei[i / 2].y = (s8_wei[i + 1] + FullRange) & 0xf;+ s4_wei[i / 2 + 1].x = (s8_wei[i + 2] + FullRange) & 0xf;+ s4_wei[i / 2 + 1].y = (s8_wei[i + 3] + FullRange) & 0xf;++ s2_wei[i / 4].a = ((s8_wei[i + 0] + FullRange) & 0x30) >> 4;+ s2_wei[i / 4].b = ((s8_wei[i + 1] + FullRange) & 0x30) >> 4;+ s2_wei[i / 4].c = ((s8_wei[i + 2] + FullRange) & 0x30) >> 4;+ s2_wei[i / 4].d = ((s8_wei[i + 3] + FullRange) & 0x30) >> 4;+ }+ if (isasym) {+ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] -= zp[i / blocksize * NTILE + j];+ }+ }+ }+ }++ kernel::avx2::decompress_kblock_s6_s8<PackRow, NTILE>(s4_wei.data(), s2_wei.data(), isasym ? zp.data() : nullptr,+ rev.data(), blocksize, NTILE, 0, 0, row_offset, NTILE, cache,+ CacheSize);+ kernel::avx2::decompress_kblock_s6_s8<PackRow, NTILE>(+ s4_wei.data() + row_offset * NTILE / 2, s2_wei.data() + row_offset * NTILE / 4, isasym ? zp.data() : nullptr,+ rev.data() + row_offset * NTILE, blocksize, NTILE, 0, row_offset, row - row_offset, NTILE, cache, CacheSize);+ ut::buffer_error(s8_ref.data(), rev.data(), rev.size(), int8_t(0));+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+static UT_avx2_decompress_s6_s8 sUT_avx2_decompress_s6_s8;+#endif++class UT_avx2_decompress_s7_s8 {+ public:+ UT_avx2_decompress_s7_s8() {+ UT_START();+ CheckISA(AVX2);+ ut<1, 24>(32);+ ut<4, 24>(32);+ ut<1, 24>(32, true);+ ut<2, 24>(32, true);+ ut<4, 24>(32, true);+ }++ template <int PackRow, int NTILE>+ void ut(int blocksize, bool isasym = false) {+ int row = blocksize * 2;+ int constexpr col = NTILE;+ printf("Test Case %s: %d %d %d\n", __FUNCTION__, row, col, blocksize);+ std::vector<utils::bit4x2> s4_wei(row * col / 2);+ avector<utils::bit1x8> s1_wei(row * col / 8);+ avector<utils::bit2x4> s2_wei(row * col / 4);++ std::vector<int8_t> s8_wei(col * row);+ std::vector<int8_t> s8_ref(col * row);+ int blks = row / blocksize;+ int row_offset = 8;+ assert(blocksize % 8 == 0);+ std::vector<int8_t> zp(col * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-64), int8_t(63));+ std::vector<int8_t> rev(col * row);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-64), int8_t(63));++ for (int i = 0; i < col * row; i += 8) {+ memcpy(&s8_ref[i], &s8_wei[i], 8 * sizeof(int8_t));+ s4_wei[i / 2].x = (s8_wei[i + 0] + 64) & 0xf;+ s4_wei[i / 2].y = (s8_wei[i + 1] + 64) & 0xf;+ s4_wei[i / 2 + 1].x = (s8_wei[i + 2] + 64) & 0xf;+ s4_wei[i / 2 + 1].y = (s8_wei[i + 3] + 64) & 0xf;+ s4_wei[i / 2 + 2].x = (s8_wei[i + 4] + 64) & 0xf;+ s4_wei[i / 2 + 2].y = (s8_wei[i + 5] + 64) & 0xf;+ s4_wei[i / 2 + 3].x = (s8_wei[i + 6] + 64) & 0xf;+ s4_wei[i / 2 + 3].y = (s8_wei[i + 7] + 64) & 0xf;++ s2_wei[i / 4].a = ((s8_wei[i + 0] + 64) & 0x30) >> 4;+ s2_wei[i / 4].b = ((s8_wei[i + 1] + 64) & 0x30) >> 4;+ s2_wei[i / 4].c = ((s8_wei[i + 2] + 64) & 0x30) >> 4;+ s2_wei[i / 4].d = ((s8_wei[i + 3] + 64) & 0x30) >> 4;+ s2_wei[i / 4 + 1].a = ((s8_wei[i + 4] + 64) & 0x30) >> 4;+ s2_wei[i / 4 + 1].b = ((s8_wei[i + 5] + 64) & 0x30) >> 4;+ s2_wei[i / 4 + 1].c = ((s8_wei[i + 6] + 64) & 0x30) >> 4;+ s2_wei[i / 4 + 1].d = ((s8_wei[i + 7] + 64) & 0x30) >> 4;++ s1_wei[i / 8].a = ((s8_wei[i + 0] + 64) & 0x40) >> 6;+ s1_wei[i / 8].b = ((s8_wei[i + 1] + 64) & 0x40) >> 6;+ s1_wei[i / 8].c = ((s8_wei[i + 2] + 64) & 0x40) >> 6;+ s1_wei[i / 8].d = ((s8_wei[i + 3] + 64) & 0x40) >> 6;+ s1_wei[i / 8].e = ((s8_wei[i + 4] + 64) & 0x40) >> 6;+ s1_wei[i / 8].f = ((s8_wei[i + 5] + 64) & 0x40) >> 6;+ s1_wei[i / 8].g = ((s8_wei[i + 6] + 64) & 0x40) >> 6;+ s1_wei[i / 8].h = ((s8_wei[i + 7] + 64) & 0x40) >> 6;+ }+ if (isasym) {+ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] -= zp[i / blocksize * NTILE + j];+ }+ }+ }+ }++ kernel::avx2::decompress_kblock_s7_s8<PackRow, NTILE>(s4_wei.data(), s2_wei.data(), s1_wei.data(),+ isasym ? zp.data() : nullptr, rev.data(), blocksize, NTILE, 0,+ 0, row_offset, NTILE, cache, CacheSize);+ kernel::avx2::decompress_kblock_s7_s8<PackRow, NTILE>(+ s4_wei.data() + row_offset * NTILE / 2, s2_wei.data() + row_offset * NTILE / 4,+ s1_wei.data() + row_offset * NTILE / 8, isasym ? zp.data() : nullptr, rev.data() + row_offset * NTILE,+ blocksize, NTILE, 0, row_offset, row - row_offset, NTILE, cache, CacheSize);+ ut::buffer_error(s8_ref.data(), rev.data(), rev.size(), int8_t(0));+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+static UT_avx2_decompress_s7_s8 sUT_avx2_decompress_s7_s8;+#endif++class UT_avx2_decompress_s5_s8 {+ public:+ UT_avx2_decompress_s5_s8() {+ UT_START();+ CheckISA(AVX2);+ ut<1, 24>(32);+ ut<4, 24>(32);+ ut<1, 24>(32, true);+ ut<2, 24>(32, true);+ ut<4, 24>(32, true);+ }++ template <int PackRow, int NTILE>+ void ut(int blocksize, bool isasym = false) {+ int row = blocksize * 2;+ int constexpr col = NTILE;+ printf("Test Case %s: %d %d %d\n", __FUNCTION__, row, col, blocksize);+ std::vector<utils::bit4x2> s4_wei(row * col / 2);+ avector<utils::bit1x8> s1_wei(row * col / 8);++ std::vector<int8_t> s8_wei(col * row);+ std::vector<int8_t> s8_ref(col * row);+ int blks = row / blocksize;+ int row_offset = 8;+ assert(blocksize % 8 == 0);+ std::vector<int8_t> zp(col * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-16), int8_t(15));+ std::vector<int8_t> rev(col * row);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-16), int8_t(15));++ for (int i = 0; i < col * row; i += 8) {+ memcpy(&s8_ref[i], &s8_wei[i], 8 * sizeof(int8_t));+ s4_wei[i / 2].x = (s8_wei[i + 0] + 16) & 0xf;+ s4_wei[i / 2].y = (s8_wei[i + 1] + 16) & 0xf;+ s4_wei[i / 2 + 1].x = (s8_wei[i + 2] + 16) & 0xf;+ s4_wei[i / 2 + 1].y = (s8_wei[i + 3] + 16) & 0xf;+ s4_wei[i / 2 + 2].x = (s8_wei[i + 4] + 16) & 0xf;+ s4_wei[i / 2 + 2].y = (s8_wei[i + 5] + 16) & 0xf;+ s4_wei[i / 2 + 3].x = (s8_wei[i + 6] + 16) & 0xf;+ s4_wei[i / 2 + 3].y = (s8_wei[i + 7] + 16) & 0xf;++ s1_wei[i / 8].a = ((s8_wei[i + 0] + 16) & 0x10) >> 4;+ s1_wei[i / 8].b = ((s8_wei[i + 1] + 16) & 0x10) >> 4;+ s1_wei[i / 8].c = ((s8_wei[i + 2] + 16) & 0x10) >> 4;+ s1_wei[i / 8].d = ((s8_wei[i + 3] + 16) & 0x10) >> 4;+ s1_wei[i / 8].e = ((s8_wei[i + 4] + 16) & 0x10) >> 4;+ s1_wei[i / 8].f = ((s8_wei[i + 5] + 16) & 0x10) >> 4;+ s1_wei[i / 8].g = ((s8_wei[i + 6] + 16) & 0x10) >> 4;+ s1_wei[i / 8].h = ((s8_wei[i + 7] + 16) & 0x10) >> 4;+ }+ if (isasym) {+ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] -= zp[i / blocksize * NTILE + j];+ }+ }+ }+ }++ kernel::avx2::decompress_kblock_s5_s8<PackRow, NTILE>(s4_wei.data(), s1_wei.data(), isasym ? zp.data() : nullptr,+ rev.data(), blocksize, NTILE, 0, 0, row_offset, NTILE, cache,+ CacheSize);+ kernel::avx2::decompress_kblock_s5_s8<PackRow, NTILE>(+ s4_wei.data() + row_offset * NTILE / 2, s1_wei.data() + row_offset * NTILE / 8, isasym ? zp.data() : nullptr,+ rev.data() + row_offset * NTILE, blocksize, NTILE, 0, row_offset, row - row_offset, NTILE, cache, CacheSize);+ ut::buffer_error(s8_ref.data(), rev.data(), rev.size(), int8_t(0));+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+static UT_avx2_decompress_s5_s8 sUT_avx2_decompress_s5_s8;+#endif++class UT_avx2_decompress_s4_s8 {+ public:+ UT_avx2_decompress_s4_s8() {+ UT_START();+ CheckISA(AVX2);+ ut<1, 24>(32);+ ut<4, 24>(32);+ ut<1, 24>(32, true);+ ut<2, 24>(32, true);+ ut<4, 24>(32, true);+ }++ template <int PackRow, int NTILE>+ void ut(int blocksize, bool isasym = false) {+ int row = blocksize * 2;+ int constexpr col = NTILE;+ printf("Test Case %s: %d %d %d\n", __FUNCTION__, row, col, blocksize);+ std::vector<utils::int4x2> s4_wei(row * col / 2);+ std::vector<int8_t> s8_wei(col * row);+ std::vector<int8_t> s8_ref(col * row);+ int blks = row / blocksize;+ int row_offset = PackRow;+ std::vector<int8_t> zp(col * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-8), int8_t(7));+ std::vector<int8_t> rev(col * row);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-8), int8_t(7));++ for (int i = 0; i < col * row; i += 2) {+ s8_ref[i] = s8_wei[i];+ s8_ref[i + 1] = s8_wei[i + 1];+ s4_wei[i / 2].x = utils::int4x2::convert(s8_wei[i]) + 8;+ s4_wei[i / 2].y = utils::int4x2::convert(s8_wei[i + 1]) + 8;+ }+ if (isasym) {+ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] -= zp[i / blocksize * NTILE + j];+ }+ }+ }+ }++ kernel::avx2::decompress_kblock_s4_s8<PackRow, NTILE>(s4_wei.data(), isasym ? zp.data() : nullptr, rev.data(),+ blocksize, NTILE, 0, 0, row_offset, NTILE, cache, CacheSize);+ kernel::avx2::decompress_kblock_s4_s8<PackRow, NTILE>(+ s4_wei.data() + row_offset * NTILE / 2, isasym ? zp.data() : nullptr, rev.data() + row_offset * NTILE,+ blocksize, NTILE, 0, row_offset, row - row_offset, NTILE, cache, CacheSize);+ ut::buffer_error(s8_ref.data(), rev.data(), rev.size(), int8_t(0));+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+static UT_avx2_decompress_s4_s8 sUT_avx2_decompress_s4_s8;+#endif++class UT_avx2_decompress_s3_s8 {+ public:+ UT_avx2_decompress_s3_s8() {+ UT_START();+ CheckISA(AVX2);+ ut<1, 24>(32);+ ut<4, 24>(32);+ ut<1, 24>(32, true);+ ut<2, 24>(32, true);+ ut<4, 24>(32, true);+ }++ template <int PackRow, int NTILE>+ void ut(int blocksize, bool isasym = false) {+ int row = blocksize * 2;+ int constexpr col = NTILE;+ printf("Test Case %s: %d %d %d\n", __FUNCTION__, row, col, blocksize);+ std::vector<utils::bit2x4> s2_wei(row * col / 4);+ avector<utils::bit1x8> s1_wei(row * col / 8);++ std::vector<int8_t> s8_wei(col * row);+ std::vector<int8_t> s8_ref(col * row);+ int blks = row / blocksize;+ int row_offset = 8;+ assert(blocksize % 8 == 0);+ std::vector<int8_t> zp(col * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-4), int8_t(3));+ std::vector<int8_t> rev(col * row);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-4), int8_t(3));++ for (int i = 0; i < col * row; i += 8) {+ memcpy(&s8_ref[i], &s8_wei[i], 8 * sizeof(int8_t));+ s2_wei[i / 4].a = (s8_wei[i + 0] + 4) & 0x3;+ s2_wei[i / 4].b = (s8_wei[i + 1] + 4) & 0x3;+ s2_wei[i / 4].c = (s8_wei[i + 2] + 4) & 0x3;+ s2_wei[i / 4].d = (s8_wei[i + 3] + 4) & 0x3;++ s2_wei[i / 4 + 1].a = (s8_wei[i + 4] + 4) & 0x3;+ s2_wei[i / 4 + 1].b = (s8_wei[i + 5] + 4) & 0x3;+ s2_wei[i / 4 + 1].c = (s8_wei[i + 6] + 4) & 0x3;+ s2_wei[i / 4 + 1].d = (s8_wei[i + 7] + 4) & 0x3;++ s1_wei[i / 8].a = ((s8_wei[i + 0] + 4) & 0x4) >> 2;+ s1_wei[i / 8].b = ((s8_wei[i + 1] + 4) & 0x4) >> 2;+ s1_wei[i / 8].c = ((s8_wei[i + 2] + 4) & 0x4) >> 2;+ s1_wei[i / 8].d = ((s8_wei[i + 3] + 4) & 0x4) >> 2;+ s1_wei[i / 8].e = ((s8_wei[i + 4] + 4) & 0x4) >> 2;+ s1_wei[i / 8].f = ((s8_wei[i + 5] + 4) & 0x4) >> 2;+ s1_wei[i / 8].g = ((s8_wei[i + 6] + 4) & 0x4) >> 2;+ s1_wei[i / 8].h = ((s8_wei[i + 7] + 4) & 0x4) >> 2;+ }+ if (isasym) {+ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] -= zp[i / blocksize * NTILE + j];+ }+ }+ }+ }++ kernel::avx2::decompress_kblock_s3_s8<PackRow, NTILE>(s2_wei.data(), s1_wei.data(), isasym ? zp.data() : nullptr,+ rev.data(), blocksize, NTILE, 0, 0, row_offset, NTILE, cache,+ CacheSize);+ kernel::avx2::decompress_kblock_s3_s8<PackRow, NTILE>(+ s2_wei.data() + row_offset * NTILE / 4, s1_wei.data() + row_offset * NTILE / 8, isasym ? zp.data() : nullptr,+ rev.data() + row_offset * NTILE, blocksize, NTILE, 0, row_offset, row - row_offset, NTILE, cache, CacheSize);+ ut::buffer_error(s8_ref.data(), rev.data(), rev.size(), int8_t(0));+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+static UT_avx2_decompress_s3_s8 sUT_avx2_decompress_s3_s8;+#endif++class UT_avx2_decompress_s1_s8 {+ public:+ UT_avx2_decompress_s1_s8() {+ UT_START();+ CheckISA(AVX2);+ ut<1, 24>(32);+ ut<4, 24>(32);+ ut<1, 24>(32, true);+ ut<2, 24>(32, true);+ ut<4, 24>(32, true);+ }++ template <int PackRow, int NTILE>+ void ut(int blocksize, bool isasym = false) {+ int row = blocksize * 2;+ int constexpr col = NTILE;+ printf("Test Case %s: %d %d %d\n", __FUNCTION__, row, col, blocksize);+ avector<utils::bit1x8> s1_wei(row * col / 8);++ std::vector<int8_t> s8_wei(col * row);+ std::vector<int8_t> s8_ref(col * row);+ int blks = row / blocksize;+ int row_offset = 8;+ assert(blocksize % 8 == 0);+ std::vector<int8_t> zp(col * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-1), int8_t(0));+ std::vector<int8_t> rev(col * row);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-1), int8_t(0));++ for (int i = 0; i < col * row; i += 8) {+ memcpy(&s8_ref[i], &s8_wei[i], 8 * sizeof(int8_t));+ s1_wei[i / 8].a = s8_wei[i + 0] + 1;+ s1_wei[i / 8].b = s8_wei[i + 1] + 1;+ s1_wei[i / 8].c = s8_wei[i + 2] + 1;+ s1_wei[i / 8].d = s8_wei[i + 3] + 1;+ s1_wei[i / 8].e = s8_wei[i + 4] + 1;+ s1_wei[i / 8].f = s8_wei[i + 5] + 1;+ s1_wei[i / 8].g = s8_wei[i + 6] + 1;+ s1_wei[i / 8].h = s8_wei[i + 7] + 1;+ }+ if (isasym) {+ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] -= zp[i / blocksize * NTILE + j];+ }+ }+ }+ }++ kernel::avx2::decompress_kblock_s1_s8<PackRow, NTILE>(s1_wei.data(), isasym ? zp.data() : nullptr, rev.data(),+ blocksize, NTILE, 0, 0, row_offset, NTILE, cache, CacheSize);+ kernel::avx2::decompress_kblock_s1_s8<PackRow, NTILE>(+ s1_wei.data() + row_offset * NTILE / 8, isasym ? zp.data() : nullptr, rev.data() + row_offset * NTILE,+ blocksize, NTILE, 0, row_offset, row - row_offset, NTILE, cache, CacheSize);+ ut::buffer_error(s8_ref.data(), rev.data(), rev.size(), int8_t(0));+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+static UT_avx2_decompress_s1_s8 sUT_avx2_decompress_s1_s8;+#endif++class UT_avx2_decompress_s2_s8 {+ public:+ UT_avx2_decompress_s2_s8() {+ UT_START();+ CheckISA(AVX2);+ ut<1, 24>(32);+ ut<4, 24>(32);+ ut<1, 24>(32, true);+ ut<2, 24>(32, true);+ ut<4, 24>(32, true);+ }++ template <int PackRow, int NTILE>+ void ut(int blocksize, bool isasym = false) {+ int row = blocksize * 2;+ int constexpr col = NTILE;+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, PackRow, row, col, blocksize, isasym);+ std::vector<utils::bit2x4> s2_wei(row * col / 4);++ std::vector<int8_t> s8_wei(col * row);+ std::vector<int8_t> s8_ref(col * row);+ int blks = row / blocksize;+ int row_offset = blocksize;+ assert(blocksize % 8 == 0);+ std::vector<int8_t> zp(col * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-2), int8_t(1));+ std::vector<int8_t> rev(col * row);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-2), int8_t(1));++ for (int i = 0; i < col * row; i += 4) {+ memcpy(&s8_ref[i], &s8_wei[i], 4 * sizeof(int8_t));+ s2_wei[i / 4].a = (s8_wei[i + 0] + 2) & 0x3;+ s2_wei[i / 4].b = (s8_wei[i + 1] + 2) & 0x3;+ s2_wei[i / 4].c = (s8_wei[i + 2] + 2) & 0x3;+ s2_wei[i / 4].d = (s8_wei[i + 3] + 2) & 0x3;+ }+ if (isasym) {+ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] -= zp[i / blocksize * NTILE + j];+ }+ }+ }+ }++ kernel::avx2::decompress_kblock_s2_s8<PackRow, NTILE>(s2_wei.data(), isasym ? zp.data() : nullptr, rev.data(),+ blocksize, NTILE, 0, 0, row_offset, NTILE, cache, CacheSize);+ kernel::avx2::decompress_kblock_s2_s8<PackRow, NTILE>(+ s2_wei.data() + row_offset * NTILE / 4, isasym ? zp.data() : nullptr, rev.data() + row_offset * NTILE,+ blocksize, NTILE, 0, row_offset, row - row_offset, NTILE, cache, CacheSize);+ ut::buffer_error(s8_ref.data(), rev.data(), rev.size(), int8_t(0));+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+static UT_avx2_decompress_s2_s8 sUT_avx2_decompress_s2_s8;+#endif++class UT_avx2_decompress_s4_fp {+ public:+ UT_avx2_decompress_s4_fp() {+ UT_START();+ CheckISA(AVX2);+ ut<4, 24, utils::bf16>(32);+ ut<1, 24, float>(32);+ ut<2, 24, float>(32);+ ut<4, 24, float>(32);+ ut<4, 24, utils::bf16, utils::bf16>(32);+ ut<1, 24, float>(32, true);+ ut<2, 24, float>(32, true);+ ut<4, 24, float>(32, true);+ ut<4, 24, utils::bf16>(32, true);+ ut<4, 24, utils::bf16, utils::bf16>(32, true);+ }++ template <int PackRow, int NTILE, typename T, typename ScaleT = float>+ void ut(int blocksize, bool isasym = false) {+ auto dst_dtype = bestla_dtype<T>;+ auto scale_dtype = bestla_dtype<ScaleT>;+ int row = blocksize * 2;+ int constexpr col = NTILE;+ printf("Test Case %s: %d %d %d Asym:%d Pack:%d %s %s\n", __FUNCTION__, row, col, blocksize, isasym, PackRow,+ utils::bestla_dtype_str(dst_dtype), bestla_dtype_str(scale_dtype));+ std::vector<utils::int4x2> s4_wei(row * col / 2);+ std::vector<int8_t> s8_wei(col * row);+ std::vector<T> s8_ref(col * row);+ int blks = row / blocksize;+ int row_offset = PackRow;+ std::vector<int8_t> zp(col * blks);+ avector<ScaleT> scale(col * blks);+ fill_buffer_randn(scale.data(), scale.size(), ScaleT(0.01f), ScaleT(0.03f));+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-8), int8_t(7));+ std::vector<T> rev(col * row);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-8), int8_t(7));++ for (int i = 0; i < col * row; i += 2) {+ s8_ref[i] = float(s8_wei[i]);+ s8_ref[i + 1] = float(s8_wei[i + 1]);+ s4_wei[i / 2].x = utils::int4x2::convert(s8_wei[i]) + 8;+ s4_wei[i / 2].y = utils::int4x2::convert(s8_wei[i + 1]) + 8;+ }+ if (isasym) {+ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ int corr_offset = i / blocksize * NTILE + j;+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] = float(s8_ref[i * NTILE + j * PackRow + ip]) - float(zp[corr_offset]);+ }+ }+ }+ }++ for (int i = 0; i < row; i += PackRow) {+ for (int j = 0; j < NTILE; j++) {+ int corr_offset = i / blocksize * NTILE + j;+ for (int ip = 0; ip < PackRow; ip++) {+ s8_ref[i * NTILE + j * PackRow + ip] =+ float(s8_ref[i * NTILE + j * PackRow + ip]) * float(scale[corr_offset]);+ }+ }+ }+ kernel::avx2::decompress_kblock_s4_fp<PackRow, NTILE>(s4_wei.data(), rev.data(), row_offset, NTILE, scale.data(),+ scale_dtype, isasym ? zp.data() : nullptr, 0, 0, blocksize,+ NTILE, cache, CacheSize);+ kernel::avx2::decompress_kblock_s4_fp<PackRow, NTILE>(+ s4_wei.data() + row_offset * NTILE / 2, rev.data() + row_offset * NTILE, row - row_offset, NTILE, scale.data(),+ scale_dtype, isasym ? zp.data() : nullptr, row_offset, 0, blocksize, NTILE, cache, CacheSize);+ float err = get_ut_err(dst_dtype);+ ut::buffer_error(s8_ref.data(), rev.data(), rev.size(), T(err));+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+static UT_avx2_decompress_s4_fp sUT_avx2_decompress_s4_fp;+#endif++// s<N>_fp share the same process: s<N>->s8->fp, so it's not necessary to test all s<N>_fp cases.+// test s<N>_s8 is just fine.+#endif++#if CompileAVXVNNI()+class UT_avx2_gemv {+ public:+ UT_avx2_gemv() {+ UT_START();+ CheckISA(AVX2);+ ut_1bit_fp32<1>(24, 128, 32, true);+ ut_1bit_fp32<1>(24, 128, 32, false);+ ut_1bit_fp32<4>(24, 128, 32, true);+ ut_1bit_fp32<4>(24, 128, 32, false);++ ut_7bit_fp32<1>(24, 128, 32, true);+ ut_7bit_fp32<1>(24, 128, 32, false);+ ut_7bit_fp32<4>(24, 128, 32, true);+ ut_7bit_fp32<4>(24, 128, 32, false);++ ut_6bit_fp32<1>(24, 128, 32, true);+ ut_6bit_fp32<1>(24, 128, 32, false);+ ut_6bit_fp32<4>(24, 128, 32, true);+ ut_6bit_fp32<4>(24, 128, 32, false);++ ut_5bit_fp32<1>(24, 128, 32, true);+ ut_5bit_fp32<1>(24, 128, 32, false);+ ut_5bit_fp32<4>(24, 128, 32, true);+ ut_5bit_fp32<4>(24, 128, 32, false);++ ut_4bit_fp32<1>(24, 128, 32, true);+ ut_4bit_fp32<1>(24, 128, 32, false);+ ut_4bit_fp32<4>(24, 128, 32, true);+ ut_4bit_fp32<4>(24, 128, 32, false);++ ut_3bit_fp32<1>(24, 128, 32, true);+ ut_3bit_fp32<1>(24, 128, 32, false);+ ut_3bit_fp32<4>(24, 128, 32, true);+ ut_3bit_fp32<4>(24, 128, 32, false);++ ut_2bit_fp32<1>(24, 128, 32, true);+ ut_2bit_fp32<1>(24, 128, 32, false);+ ut_2bit_fp32<4>(24, 128, 32, true);+ ut_2bit_fp32<4>(24, 128, 32, false);++ CheckISA(AVX_VNNI);+ ut_1bit_u8s8<1>(24, 128, 32, true);+ ut_1bit_u8s8<1>(24, 128, 32, false);+ ut_1bit_u8s8<4>(24, 128, 32, true);+ ut_1bit_u8s8<4>(24, 128, 32, false);++ ut_1bit_s8s8<1>(24, 128, 32, true);+ ut_1bit_s8s8<1>(24, 128, 32, false);+ ut_1bit_s8s8<4>(24, 128, 32, true);+ ut_1bit_s8s8<4>(24, 128, 32, false);++ ut_7bit_u8s8<1>(24, 128, 32, true);+ ut_7bit_u8s8<1>(24, 128, 32, false);+ ut_7bit_u8s8<4>(24, 128, 32, true);+ ut_7bit_u8s8<4>(24, 128, 32, false);++ ut_7bit_s8s8<1>(24, 128, 32, true);+ ut_7bit_s8s8<1>(24, 128, 32, false);+ ut_7bit_s8s8<4>(24, 128, 32, true);+ ut_7bit_s8s8<4>(24, 128, 32, false);++ ut_4bit<1>(24, 128, 32, true);+ ut_4bit<1>(24, 128, 32, false);+ ut_4bit<4>(24, 128, 32, false);+ ut_4bit<4>(24, 128, 32, true);++ ut_4bit_s8s8<1>(24, 128, 32, true);+ ut_4bit_s8s8<1>(24, 128, 32, false);+ ut_4bit_s8s8<4>(24, 128, 32, true);+ ut_4bit_s8s8<4>(24, 128, 32, false);++ ut_2bit<1>(24, 128, 32, true);+ ut_2bit<1>(24, 128, 32, false);+ ut_2bit<4>(24, 128, 32, true);+ ut_2bit<4>(24, 128, 32, false);++ ut_2bit_s8s8<1>(24, 128, 32, true);+ ut_2bit_s8s8<1>(24, 128, 32, false);+ ut_2bit_s8s8<4>(24, 128, 32, true);+ ut_2bit_s8s8<4>(24, 128, 32, false);++ ut_3bit_u8s8<1>(24, 128, 32, true);+ ut_3bit_u8s8<1>(24, 128, 32, false);+ ut_3bit_u8s8<4>(24, 128, 32, true);+ ut_3bit_u8s8<4>(24, 128, 32, false);++ ut_3bit_s8s8<1>(24, 128, 32, true);+ ut_3bit_s8s8<1>(24, 128, 32, false);+ ut_3bit_s8s8<4>(24, 128, 32, true);+ ut_3bit_s8s8<4>(24, 128, 32, false);++ ut_6bit_u8s8<1>(24, 128, 32, true);+ ut_6bit_u8s8<1>(24, 128, 32, false);+ ut_6bit_u8s8<4>(24, 128, 32, true);+ ut_6bit_u8s8<4>(24, 128, 32, false);++ ut_6bit_s8s8<1>(24, 128, 32, true);+ ut_6bit_s8s8<1>(24, 128, 32, false);+ ut_6bit_s8s8<4>(24, 128, 32, true);+ ut_6bit_s8s8<4>(24, 128, 32, false);++ ut_5bit_u8s8<1>(24, 128, 32, true);+ ut_5bit_u8s8<1>(24, 128, 32, false);+ ut_5bit_u8s8<4>(24, 128, 32, true);+ ut_5bit_u8s8<4>(24, 128, 32, false);++ ut_5bit_s8s8<1>(24, 128, 32, true);+ ut_5bit_s8s8<1>(24, 128, 32, false);+ ut_5bit_s8s8<4>(24, 128, 32, true);+ ut_5bit_s8s8<4>(24, 128, 32, false);+ }++ template <int MTILE>+ void ut_4bit(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b2(n * k / 2);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<uint8_t> A(MTILE * k), azp(MTILE * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(A.data(), A.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(azp.data(), azp.size(), uint8_t(100), uint8_t(150));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j]) - azp[bid]) * scalea[bid];+ }+ }+ }++ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ auto b24 = b2[(i * n + j * 4) / 2];+ auto b42 = b2[(i * n + j * 4 + 2) / 2];+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b24.x - 8) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b24.y - 8) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b42.x - 8) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b42.y - 8) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b24.x - 8)) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b24.y - 8)) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b42.x - 8)) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b42.y - 8)) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{(uint8_t*)b2.data(), nullptr, nullptr, scaleb.data(),+ iasym ? bzp.data() : nullptr, 2, n};+ kernel::avx2::vnni::gemv_4bit_u8s8_fp32<float, 24, MTILE>({A.data(), scalea.data(), azp.data(), k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_4bit_fp32(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b2(n * k / 2);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(Af32.data(), Af32.size(), -0.5f, 0.5f);+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);++ for (int i = 0; i < k; i += 1) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 2) {+ auto b24 = b2[(i * n + j) / 2];+ if (iasym) {+ Bf32[(i)*n + j + 0] = (int(b24.x - 8) - bzp[bid * n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (int(b24.y - 8) - bzp[bid * n + j + 1]) * scaleb[bid * n + j + 1];+ } else {+ Bf32[(i)*n + j + 0] = (int(b24.x - 8)) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (int(b24.y - 8)) * scaleb[bid * n + j + 1];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{(uint8_t*)b2.data(), nullptr, nullptr, scaleb.data(),+ iasym ? bzp.data() : nullptr, 4, n};+ kernel::avx2::gemv_4bit_fp32_fp32<float, 24, MTILE>(Af32.data(), k, B, Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_4bit_s8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b2(n * k / 2);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<int8_t> A(MTILE * k);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(A.data(), A.size(), int8_t(-127), int8_t(127));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j])) * scalea[bid];+ }+ }+ }++ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ auto b24 = b2[(i * n + j * 4) / 2];+ auto b42 = b2[(i * n + j * 4 + 2) / 2];+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b24.x - 8) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b24.y - 8) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b42.x - 8) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b42.y - 8) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b24.x - 8)) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b24.y - 8)) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b42.x - 8)) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b42.y - 8)) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{(uint8_t*)b2.data(), nullptr, nullptr, scaleb.data(),+ iasym ? bzp.data() : nullptr, 2, n};+ kernel::avx2::vnni::gemv_4bit_s8s8_fp32<float, 24, MTILE>({(uint8_t*)A.data(), scalea.data(), nullptr, k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_2bit(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit2x4> b2(n * k / 4);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<uint8_t> A(MTILE * k), azp(MTILE * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(A.data(), A.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-2), int8_t(1));+ fill_buffer_randn(azp.data(), azp.size(), uint8_t(100), uint8_t(150));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int i = 0; i < MTILE; i++) {+ for (int j = 0; j < k; j++) {+ Af32[i * k + j] = (int(A[i * k + j]) - azp[i * blks + j / kblock]) * scalea[i * blks + j / kblock];+ }+ }+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ auto b24 = b2[(i * n + j * 4) / 4];+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b24.a - 2) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b24.b - 2) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b24.c - 2) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b24.d - 2) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b24.a - 2)) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b24.b - 2)) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b24.c - 2)) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b24.d - 2)) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ kernel::avx2::vnni::gemv_2bit_u8s8_fp32<float, 24, MTILE>(+ {A.data(), scalea.data(), azp.data(), k, blks},+ {nullptr, (uint8_t*)b2.data(), nullptr, scaleb.data(), iasym ? bzp.data() : nullptr, 2, n}, Cf32.data(), n, k,+ kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_2bit_s8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit2x4> b2(n * k / 4);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<int8_t> A(MTILE * k);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(A.data(), A.size(), int8_t(0), int8_t(127));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-2), int8_t(1));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int i = 0; i < MTILE; i++) {+ for (int j = 0; j < k; j++) {+ Af32[i * k + j] = (int(A[i * k + j])) * scalea[i * blks + j / kblock];+ }+ }+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ auto b24 = b2[(i * n + j * 4) / 4];+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b24.a - 2) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b24.b - 2) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b24.c - 2) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b24.d - 2) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b24.a - 2)) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b24.b - 2)) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b24.c - 2)) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b24.d - 2)) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ kernel::avx2::vnni::gemv_2bit_s8s8_fp32<float, 24, MTILE>(+ {(uint8_t*)A.data(), scalea.data(), nullptr, k, blks},+ {nullptr, (uint8_t*)b2.data(), nullptr, scaleb.data(), iasym ? bzp.data() : nullptr, 2, n}, Cf32.data(), n, k,+ kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_2bit_fp32(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit2x4> b2(n * k / 4);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(Af32.data(), Af32.size(), -0.5f, 0.5f);+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);++ for (int i = 0; i < k; i += 1) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 4) {+ auto b24 = b2[(i * n + j) / 4];+ if (iasym) {+ Bf32[(i)*n + j + 0] = (int(b24.a - 2) - bzp[bid * n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (int(b24.b - 2) - bzp[bid * n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (int(b24.c - 2) - bzp[bid * n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (int(b24.d - 2) - bzp[bid * n + j + 3]) * scaleb[bid * n + j + 3];+ } else {+ Bf32[(i)*n + j + 0] = (int(b24.a - 2)) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (int(b24.b - 2)) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (int(b24.c - 2)) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (int(b24.d - 2)) * scaleb[bid * n + j + 3];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{nullptr, (uint8_t*)b2.data(), nullptr, scaleb.data(), iasym ? bzp.data() : nullptr, 2,+ n};+ kernel::avx2::gemv_2bit_fp32_fp32<float, 24, MTILE>(Af32.data(), k, B, Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_3bit_fp32(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit2x4> b2(n * k / 4);+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(Af32.data(), Af32.size(), -0.5f, 0.5f);+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s3_s8(b2.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 1) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 4) {+ if (iasym) {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0] - bzp[bid * n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1] - bzp[bid * n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2] - bzp[bid * n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3] - bzp[bid * n + j + 3]) * scaleb[bid * n + j + 3];+ } else {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3]) * scaleb[bid * n + j + 3];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ nullptr, (uint8_t*)b2.data(), (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr, 2, n};+ kernel::avx2::gemv_3bit_fp32_fp32<float, 24, MTILE>(Af32.data(), k, B, Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_1bit_fp32(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(Af32.data(), Af32.size(), -0.5f, 0.5f);+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-1), int8_t(0));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s1_s8(b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 1) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 4) {+ if (iasym) {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0] - bzp[bid * n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1] - bzp[bid * n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2] - bzp[bid * n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3] - bzp[bid * n + j + 3]) * scaleb[bid * n + j + 3];+ } else {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3]) * scaleb[bid * n + j + 3];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{nullptr, nullptr, (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr,+ 1, n};+ kernel::avx2::gemv_1bit_fp32_fp32<float, 24, MTILE>(Af32.data(), k, B, Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_1bit_u8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-1), int8_t(0));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<uint8_t> A(MTILE * k), azp(MTILE * blks);+ fill_buffer_randn(A.data(), A.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(azp.data(), azp.size(), uint8_t(100), uint8_t(150));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j]) - azp[bid]) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s1_s8(b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{nullptr, nullptr, (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr,+ 2, n};+ kernel::avx2::vnni::gemv_1bit_u8s8_fp32<float, 24, MTILE>({A.data(), scalea.data(), azp.data(), k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_1bit_s8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-1), int8_t(0));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> A(MTILE * k);+ fill_buffer_randn(A.data(), A.size(), int8_t(0), int8_t(127));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j])) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s1_s8(b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{nullptr, nullptr, (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr,+ 2, n};+ kernel::avx2::vnni::gemv_1bit_s8s8_fp32<float, 24, MTILE>({(uint8_t*)A.data(), scalea.data(), nullptr, k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_3bit_u8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit2x4> b2(n * k / 4);+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<uint8_t> A(MTILE * k), azp(MTILE * blks);+ fill_buffer_randn(A.data(), A.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(azp.data(), azp.size(), uint8_t(100), uint8_t(150));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j]) - azp[bid]) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s3_s8(b2.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ nullptr, (uint8_t*)b2.data(), (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr, 2, n};+ kernel::avx2::vnni::gemv_3bit_u8s8_fp32<float, 24, MTILE>({A.data(), scalea.data(), azp.data(), k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_3bit_s8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit2x4> b2(n * k / 4);+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> A(MTILE * k);+ fill_buffer_randn(A.data(), A.size(), int8_t(0), int8_t(127));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j])) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s3_s8(b2.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ nullptr, (uint8_t*)b2.data(), (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr, 2, n};+ kernel::avx2::vnni::gemv_3bit_s8s8_fp32<float, 24, MTILE>({(uint8_t*)A.data(), scalea.data(), nullptr, k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_6bit_fp32(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit2x4> b2(n * k / 4);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(Af32.data(), Af32.size(), -0.5f, 0.5f);+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-32), int8_t(31));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s6_s8(b4.data(), b2.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 1) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 4) {+ if (iasym) {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0] - bzp[bid * n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1] - bzp[bid * n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2] - bzp[bid * n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3] - bzp[bid * n + j + 3]) * scaleb[bid * n + j + 3];+ } else {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3]) * scaleb[bid * n + j + 3];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ (uint8_t*)b4.data(), (uint8_t*)b2.data(), nullptr, scaleb.data(), iasym ? bzp.data() : nullptr, 6, n};+ kernel::avx2::gemv_6bit_fp32_fp32<float, 24, MTILE>(Af32.data(), k, B, Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_6bit_u8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit2x4> b2(n * k / 4);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-32), int8_t(31));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<uint8_t> A(MTILE * k), azp(MTILE * blks);+ fill_buffer_randn(A.data(), A.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(azp.data(), azp.size(), uint8_t(100), uint8_t(150));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j]) - azp[bid]) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s6_s8(b4.data(), b2.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ (uint8_t*)b4.data(), (uint8_t*)b2.data(), nullptr, scaleb.data(), iasym ? bzp.data() : nullptr, 6, n};+ kernel::avx2::vnni::gemv_6bit_u8s8_fp32<float, 24, MTILE>({A.data(), scalea.data(), azp.data(), k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_6bit_s8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit2x4> b2(n * k / 4);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-32), int8_t(31));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> A(MTILE * k);+ fill_buffer_randn(A.data(), A.size(), int8_t(0), int8_t(127));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j])) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s6_s8(b4.data(), b2.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ (uint8_t*)b4.data(), (uint8_t*)b2.data(), nullptr, scaleb.data(), iasym ? bzp.data() : nullptr, 5, n};+ kernel::avx2::vnni::gemv_6bit_s8s8_fp32<float, 24, MTILE>({(uint8_t*)A.data(), scalea.data(), nullptr, k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_5bit_fp32(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(Af32.data(), Af32.size(), -0.5f, 0.5f);+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-16), int8_t(15));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s5_s8(b4.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 1) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 4) {+ if (iasym) {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0] - bzp[bid * n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1] - bzp[bid * n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2] - bzp[bid * n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3] - bzp[bid * n + j + 3]) * scaleb[bid * n + j + 3];+ } else {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3]) * scaleb[bid * n + j + 3];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ (uint8_t*)b4.data(), nullptr, (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr, 5, n};+ kernel::avx2::gemv_5bit_fp32_fp32<float, 24, MTILE>(Af32.data(), k, B, Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_5bit_u8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-16), int8_t(15));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<uint8_t> A(MTILE * k), azp(MTILE * blks);+ fill_buffer_randn(A.data(), A.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(azp.data(), azp.size(), uint8_t(100), uint8_t(150));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j]) - azp[bid]) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s5_s8(b4.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ (uint8_t*)b4.data(), nullptr, (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr, 2, n};+ kernel::avx2::vnni::gemv_5bit_u8s8_fp32<float, 24, MTILE>({A.data(), scalea.data(), azp.data(), k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_5bit_s8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-16), int8_t(15));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> A(MTILE * k);+ fill_buffer_randn(A.data(), A.size(), int8_t(0), int8_t(127));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j])) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s5_s8(b4.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{+ (uint8_t*)b4.data(), nullptr, (uint8_t*)b1.data(), scaleb.data(), iasym ? bzp.data() : nullptr, 5, n};+ kernel::avx2::vnni::gemv_5bit_s8s8_fp32<float, 24, MTILE>({(uint8_t*)A.data(), scalea.data(), nullptr, k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_7bit_fp32(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit1x8> b1(n * k / 8);+ avector<bit2x4> b2(n * k / 4);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(Af32.data(), Af32.size(), -0.5f, 0.5f);+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-64), int8_t(63));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s7_s8(b4.data(), b2.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 1) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 4) {+ if (iasym) {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0] - bzp[bid * n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1] - bzp[bid * n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2] - bzp[bid * n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3] - bzp[bid * n + j + 3]) * scaleb[bid * n + j + 3];+ } else {+ Bf32[(i)*n + j + 0] = (b8[(i)*n + j + 0]) * scaleb[bid * n + j + 0];+ Bf32[(i)*n + j + 1] = (b8[(i)*n + j + 1]) * scaleb[bid * n + j + 1];+ Bf32[(i)*n + j + 2] = (b8[(i)*n + j + 2]) * scaleb[bid * n + j + 2];+ Bf32[(i)*n + j + 3] = (b8[(i)*n + j + 3]) * scaleb[bid * n + j + 3];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{(uint8_t*)b4.data(),+ (uint8_t*)b2.data(),+ (uint8_t*)b1.data(),+ scaleb.data(),+ iasym ? bzp.data() : nullptr,+ 7,+ n};+ kernel::avx2::gemv_7bit_fp32_fp32<float, 24, MTILE>(Af32.data(), k, B, Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_7bit_u8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit2x4> b2(n * k / 4);+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-64), int8_t(63));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<uint8_t> A(MTILE * k), azp(MTILE * blks);+ fill_buffer_randn(A.data(), A.size(), uint8_t(0), uint8_t(128));+ fill_buffer_randn(azp.data(), azp.size(), uint8_t(50), uint8_t(64));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j]) - azp[bid]) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s7_s8(b4.data(), b2.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{(uint8_t*)b4.data(),+ (uint8_t*)b2.data(),+ (uint8_t*)b1.data(),+ scaleb.data(),+ iasym ? bzp.data() : nullptr,+ 7,+ n};+ kernel::avx2::vnni::gemv_7bit_u8s8_fp32<float, 24, MTILE>({A.data(), scalea.data(), azp.data(), k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }++ template <int MTILE>+ void ut_7bit_s8s8(int n, int k, int kblock, bool iasym) {+ printf("Test Case %s_%d: %d %d %d Asym:%d\n", __FUNCTION__, MTILE, n, k, kblock, iasym);+ int blks = k / kblock;+ avector<bit4x2> b4(n * k / 2);+ avector<bit2x4> b2(n * k / 4);+ avector<bit1x8> b1(n * k / 8);+ avector<float> scaleb(n * blks), scalea(MTILE * blks);+ avector<int8_t> bzp(n * blks);+ avector<float> Af32(MTILE * k), Bf32(n * k), Cf32(MTILE * n), Cref(MTILE * n);+ fill_buffer_randn((uint8_t*)b4.data(), b4.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b2.data(), b2.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn((uint8_t*)b1.data(), b1.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(bzp.data(), bzp.size(), int8_t(-64), int8_t(63));+ fill_buffer_randn(scaleb.data(), scaleb.size(), 0.01f, 0.02f);+ avector<int8_t> A(MTILE * k);+ fill_buffer_randn(A.data(), A.size(), int8_t(0), int8_t(127));+ fill_buffer_randn(scalea.data(), scalea.size(), 0.01f, 0.02f);+ for (int im = 0; im < MTILE; im++) {+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock + im * blks;+ for (int j = 0; j < 4; j++) {+ Af32[im * k + i + j] = (int(A[im * k + i + j])) * scalea[bid];+ }+ }+ }++ avector<int8_t> b8(n * k);+ kernel::ref::decompress_s7_s8(b4.data(), b2.data(), b1.data(), b8.data(), b8.size(), cache, CacheSize);+ for (int i = 0; i < k; i += 4) {+ int bid = i / kblock;+ for (int j = 0; j < n; j += 1) {+ if (iasym) {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3]) - bzp[bid * n + j]) * scaleb[bid * n + j];+ } else {+ Bf32[(i + 0) * n + j] = (int(b8[i * n + j * 4 + 0])) * scaleb[bid * n + j];+ Bf32[(i + 1) * n + j] = (int(b8[i * n + j * 4 + 1])) * scaleb[bid * n + j];+ Bf32[(i + 2) * n + j] = (int(b8[i * n + j * 4 + 2])) * scaleb[bid * n + j];+ Bf32[(i + 3) * n + j] = (int(b8[i * n + j * 4 + 3])) * scaleb[bid * n + j];+ }+ }+ }+ gemmref_fp32fp32fp32(MTILE, n, k, Af32.data(), Bf32.data(), Cref.data(), k, n, n);+ utils::GemvParamB<float> B{(uint8_t*)b4.data(),+ (uint8_t*)b2.data(),+ (uint8_t*)b1.data(),+ scaleb.data(),+ iasym ? bzp.data() : nullptr,+ 7,+ n};+ kernel::avx2::vnni::gemv_7bit_s8s8_fp32<float, 24, MTILE>({(uint8_t*)A.data(), scalea.data(), nullptr, k, blks}, B,+ Cf32.data(), n, k, kblock, cache, CacheSize);+ buffer_error(Cref.data(), Cf32.data(), Cref.size(), FP32_ERR);+ }+};+#ifdef BTLA_UT_KERNEL_INTRIN+UT_avx2_gemv sUT_avx2_gemv;+#endif+#endif++} // namespace ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/kernel_jit.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/kernel_jit.cppnew file mode 100644index 000000000..62cdba84d--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/kernel_jit.cpp@@ -0,0 +1,238 @@+#include "kernel_jit.h"+#include "kernel_ut.h"++namespace bestla {+using namespace utils;+namespace ut {+class UT_Memcpy2D_AVX512F {+ public:+ UT_Memcpy2D_AVX512F() {+ UT_START();+ CheckISA(AVX512F);+ ut(512, 432, 432, 432);+ ut(4, 432 * 1024, 432 * 2048, 432 * 1024);+ ut(16, 432 * 1024, 432 * 1024, 432 * 1024);+ }+ void ut(int row, int col, int srcstep, int dststep) {+ printf("Test Case: %d %d %d %d\n", row, col, srcstep, dststep);+ std::vector<float> src(row * srcstep), dst(row * dststep), dstref(row * dststep);+ for (int i = 0; i < src.size(); i++) {+ src[i] = float(i);+ }+ utils::timer<utils::microseconds> tm;+ size_t tsize = (size_t)row * col;++ int constexpr TestLoop = 1000;+ for (int i = 0; i < TestLoop; i++) {+ kernel::jit::JitMemcpy2DAvx512f::forward<float, float>(src.data(), dst.data(), row, col, srcstep, dststep);+ }+ tm.start();+ parallel::Scheduler2D para({UT_Threading::get()->num_threads(), row, col, 4, 64, 0, 0});+ for (size_t i = 0; i < TestLoop; i++) {+ UT_Threading::get()->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ para.getIndex(thdp);+ if (thdp.valid) {+ kernel::jit::JitMemcpy2DAvx512f::forward<float, float>( //+ src.data() + thdp.loc[0] * srcstep + thdp.loc[1], //+ dst.data() + thdp.loc[0] * dststep + thdp.loc[1], //+ thdp.size[0], thdp.size[1], srcstep, dststep);+ }+ });+ }++ auto tper = tm.stop() / TestLoop;+ printf("Kernel Time: %f us\n", tper);+ printf("Bandwidth: %f GB/s\n", tsize / tper / 1000);++ tm.start();+ for (size_t i = 0; i < TestLoop; i++) {+ UT_Threading::get()->parallel_for([&](int tidx) {+ parallel::ThreadProblem2D thdp{tidx};+ para.getIndex(thdp);+ if (thdp.valid) {+ kernel::ref::memcpy2d( //+ src.data() + thdp.loc[0] * srcstep + thdp.loc[1], //+ dstref.data() + thdp.loc[0] * dststep + thdp.loc[1], //+ thdp.size[0], thdp.size[1] * sizeof(float), srcstep * sizeof(float), dststep * sizeof(float));+ }+ });+ }+ tper = tm.stop() / TestLoop;+ printf("Ref Time: %f us\n", tper);+ printf("Bandwidth: %f GB/s\n", tsize / tper / 1000);+ ut::buffer_error<float>(dstref.data(), dst.data(), dstref.size());+ }+};+#ifdef BTLA_UT_KERNEL_JIT+static UT_Memcpy2D_AVX512F sUT_Memcpy2D_AVX512F;+#endif++class UT_Memcpy2D_AVX2 {+ public:+ UT_Memcpy2D_AVX2() {+ UT_START();+ CheckISA(AVX2);+ ut(1, 1, 1, 4);+ ut(2, 2, 2, 4);+ }+ void ut(int row, int col, int srcstep, int dststep) {+ printf("Test Case: %d %d %d %d\n", row, col, srcstep, dststep);+ std::vector<float> src(row * srcstep), dst(row * dststep), dstref(row * dststep);+ for (int i = 0; i < src.size(); i++) {+ src[i] = float(i + 128);+ }++ kernel::jit::JitMemcpy2DAvx2::forward<float, float>(src.data(), dst.data(), row, col, srcstep, dststep);+ kernel::ref::memcpy2d( //+ src.data(), //+ dstref.data(), //+ row, col * sizeof(src[0]), srcstep * sizeof(src[0]), dststep * sizeof(src[0]));+ ut::buffer_error<float>(dstref.data(), dst.data(), dstref.size());+ }+};+#ifdef BTLA_UT_KERNEL_JIT+static UT_Memcpy2D_AVX2 sUT_Memcpy2D_AVX2;+#endif++class UT_PaddngInterleaveCvt {+ public:+ UT_PaddngInterleaveCvt() {+ UT_START();+ CheckISA(AVX512_BF16);+ ut<48, float, bf16>(77, 256, 96);+ ut<48, float, bf16>(512, 432, 512);+ ut<48, float, bf16>(4, 432 * 1024, 4);+ ut<48, float, bf16>(16, 432 * 1024, 16);+ }+ template <int NTile, typename T_SRC, typename T_DST = T_SRC, int RowPack = 4 / sizeof(T_DST)>+ void ut(int rows, int cols, int rows_pad) {+ printf("\ntest_case: %s\t", __PRETTY_FUNCTION__);+ printf("row_%d col_%d row_pad_%d\n", rows, cols, rows_pad);+ const auto cols_pad = utils::padto(cols, NTile);+ const auto src_step = cols;+ const auto dst_step = rows_pad;+ std::vector<T_SRC> src(rows * cols);+ for (size_t i = 0; i < src.size(); i++) src[i] = static_cast<float>(i);+ std::vector<T_DST> dst(cols_pad * dst_step), dst_ref(cols_pad * dst_step);++ utils::timer<utils::microseconds> tm;+ constexpr int TestLoop = 100;+ for (int i = 0; i < TestLoop; i++) {+ kernel::jit::PaddingInterleaveCvt::forward<NTile, T_SRC, T_DST, RowPack>( //+ src.data(), dst.data(), rows, cols, rows_pad, cols_pad, src_step, dst_step);+ }+ tm.start();+ for (int i = 0; i < TestLoop; i++) {+ kernel::jit::PaddingInterleaveCvt::forward<NTile, T_SRC, T_DST, RowPack>( //+ src.data(), dst.data(), rows, cols, rows_pad, cols_pad, src_step, dst_step);+ }+ const auto data_size = sizeof(T_DST) * rows_pad * cols_pad;+ const auto t_kern = tm.stop() / TestLoop;+ printf("Kernel Time: %f us\n", t_kern);+ printf("Bandwidth: %f GB/s\n", data_size / t_kern / 1000);++ tm.start();+ for (int i = 0; i < TestLoop; i++) {+ kernel::jit::PaddingInterleaveCvt::reference<NTile, T_SRC, T_DST, RowPack>( //+ src.data(), dst_ref.data(), rows, cols, rows_pad, cols_pad, src_step, dst_step);+ }+ const auto t_ref = tm.stop() / TestLoop;+ printf("Ref Time: %f us\n", t_ref);+ printf("Bandwidth: %f GB/s\n", data_size / t_ref / 1000);+ ut::buffer_error<T_DST>(dst_ref.data(), dst.data(), dst_ref.size());+ }+};+#ifdef BTLA_UT_KERNEL_JIT+static UT_PaddngInterleaveCvt sUT_Pading_InterleaveCvt;+#endif++class UT_PaddingTransInterleaveCvt {+ public:+ UT_PaddingTransInterleaveCvt() {+ UT_START();+ CheckISA(AVX512_BF16);+ ut<48, float, bf16>(48, 32, 32);+ ut<48, float, bf16>(77, 77, 96);+ ut<48, float, bf16>(77, 256, 256);+ }+ template <int MTile, typename T_SRC, typename T_DST = T_SRC, int ColPack = 4 / sizeof(T_DST)>+ void ut(int rows, int cols, int cols_pad) {+ printf("\ntest_case: %s\t", __PRETTY_FUNCTION__);+ printf("rows_%d cols_%d cols_pad_%d\n", rows, cols, cols_pad);+ const auto rows_pad = utils::padto(rows, MTile);+ const auto src_step = cols;+ const auto dst_step = cols_pad;+ std::vector<T_SRC> src(rows * cols);+ for (size_t i = 0; i < src.size(); i++) src[i] = static_cast<float>(i);+ std::vector<T_DST> dst(rows_pad * dst_step), dst_ref(rows_pad * dst_step);++ utils::timer<utils::microseconds> tm;+ constexpr int TestLoop = 100;+ for (int i = 0; i < TestLoop; i++) {+ kernel::jit::PaddingTransInterleaveCvt::forward<MTile, T_SRC, T_DST, ColPack>( //+ src.data(), dst.data(), rows, cols, rows_pad, cols_pad, src_step, dst_step);+ }+ tm.start();+ for (int i = 0; i < TestLoop; i++) {+ kernel::jit::PaddingTransInterleaveCvt::forward<MTile, T_SRC, T_DST, ColPack>( //+ src.data(), dst.data(), rows, cols, rows_pad, cols_pad, src_step, dst_step);+ }+ const auto data_size = sizeof(T_DST) * cols_pad * rows_pad;+ const auto t_kern = tm.stop() / TestLoop;+ printf("Kernel Time: %f us\n", t_kern);+ printf("Bandwidth: %f GB/s\n", data_size / t_kern / 1000);++ tm.start();+ for (int i = 0; i < TestLoop; i++) {+ kernel::jit::PaddingTransInterleaveCvt::reference<MTile, T_SRC, T_DST, ColPack>( //+ src.data(), dst_ref.data(), rows, cols, rows_pad, cols_pad, src_step, dst_step);+ }+ const auto t_ref = tm.stop() / TestLoop;+ printf("Ref Time: %f us\n", t_ref);+ printf("Bandwidth: %f GB/s\n", data_size / t_ref / 1000);+ ut::buffer_error<T_DST>(dst_ref.data(), dst.data(), dst_ref.size());+ }+};+#ifdef BTLA_UT_KERNEL_JIT+static UT_PaddingTransInterleaveCvt sUT_PaddingTransInterleaveCvt;+#endif++class UT_CScaleInterleavedBF16FP16 {+ public:+ UT_CScaleInterleavedBF16FP16() {+ UT_START();+ CheckISA(AVX512_BF16);+ CheckISA(AVX512_FP16);+ ut<48, 2>(32, 96, 0);+ ut<48, 2>(32, 96, 4);+ }+ template <int NTile, int RowPack = 2>+ void ut(int rows, int cols, int n_offset) {+ printf("\ntest_case: %s\t", __PRETTY_FUNCTION__);+ printf("rows_%d cols_%d n_offset_%d\n", rows, cols, n_offset);+ const auto src_step = rows;+ std::vector<utils::bf16> data(rows * cols);+ std::vector<utils::bf16> data_ref(data.size());+ for (size_t i = 0; i < data.size(); i++) data[i] = static_cast<utils::bf16>(static_cast<float>(i % 523 - 300));+ std::copy_n(data.cbegin(), data.size(), data_ref.begin());++ std::vector<utils::fp16> scale(rows);+ for (size_t i = 0; i < scale.size(); ++i)+ scale[i] = static_cast<utils::fp16>(static_cast<float>(i % 523 - 300) / 10.f);++ kernel::jit::CScaleInterleavedBF16FP16::forward<NTile, RowPack>( //+ data.data(), scale.data(), rows, cols, src_step, n_offset);+ kernel::jit::CScaleInterleavedBF16FP16::reference<NTile, RowPack>( //+ data_ref.data(), scale.data(), rows, cols, src_step, n_offset);++ ut::buffer_error<utils::bf16>(data.data(), data_ref.data(), data.size());+ printf("\n");+ }+};+#ifdef BTLA_UT_KERNEL_JIT+static UT_CScaleInterleavedBF16FP16 sUT_CScaleInterleavedBF16FP16;+#endif++} // namespace ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/kernel_ut.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/kernel_ut.hnew file mode 100644index 000000000..9f6ea32c7--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/kernel_ut.h@@ -0,0 +1,7 @@+#pragma once+#include "bestla_ut.h"+#include "kernel_ref.h"++#ifdef _MSC_VER+#define __PRETTY_FUNCTION__ __FUNCSIG__+#endifdiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/kernel_wrapper.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/kernel_wrapper.cppnew file mode 100644index 000000000..f3bbd8db3--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/kernel_wrapper.cpp@@ -0,0 +1,257 @@+#include "kernel_ut.h"+#include "kernel_wrapper.h"+namespace bestla {+using namespace utils;+namespace ut {+#if 0+class UT_DecompressKBlockS4FP {+ public:+ UT_DecompressKBlockS4FP() {+ UT_START();+ CheckISA(AVX2);+ ut_avx2<BTLA_DTYPE::S4_CLIP, 1, float, float>(410, 24, 24, 24, 0, 128, 24);+ ut_avx2<BTLA_DTYPE::S4_CLIP, 1, float, float>(410, 48, 48, 48, 0, 128, 48);+ }+++ template <BTLA_DTYPE S4_T, int PACK_ROW, typename ST_T, typename DST_T>+ void ut_avx2(int row, int col, int ld_src, int ld_dst, int k_offset, int kblock, int NPad, bool asym = false) {+ printf("Test Case %s_%d_%d: %d %d %d %d %d %d %d %d\n", __FUNCTION__, int(S4_T), PACK_ROW, row, col, ld_src, ld_dst,+ k_offset, kblock, NPad, asym);+ int nk_blk = updiv(row, kblock);+ std::vector<utils::int4x2> s4_wei(row * col / 2);+ std::vector<int8_t> s8_wei(col * row);+ std::vector<DST_T> bf16_wei(ld_dst * row);+ std::vector<DST_T> ref_wei(ld_dst * row);+ std::vector<ST_T> scales(NPad * nk_blk);+ std::vector<int8_t> zero_points(NPad * nk_blk);+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-128), int8_t(127));+ fill_buffer_randn(scales.data(), scales.size(), ST_T(0.01f), ST_T(0.02f));+ fill_buffer_randn(zero_points.data(), zero_points.size(), (int8_t)(-5), (int8_t)(5));++ for (int i = 0; i < col * row; i += 2) {+ s4_wei[i / 2].x = utils::int4x2::convert(s8_wei[i]);+ s4_wei[i / 2].y = utils::int4x2::convert(s8_wei[i + 1]);+ }+ kernel::wrapper::DecompressKBlockS4Fp<DST_T, PACK_ROW>::template forward<BTLA_ISA::NoSIMD, ST_T, S4_T>(+ s4_wei.data(), bf16_wei.data(), row, col, ld_src, ld_dst, scales.data(), asym ? zero_points.data() : nullptr,+ k_offset, kblock, NPad, cache, CacheSize);+ kernel::wrapper::DecompressKBlockS4Fp<DST_T, PACK_ROW>::template forward<BTLA_ISA::AVX2, ST_T, S4_T>(+ s4_wei.data(), ref_wei.data(), row, col, ld_src, ld_dst, scales.data(), asym ? zero_points.data() : nullptr,+ k_offset, kblock, NPad, cache, CacheSize);+ DST_T thres = DST_T(0.01f);+ if constexpr (std::is_same_v<DST_T, utils::bf16>) {+ thres = DST_T(BF16_ERR);+ }+ ut::buffer_error(ref_wei.data(), bf16_wei.data(), bf16_wei.size(), thres);+ }+};+#ifdef BTLA_UT_KERNEL_WRAPPER+static UT_DecompressKBlockS4FP sUT_DecompressKBlockS4FP;+#endif+#endif++class UT_DecompressKBlockF4FP {+ public:+ UT_DecompressKBlockF4FP() {+ UT_START();+ CheckISA(AVX2);+ ut<float, 1, BTLA_DTYPE::F4_BNB, BTLA_ISA::AVX2>(35, 48, 48, 48, 0, 12, 48);+ ut<float, 1, BTLA_DTYPE::F4_BNB, BTLA_ISA::AVX2>(11, 48, 48, 48, 0, 20, 48);+ CheckISA(AVX512F);+ ut<float, 1, BTLA_DTYPE::F4_BNB, BTLA_ISA::AVX512F>(35, 48, 48, 48, 0, 12, 48);+ ut<float, 1, BTLA_DTYPE::F4_BNB, BTLA_ISA::AVX512F>(11, 48, 48, 48, 0, 20, 48);+ }++ template <typename T, int PACK_ROW, BTLA_DTYPE F4_T, BTLA_ISA ISA_T>+ void ut(int row, int col, int ld_src, int ld_dst, int k_offset, int kblock, int NPad) {+ std::vector<utils::f4x2> f4_wei(row * col / 2);+ std::vector<int8_t> s8_wei(col * row);+ std::vector<T> wei(col * row);+ std::vector<T> ref_wei(col * row);+ std::vector<T> scales(col * updiv(row, kblock));+ fill_buffer_randn(s8_wei.data(), s8_wei.size(), int8_t(-127), int8_t(127));+ fill_buffer_randn(scales.data(), scales.size(), T(1.f), T(10.f));+ for (int i = 0; i < col * row; i += 2) {+ f4_wei[i / 2].x = utils::int4x2::convert(s8_wei[i]);+ f4_wei[i / 2].y = utils::int4x2::convert(s8_wei[i + 1]);+ }+ kernel::wrapper::DecompressKBlockF4Fp<T, PACK_ROW>::template forward<ISA_T, T, F4_T>(+ f4_wei.data(), wei.data(), row, col, ld_src, ld_dst, scales.data(), k_offset, kblock, NPad, cache, CacheSize);+ kernel::wrapper::DecompressKBlockF4Fp<T, PACK_ROW>::template forward<BTLA_ISA::NoSIMD, T, F4_T>(+ f4_wei.data(), ref_wei.data(), row, col, ld_src, ld_dst, scales.data(), k_offset, kblock, NPad, cache,+ CacheSize);+ ut::buffer_error(ref_wei.data(), wei.data(), wei.size(), T(0.01f));+ }+};+#ifdef BTLA_UT_KERNEL_WRAPPER+static UT_DecompressKBlockF4FP sUT_DecompressKBlockF4FP;+#endif++class UT_memcpy2d {+ public:+ UT_memcpy2d() {+ UT_START();+ CheckISA(AVX2);+ ut<utils::bf16, BTLA_ISA::AVX2>(32, 64);+ ut<utils::bf16, BTLA_ISA::AVX2>(32, 64 + 3);+ ut<utils::fp16, BTLA_ISA::AVX2>(32, 64);+ ut<utils::fp16, BTLA_ISA::AVX2>(32, 64 + 3);+ CheckISA(AVX512F);+ ut<utils::bf16, BTLA_ISA::AVX512F>(32, 64);+ ut<utils::bf16, BTLA_ISA::AVX512F>(32, 64 + 3);+ ut<utils::fp16, BTLA_ISA::AVX512F>(32, 64);+ ut<utils::fp16, BTLA_ISA::AVX512F>(32, 64 + 3);+ }++ template <typename T_SRC, BTLA_ISA ISA_T>+ void ut(int row, int col) {+ printf("%s %d %d\n", __FUNCTION__, row, col);+ std::vector<T_SRC> src(row * col);+ std::vector<float> dst(row * col);+ std::vector<T_SRC> src_revert(row * col);++ fill_buffer_randn(src.data(), src.size(), T_SRC(-127), T_SRC(127));+ if (std::is_same_v<T_SRC, utils::bf16>) {+ kernel::wrapper::Memcpy2DBf16CvtFp32::forward<ISA_T>(src.data(), dst.data(), row, col, col * sizeof(T_SRC),+ col * sizeof(float), true);+ kernel::wrapper::Memcpy2DFp32CvtBf16::forward<ISA_T>(dst.data(), src_revert.data(), row, col, col * sizeof(float),+ col * sizeof(T_SRC), true);+ }+ if (std::is_same_v<T_SRC, utils::fp16>) {+ kernel::wrapper::Memcpy2DFp16CvtFp32::forward<ISA_T>(src.data(), dst.data(), row, col, col * sizeof(T_SRC),+ col * sizeof(float), true);+ kernel::wrapper::Memcpy2DFp32CvtFp16::forward<ISA_T>(dst.data(), src_revert.data(), row, col, col * sizeof(float),+ col * sizeof(T_SRC), true);+ }+ ut::buffer_error(src.data(), src_revert.data(), src.size(), T_SRC(1.f));+ }+};+#ifdef BTLA_UT_KERNEL_WRAPPER+static UT_memcpy2d sUT_memcpy2d;+#endif++class UT_PaddingInterleaveMN {+ public:+ UT_PaddingInterleaveMN() {+ UT_START();+ // ut<48, 2, bf16, bf16>(128, 128, 2); // TO IMPLEMENT+ CheckISA(AVX512_FP16);+ ut<32, 2, fp16, bf16>(128, 128, 2);+ }+ template <int NTile, int RowPack, typename T_SRC, typename T_DST>+ void ut(int row, int col, int row_tile) {+ printf("%s %d %d %d\n", __FUNCTION__, row, col, row_tile);+ int row_pad = padto(row, row_tile);+ int col_pad = padto(col, NTile);++ aligned_vector<T_SRC> src(row * col);+ aligned_vector<T_DST> dst(row_pad * col_pad), ref(row_pad * col_pad);+ for (size_t i = 0; i < src.size(); i++) src[i] = static_cast<T_SRC>(float(i));++ kernel::wrapper::PaddingInterleaveMN<NTile, RowPack, T_SRC, T_DST>::template forward<BTLA_ISA::NoSIMD>(+ src.data(), ref.data(), row, col, row_pad, col_pad, row_pad, col);+ kernel::wrapper::PaddingInterleaveMN<NTile, RowPack, T_SRC, T_DST>::template forward<BTLA_ISA::AVX512_FP16>(+ src.data(), dst.data(), row, col, row_pad, col_pad, col, row_pad);+ ut::buffer_error(dst.data(), ref.data(), dst.size(), T_DST(100));+ }+};+#ifdef BTLA_UT_KERNEL_WRAPPER+static UT_PaddingInterleaveMN sUT_PaddingInterleaveMN;+#endif++class UT_PaddingTransInterleaveMN {+ public:+ UT_PaddingTransInterleaveMN() {+ UT_START();+ CheckISA(AVX512_FP16);+ // ut<48, 2, bf16, bf16>(128, 128, 2); // TO IMPLEMENT+ ut<32, 2, fp16, bf16>(128, 128, 2);+ }+ template <int MTile, int ColPack, typename T_SRC, typename T_DST>+ void ut(int row, int col, int col_tile) {+ printf("%s %d %d %d\n", __FUNCTION__, row, col, col_tile);+ int row_pad = padto(row, MTile);+ int col_pad = padto(col, col_tile);++ aligned_vector<T_SRC> src(row * col);+ aligned_vector<T_DST> dst(col_pad * row_pad), ref(col_pad * row_pad);+ for (size_t i = 0; i < src.size(); i++) src[i] = static_cast<T_SRC>(float(i));++ kernel::wrapper::PaddingTransInterleaveMN<MTile, ColPack, T_SRC, T_DST>::template forward<BTLA_ISA::NoSIMD>(+ src.data(), ref.data(), row, col, row_pad, col_pad, row_pad, col);+ kernel::wrapper::PaddingTransInterleaveMN<MTile, ColPack, T_SRC, T_DST>::template forward<BTLA_ISA::AVX512_FP16>(+ src.data(), dst.data(), row, col, row_pad, col_pad, col, row_pad);+ ut::buffer_error(dst.data(), ref.data(), dst.size(), T_DST(100));+ }+};+#ifdef BTLA_UT_KERNEL_WRAPPER+static UT_PaddingTransInterleaveMN sUT_PaddingTransInterleaveMN;+#endif++class UT_RevertPaddingInterleaveMN {+ public:+ UT_RevertPaddingInterleaveMN() {+ UT_START();+ ut<48, 4, char>(128, 128, 4); // vnni+ ut<48, 1, float>(128, 128, 1); // 512f+ ut<48, 4, char>(128, 32, 64); // amxint8+ }+ template <int NTile, int PackRow, typename T>+ void ut(int row, int col, int rowtile) {+ printf("%s %d %d %d\n", __FUNCTION__, row, col, rowtile);+ int rowpad = padto(row, rowtile);+ int colpad = padto(col, NTile);+ aligned_vector<T> src(row * col), packed(rowpad * colpad);+ for (size_t i = 0; i < src.size(); i++) {+ src[i] = static_cast<T>(i);+ }+ aligned_vector<T> reverted(row * col);+ kernel::wrapper::PaddingInterleaveMN<NTile, PackRow, T>::template forward<BTLA_ISA::NoSIMD>(+ src.data(), packed.data(), row, col, rowpad, colpad, col, rowpad);+ kernel::wrapper::RevertPaddingInterleaveMN<NTile, PackRow, T>::template forward<BTLA_ISA::NoSIMD>(+ packed.data(), reverted.data(), row, col, rowpad, colpad, rowpad, col);+ ut::buffer_error(src.data(), reverted.data(), reverted.size());+ }+};+#ifdef BTLA_UT_KERNEL_WRAPPER+static UT_RevertPaddingInterleaveMN sUT_RevertPaddingInterleaveMN;+#endif++class UT_LayerNormalization {+ public:+ UT_LayerNormalization() {+ UT_START();+ CheckISA(AVX2);+ ut<float, BTLA_ISA::AVX2>(4096, false, true, true);+ ut<float, BTLA_ISA::AVX2>(4096, false, false, false);+ ut<float, BTLA_ISA::AVX2>(111, false, true, true);+ ut<float, BTLA_ISA::AVX2>(111, true, true, true);+ CheckISA(AVX512F);+ ut<float, BTLA_ISA::AVX512F>(4096, false, true, true);+ ut<float, BTLA_ISA::AVX512F>(4096, false, false, false);+ ut<float, BTLA_ISA::AVX512F>(111, false, true, true);+ ut<float, BTLA_ISA::AVX512F>(111, true, true, true);+ }+ template <typename T, BTLA_ISA ISA>+ void ut(int norm_size, bool simplified, bool hasscale, bool hasbias) {+ printf("%s %d\n", __FUNCTION__, norm_size);+ aligned_vector<T> src(norm_size), dst(norm_size), bias(norm_size), scale(norm_size), ref(norm_size);+ fill_buffer_randn(src.data(), src.size(), -0.5f, 0.5f);+ fill_buffer_randn(bias.data(), bias.size(), -0.5f, 0.5f);+ fill_buffer_randn(scale.data(), scale.size(), 0.1f, 1.f);+ T mean = 0.f, mean_square = 0.f;+ kernel::wrapper::LayerNormalization::forward<BTLA_ISA::NoSIMD>(src.data(), hasscale ? scale.data() : nullptr,+ hasbias ? bias.data() : nullptr, 0.00001f, norm_size,+ ref.data(), &mean, &mean_square, simplified);+ kernel::wrapper::LayerNormalization::forward<ISA>(src.data(), hasscale ? scale.data() : nullptr,+ hasbias ? bias.data() : nullptr, 0.00001f, norm_size, dst.data(),+ &mean, &mean_square, simplified);+ buffer_error(ref.data(), dst.data(), ref.size(), 0.01f);+ }+};+#ifdef BTLA_UT_KERNEL_WRAPPER+UT_LayerNormalization sUT_LayerNormalization;+#endif+} // namespace ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_benchmark.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_benchmark.cppnew file mode 100644index 000000000..71d140a96--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_benchmark.cpp@@ -0,0 +1,1942 @@+#include <stdio.h>+#include <sycl/detail/builtins/builtins.hpp>+#include <sycl/functional.hpp>+#include <type_traits>+#include "bestla_wrapper.h"+#include "bestla_ut.h"+#include "sycl_ut.h"+#include "sycl/sycl_wrapper.h"++#if 0+namespace c10 {+using DeviceIndex = int8_t;+}+#define ARK_XPU 1+#include "../../../dispatcher/include/dnnl_wrapper.hpp"+using namespace ark;+#endif++namespace bestla {+using namespace ut;+using namespace utils;+using namespace sycl_utils;+using namespace sycl_gemm;+namespace sycl_ut {+namespace {+template <bool IsE4M3>+inline uint8_t sanitize_fp8_byte(uint8_t raw) {+ if constexpr (IsE4M3) {+ uint8_t exp = (raw >> 3) & 0xF;+ uint8_t mant = raw & 0x7;+ if (exp == 0xF && mant == 0x7) {+ raw = (raw & 0xF8) | 0x6;+ }+ } else {+ uint8_t exp = (raw >> 2) & 0x1F;+ if (exp == 0x1F) {+ raw = (raw & 0x83) | (0x1E << 2);+ }+ }+ return raw;+}++template <bool IsE4M3>+inline void sanitize_fp8_buffer(uint8_t* data, size_t size) {+#pragma omp parallel for+ for (size_t i = 0; i < size; ++i) {+ data[i] = sanitize_fp8_byte<IsE4M3>(data[i]);+ }+}++template <bool IsE4M3>+inline float decode_fp8_byte(uint8_t raw) {+ const uint8_t mag = raw & 0x7F;+ const float v = IsE4M3 ? sycl_prologue_b::fp8_lut::lut_e4m3_128[mag] : sycl_prologue_b::fp8_lut::lut_e5m2_128[mag];+ return (raw & 0x80) ? -v : v;+}+} // namespace++int constexpr TestMs = 1000;++class Benchmark_Fp32Fp32 {+ public:+ Benchmark_Fp32Fp32() {+ UT_START();+ benchmark_all(1, 2, 3, true);+ benchmark_all(1, 1024, 768, true);+ benchmark_all(500, 1024, 768, true);+ // benchmark_all(1, 4096, 4096);+ // benchmark_all(4096, 4096, 4096);+ }++ using AType = float;+ using BType = float;+ using CType = float;++ void benchmark(int m, int n, int k, int batch, AType* A, BType* B, CType* C, CType* Bias, bool verify) {+ using LOG = timer_statistics_logger<TestMs * 2>;+ LOG log;+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ float timems = TestMs;+ utils::timer<std::chrono::milliseconds> tm;+ auto A_d = A;+ auto B_d = B;+ auto C_d = C;+ auto psize = (size_t)m * n * k * 2;+ using CFG = xve::SGemmCfg;+ q->wait();+ for (size_t i = 0; i < batch; i++) {+ Launcher<CFG, xve::GemmCore>::run(+ q, {A_d + i * m * k, B_d + i * n * k, C_d + i * m * n, m, n, k, k, k, n, Bias + i * n});+ }+ if (verify) {+ return;+ }+ tm.start();+ utils::timer<std::chrono::microseconds> tm1;+ while (tm.stop() < timems) {+ tm1.start();+ for (size_t i = 0; i < batch; i++) {+ Launcher<CFG, xve::GemmCore>::run(+ q, {A_d + i * m * k, B_d + i * n * k, C_d + i * m * n, m, n, k, k, k, n, Bias + i * n});+ }+ q->wait();+ log.add(tm1.stop() / batch);+ if (tm.stop() >= timems) {+ break;+ }+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ auto memsize = gemm_memsize(m, n, k, BTLA_DTYPE::F32, BTLA_DTYPE::F32, BTLA_DTYPE::F32);+ double mms = double(memsize) / log.min_val / 1e6;+ printf(" %s Flops:%.3f GOPS Bandwidth:%.3f GB/s\n", log.get_log_str(), flops, mms);+ }++ void benchmark_all(int m, int n, int k, bool verify = false) {+ auto memsize = gemm_memsize(m, n, k, BTLA_DTYPE::F32, BTLA_DTYPE::F32, BTLA_DTYPE::F32);+ auto batch = verify ? 16 : auto_batch(memsize);+ printf("%d %d %d %d %s %s %s\n", m, n, k, batch, bestla_dtype_str(BTLA_DTYPE::F32),+ bestla_dtype_str(BTLA_DTYPE::F32), bestla_dtype_str(BTLA_DTYPE::F32));+ float testtime = float(TestMs);+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ sycl_vector<AType> dA(size_t(m) * k * batch, q);+ sycl_vector<BType> dB(size_t(n) * k * batch, q);+ sycl_vector<CType> dC(size_t(m) * n * batch, q);+ sycl_vector<CType> dBias(n * batch, q);+ avector<float> matC(m * n), ref(m * n), bias(n);+ avector<float> matA(m * k), matBT(k * n);+ if (verify) {+ fill_buffer_randn(matA.data(), matA.size(), (-0.5f), (0.5f));+ fill_buffer_randn(matBT.data(), matBT.size(), (-0.5f), (0.5f));+ fill_buffer_randn(bias.data(), bias.size(), (-0.5f), (0.5f));+ for (size_t i = 0; i < batch; i++) {+ q->memcpy(dA.data() + i * m * k, matA.data(), m * k * 4);+ q->memcpy(dB.data() + i * n * k, matBT.data(), n * k * 4);+ q->memcpy(dBias.data() + i * n, bias.data(), n * 4);+ }+ q->wait();+ }++ benchmark(m, n, k, batch, dA.data(), dB.data(), dC.data(), dBias.data(), verify);++ if (verify) {+ gemmref_fp32fp32fp32(m, n, k, matA.data(), matBT.data(), ref.data(), k, k, n, bias.data(), true);+ q->memcpy(matC.data(), dC.data(), m * n * 4).wait();+ buffer_error(ref.data(), matC.data(), matC.size(), (1.f));+ }+ }+};+#ifdef BTLA_UT_SYCL+static Benchmark_Fp32Fp32 sBenchmark_Fp32Fp32;+#endif++class Benchmark_Fp16Fp16 {+ public:+ Benchmark_Fp16Fp16() {+ UT_START();+ benchmark_all(500, 1024, 768, true);+ benchmark_all(1, 1024, 768, true);+ benchmark_all(1, 4096, 4096);+ benchmark_all(4096, 4096, 4096);+ }++ using AType = sycl::half;+ using BType = sycl::half;+ using CType = sycl::half;++ void benchmark(int m, int n, int k, int batch, AType* A, BType* B, CType* C, CType* Bias, bool verify) {+ using LOG = timer_statistics_logger<TestMs * 2>;+ LOG log;+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ float timems = TestMs;+ utils::timer<std::chrono::milliseconds> tm;+ auto A_d = A;+ auto B_d = B;+ auto C_d = C;+ auto psize = (size_t)m * n * k * 2;+ using CFG = xmx::HGemmCfg;+ q->wait();+ for (size_t i = 0; i < batch; i++) {+ Launcher<CFG, xmx::GemmCore>::run(+ q, {A_d + i * m * k, B_d + i * n * k, C_d + i * m * n, m, n, k, k, k, n, Bias + i * n});+ }+ if (verify) {+ return;+ }+ tm.start();+ utils::timer<std::chrono::microseconds> tm1;+ while (tm.stop() < timems) {+ tm1.start();+ for (size_t i = 0; i < batch; i++) {+ Launcher<CFG, xmx::GemmCore>::run(+ q, {A_d + i * m * k, B_d + i * n * k, C_d + i * m * n, m, n, k, k, k, n, Bias + i * n});+ }+ q->wait();+ log.add(tm1.stop() / batch);+ if (tm.stop() >= timems) {+ break;+ }+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ auto memsize = gemm_memsize(m, n, k, BTLA_DTYPE::F16, BTLA_DTYPE::F16, BTLA_DTYPE::F16);+ double mms = double(memsize) / log.min_val / 1e6;+ printf(" %s Flops:%.3f GOPS Bandwidth:%.3f GB/s\n", log.get_log_str(), flops, mms);+ }++ void benchmark_all(int m, int n, int k, bool verify = false) {+ auto memsize = gemm_memsize(m, n, k, BTLA_DTYPE::F16, BTLA_DTYPE::F16, BTLA_DTYPE::F16);+ auto batch = verify ? 16 : auto_batch(memsize);+ auto dev = UT_Device::get();+ printf("%s %d %d %d %d %s %s %s\n", dev->getName().c_str(), m, n, k, batch, bestla_dtype_str(BTLA_DTYPE::F16),+ bestla_dtype_str(BTLA_DTYPE::F16), bestla_dtype_str(BTLA_DTYPE::F16));+ float testtime = float(TestMs);++ auto q = dev->getQueue();++ sycl_vector<AType> dA(size_t(m) * k * batch, q);+ sycl_vector<BType> dB(size_t(n) * k * batch, q);+ sycl_vector<CType> dC(size_t(m) * n * batch, q);+ sycl_vector<CType> dBias(n * batch, q);+ avector<utils::fp16> matC(m * n), ref(m * n), bias(n);+ avector<utils::fp16> matA(m * k), matBT(k * n);+ if (verify) {+ fill_buffer_randn(matA.data(), matA.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ fill_buffer_randn(matBT.data(), matBT.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ fill_buffer_randn(bias.data(), bias.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ for (size_t i = 0; i < batch; i++) {+ q->memcpy(dA.data() + i * m * k, matA.data(), m * k * 2);+ q->memcpy(dB.data() + i * n * k, matBT.data(), n * k * 2);+ q->memcpy(dBias.data() + i * n, bias.data(), n * 2);+ }+ q->wait();+ }++ benchmark(m, n, k, batch, dA.data(), dB.data(), dC.data(), dBias.data(), verify);++ if (verify) {+ gemmref_fp16fp16fp16(m, n, k, matA.data(), matBT.data(), ref.data(), k, k, n, bias.data(), true);+ q->memcpy(matC.data(), dC.data(), m * n * 2).wait();+ buffer_error(ref.data(), matC.data(), matC.size(), utils::fp16(1.f));+ }+ }+};+#ifdef BTLA_UT_SYCL+static Benchmark_Fp16Fp16 sBenchmark_Fp16Fp16;+#endif++class Benchmark_Bf16Bf16 {+ public:+ Benchmark_Bf16Bf16() {+ UT_START();+ benchmark_all(500, 1024, 768, true);+ benchmark_all(1, 1024, 768, true);+ benchmark_all(1, 4096, 4096);+ benchmark_all(4096, 4096, 4096);+ }++ using AType = sycl::ext::oneapi::bfloat16;+ using BType = sycl::ext::oneapi::bfloat16;+ using CType = sycl::ext::oneapi::bfloat16;++ void benchmark(int m, int n, int k, int batch, AType* A, BType* B, CType* C, CType* Bias, bool verify) {+ using LOG = timer_statistics_logger<TestMs * 2>;+ LOG log;+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ float timems = TestMs;+ utils::timer<std::chrono::milliseconds> tm;+ auto A_d = A;+ auto B_d = B;+ auto C_d = C;+ auto psize = (size_t)m * n * k * 2;+ using CFG = xmx::HGemmBf16Cfg;+ q->wait();+ for (size_t i = 0; i < batch; i++) {+ Launcher<CFG, xmx::GemmCore>::run(+ q, {A_d + i * m * k, B_d + i * n * k, C_d + i * m * n, m, n, k, k, k, n, Bias + i * n});+ }+ if (verify) {+ return;+ }+ tm.start();+ utils::timer<std::chrono::microseconds> tm1;+ while (tm.stop() < timems) {+ tm1.start();+ for (size_t i = 0; i < batch; i++) {+ Launcher<CFG, xmx::GemmCore>::run(+ q, {A_d + i * m * k, B_d + i * n * k, C_d + i * m * n, m, n, k, k, k, n, Bias + i * n});+ }+ q->wait();+ log.add(tm1.stop() / batch);+ if (tm.stop() >= timems) {+ break;+ }+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ auto memsize = gemm_memsize(m, n, k, BTLA_DTYPE::BF16, BTLA_DTYPE::BF16, BTLA_DTYPE::BF16);+ double mms = double(memsize) / log.min_val / 1e6;+ printf(" %s Flops:%.3f GOPS Bandwidth:%.3f GB/s\n", log.get_log_str(), flops, mms);+ }++ void benchmark_all(int m, int n, int k, bool verify = false) {+ auto memsize = gemm_memsize(m, n, k, BTLA_DTYPE::BF16, BTLA_DTYPE::BF16, BTLA_DTYPE::BF16);+ auto batch = verify ? 16 : auto_batch(memsize);+ printf("%d %d %d %d %s %s %s\n", m, n, k, batch, bestla_dtype_str(BTLA_DTYPE::BF16),+ bestla_dtype_str(BTLA_DTYPE::BF16), bestla_dtype_str(BTLA_DTYPE::BF16));+ float testtime = float(TestMs);+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ sycl_vector<AType> dA(size_t(m) * k * batch, q);+ sycl_vector<BType> dB(size_t(n) * k * batch, q);+ sycl_vector<CType> dC(size_t(m) * n * batch, q);+ sycl_vector<CType> dBias(n * batch, q);+ avector<utils::bf16> matC(m * n), ref(m * n), bias(n);+ avector<utils::bf16> matA(m * k), matBT(k * n);+ if (verify) {+ fill_buffer_randn(matA.data(), matA.size(), utils::bf16(-0.5f), utils::bf16(0.5f));+ fill_buffer_randn(matBT.data(), matBT.size(), utils::bf16(-0.5f), utils::bf16(0.5f));+ fill_buffer_randn(bias.data(), bias.size(), utils::bf16(-0.5f), utils::bf16(0.5f));+ for (size_t i = 0; i < batch; i++) {+ q->memcpy(dA.data() + i * m * k, matA.data(), m * k * 2);+ q->memcpy(dB.data() + i * n * k, matBT.data(), n * k * 2);+ q->memcpy(dBias.data() + i * n, bias.data(), n * 2);+ }+ q->wait();+ }++ benchmark(m, n, k, batch, dA.data(), dB.data(), dC.data(), dBias.data(), verify);++ if (verify) {+ gemmref_bf16bf16bf16(m, n, k, matA.data(), matBT.data(), ref.data(), k, k, n, bias.data(), true);+ q->memcpy(matC.data(), dC.data(), m * n * 2).wait();+ buffer_error(ref.data(), matC.data(), matC.size(), utils::bf16(1.f));+ }+ }+};+#ifdef BTLA_UT_SYCL+static Benchmark_Bf16Bf16 sBenchmark_Bf16Bf16;+#endif++#if 0+++class Benchmark_DequantF8 {+ public:+ Benchmark_DequantF8() {+ UT_START();+ benchmark_all<true>(4096, 4096, 64);+ benchmark_all<false>(4096, 4096, 64);+ }++ private:+ template <bool IsE4M3>+ void benchmark_all(int n, int k, int blocksize) {+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Benchmark FP8 %s dequant: n=%d k=%d blk=%d Device:%s\n", IsE4M3 ? "E4M3" : "E5M2", n, k, blocksize,+ dev->getName().c_str());+ int blks = k / blocksize;+ avector<uint8_t> rawB(size_t(k) * n);+ avector<float> scale(size_t(blks) * n);+ fill_buffer_randn(scale.data(), scale.size(), 0.01f, 0.03f);+ fill_buffer_randn(rawB.data(), rawB.size(), uint8_t(0), uint8_t(255));+ sanitize_fp8_buffer<IsE4M3>(rawB.data(), rawB.size());++ sycl_vector<uint8_t> dB(rawB.size(), q);+ sycl_vector<float> dS(scale.size(), q);+ sycl_vector<float> dDst(size_t(n) * k, q);+ q->memcpy(dB.data(), rawB.data(), rawB.size()).wait();+ q->memcpy(dS.data(), scale.data(), scale.size() * sizeof(float)).wait();++ using LOG = timer_statistics_logger<TestMs * 2>;+ float testtime = float(TestMs);+ printf(" Layout:NT\n");+ benchmark_impl<IsE4M3, false, LOG>(n, k, blocksize, blks, dB.data(), dS.data(), dDst.data(), q, testtime);+ printf(" Layout:T\n");+ benchmark_impl<IsE4M3, true, LOG>(n, k, blocksize, blks, dB.data(), dS.data(), dDst.data(), q, testtime);+#if 0+ printf(" oneDNN reorder\n");+ benchmark_onednn<IsE4M3, LOG>(n, k, blocksize, blks, dB.data(), dS.data(), dDst.data(), q, testtime);+#endif+ }++ template <bool IsE4M3, bool IsTrans, typename LOG_T>+ void benchmark_impl(int n, int k, int blocksize, int blks, uint8_t* B, float* S, float* Out, sycl::queue* q,+ float timems) {+ using ProB = sycl_prologue_b::WeightF8T<xve::DefaultSGemmCore, float, IsE4M3>;+ LOG_T log;+ utils::timer<std::chrono::milliseconds> tm;+ tm.start();+ while (tm.stop() < timems) {+ sycl::event ev;+ if constexpr (IsTrans) {+ ev = ProB::template dequant_T<sycl_prologue_b::KernelConfigTrans>(n, k, blocksize, {B, S, blks}, Out, q);+ } else {+ ev = ProB::template dequant<sycl_prologue_b::KernelConfigTrans>(n, k, blocksize, {B, S, blks}, Out, q);+ }+ ev.wait();+ log.add(event_helper::execute_time(ev) * 1000);+ if (tm.stop() >= timems) {+ break;+ }+ }+ log.record();+ size_t memsize = size_t(n) * k * sizeof(float) + size_t(n) * blks * sizeof(float) + size_t(n) * k;+ double band = double(memsize) / log.min_val / 1e6;+ printf(" %s MemoryBandwidth:%.3fGB/s\n", log.get_log_str(), band);+ }+#if 0+ template <bool IsE4M3, typename LOG_T>+ void benchmark_onednn(int n, int k, int blocksize, int blks, uint8_t* B, float* S, float* Out, sycl::queue* q,+ float timems) {+ LOG_T log;+ utils::timer<std::chrono::milliseconds> tm;+ auto bt = IsE4M3 ? DnnlWrapper::dt::f8_e4m3 : DnnlWrapper::dt::f8_e5m2;+ tm.start();+ while (tm.stop() < timems) {+ utils::timer<utils::microseconds> kernel_tm;+ kernel_tm.start();+ DnnlWrapper::dequant_fp8(*q, 0, k, n, B, bt, Out, DnnlWrapper::dt::f32, S, DnnlWrapper::dt::f32, blocksize, true);+ log.add(kernel_tm.stop());+ if (tm.stop() >= timems) {+ break;+ }+ }+ log.record();+ size_t memsize = size_t(n) * k * sizeof(float) + size_t(n) * blks * sizeof(float) + size_t(n) * k;+ double band = double(memsize) / log.min_val / 1e6;+ printf(" %s [oneDNN] MemoryBandwidth:%.3fGB/s\n", log.get_log_str(), band);+ }+#endif+};+#ifdef BTLA_UT_SYCL+static Benchmark_DequantF8 sBenchmark_DequantF8;+#endif++#endif++class Benchmark_S4 {+ public:+ Benchmark_S4() {+ UT_START();+ gemv_next<float>(1, 4096, 4096, 32);+ gemv_next<float, true>(1, 4096, 4096, 32);+ gemv_next<utils::fp16>(1, 4096, 4096, 32);+ gemv_next<utils::fp16, true>(1, 4096, 4096, 32);++ dequant_next_s8<float, true>(4096, 4096, 32, 0);+ dequant_next_s8<float>(4096, 4096, 32, 0);+ dequant_next_s8<utils::fp16, true>(4096, 4096, 32, 0);+ dequant_next_s8<utils::fp16>(4096, 4096, 32, 0);+ dequant_next_s8<float, true>(4096, 4096, 32, 1);+ dequant_next_s8<float>(4096, 4096, 32, 1);+ dequant_next_s8<utils::fp16, true>(4096, 4096, 32, 1);+ dequant_next_s8<utils::fp16>(4096, 4096, 32, 1);++ dequant_next<float>(4096, 4096, 32);+ dequant_next<utils::fp16>(4096, 4096, 32);+ dequant_next<float, true>(4096, 4096, 32);+ dequant_next<utils::fp16, true>(4096, 4096, 32);+ }+ static constexpr size_t nbits = 4;++ template <typename T, bool asym = false>+ void gemv_next(int m, int n, int k, int blocksize) {+ int blks = k / blocksize;+ auto memsize = (size_t)(n * k * nbits / 8 + n * blks * sizeof(T)) + (m * k + m * n) * sizeof(T);+ if (asym) memsize += n * blks;+ auto batch = auto_batch(memsize);+ using LOG = timer_statistics_logger<TestMs * 2>;+ LOG log;+ auto dev = UT_Device::get();+ printf("Test Case %s: %d %d %d Device:%s\n", __FUNCTION__, n, k, blocksize, dev->getName().c_str());+ auto q = dev->getQueue();+ using ST = std::conditional_t<std::is_same_v<T, utils::fp16>, sycl::half, float>;+ avector<uint8_t> rawB(k * n / 2);+ avector<T> scale(blks * n), A(m * k), C(n * m);+ avector<float> Bf32(n * k), Af32(m * k), ref(n * m);+ avector<int8_t> zp(n * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(A.data(), A.size(), T(-.5f), T(0.5f));+ fill_buffer_randn(scale.data(), scale.size(), T(0.01f), T(0.03f));+ fill_buffer_randn(rawB.data(), rawB.size(), uint8_t(0), uint8_t(255));+ auto srcptr = (utils::int4x2*)rawB.data();+ Af32 = A.template to<float>();+ for (int j = 0; j < n; j += 1) {+ for (int i = 0; i < k; i += 2) {+ auto tmp = srcptr[i / 2 + j * k / 2];+ auto noffset = i / blocksize + j * blks;+ if constexpr (asym) {+ Bf32[i * n + j] = static_cast<float>(static_cast<int8_t>(tmp.x) - 8 - zp[noffset]) * (float)scale[noffset];+ Bf32[(i + 1) * n + j] =+ static_cast<float>(static_cast<int8_t>(tmp.y) - 8 - zp[noffset]) * (float)scale[noffset];+ } else {+ Bf32[i * n + j] = static_cast<float>(static_cast<int8_t>(tmp.x) - 8) * (float)scale[noffset];+ Bf32[(i + 1) * n + j] = static_cast<float>(static_cast<int8_t>(tmp.y) - 8) * (float)scale[noffset];+ }+ }+ }+ gemmref_fp32fp32fp32(m, n, k, Af32.data(), Bf32.data(), ref.data(), k, n, n);+ sycl_vector<ST> dS(scale.size() * batch, q), dA(A.size() * batch, q), dC((size_t)n * k * batch, q);+ sycl_vector<uint8_t> dB(rawB.size() * batch, q);+ sycl_vector<int8_t> dZP(zp.size() * batch, q);+ for (size_t i = 0; i < batch; i++) {+ q->memcpy(dS.data() + i * scale.size(), scale.data(), scale.size() * sizeof(T));+ q->memcpy(dB.data() + i * rawB.size(), rawB.data(), rawB.size() * 1);+ q->memcpy(dA.data() + i * A.size(), A.data(), A.size() * sizeof(T));+ q->memcpy(dZP.data() + i * zp.size(), zp.data(), zp.size() * 1);+ }+ q->wait();+ utils::timer<std::chrono::milliseconds> tm;+ utils::timer<std::chrono::microseconds> tm1;+ using ProB = sycl_prologue_b::WeightS4T<ST>;+ using Cfg = std::conditional_t<std::is_same_v<T, float>, typename ProB::CfgGemvF32, typename ProB::CfgGemvF16>;+ auto A_d = dA.data();+ auto B_d = dB.data();+ auto C_d = dC.data();+ auto S_d = dS.data();+ auto psize = (size_t)m * n * k * 2;+ tm.start();+ while (tm.stop() < TestMs) {+ tm1.start();+ for (size_t i = 0; i < batch; i++) {+ ProB::template gemv<Cfg, ST>(+ A_d + i * m * k,+ {B_d + i * n * k / 2, S_d + i * n * blks, blks, nullptr, asym ? dZP.data() + i * zp.size() : nullptr},+ C_d + i * m * n, n, k, blocksize, q);+ }+ q->wait();+ log.add(tm1.stop() / (float)batch);+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;++ double band = double(memsize) / log.min_val / 1e6;+ printf(" %s Flops:%.3f MemoryBandwidth:%.3fGB/s\n", log.get_log_str(), flops, band);+ q->memcpy(C.data(), C_d, C.size() * sizeof(T)).wait();+ auto Cf32 = C.template to<float>();+ buffer_error(ref.data(), Cf32.data(), ref.size(), float(0.1f));+ }++ template <typename T, bool asym = false>+ void dequant_next(int n, int k, int blocksize) {+ int blks = updiv(k, blocksize);+ auto psize = (size_t)n * k * sizeof(T) + n * k / 2 + n * k / blocksize * sizeof(T);+ if (asym) psize += n * blks;+ size_t batch = auto_batch(psize);+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case %s: %d %d %d Device:%s\n", __FUNCTION__, n, k, blocksize, dev->getName().c_str());+ avector<uint8_t> rawB(k * n / 2);+ using ST = std::conditional_t<std::is_same_v<T, utils::fp16>, sycl::half, float>;+ avector<T> scale(blks * n), dequant(n * k), ref(n * k);+ avector<int8_t> zp(n * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(scale.data(), scale.size(), T(0.01f), T(0.03f));+ fill_buffer_randn(rawB.data(), rawB.size(), uint8_t(0), uint8_t(255));+ auto srcptr = (utils::int4x2*)rawB.data();+ for (int j = 0; j < n; j += 1) {+ for (int i = 0; i < k; i += 2) {+ auto tmp = srcptr[i / 2 + j * k / 2];+ auto noffset = i / blocksize + j * blks;+ if constexpr (asym) {+ ref[i + j * k] = static_cast<float>(static_cast<int8_t>(tmp.x) - 8 - zp[noffset]) * (float)scale[noffset];+ ref[i + 1 + j * k] = static_cast<float>(static_cast<int8_t>(tmp.y) - 8 - zp[noffset]) * (float)scale[noffset];+ } else {+ ref[i + j * k] = static_cast<float>(static_cast<int8_t>(tmp.x) - 8) * (float)scale[noffset];+ ref[i + 1 + j * k] = static_cast<float>(static_cast<int8_t>(tmp.y) - 8) * (float)scale[noffset];+ }+ }+ }+ sycl_vector<ST> dS(scale.size() * batch, q), dequantB((size_t)n * k * batch, q);+ sycl_vector<uint8_t> dB(rawB.size() * batch, q);+ sycl_vector<int8_t> dZP(zp.size() * batch, q);+ for (size_t i = 0; i < batch; i++) {+ q->memcpy(dS.data() + i * scale.size(), scale.data(), scale.size() * sizeof(T)).wait();+ q->memcpy(dB.data() + i * rawB.size(), rawB.data(), rawB.size() * 1).wait();+ q->memcpy(dZP.data() + i * zp.size(), zp.data(), zp.size() * 1).wait();+ }++ auto S_d = dS.data();+ auto B_d = dB.data();+ auto DB_d = dequantB.data();+ using LOG = timer_statistics_logger<TestMs * 2>;+ LOG log;+ using ProB = sycl_prologue_b::WeightS4T<ST>;+ utils::timer<std::chrono::milliseconds> tm;+ utils::timer<std::chrono::microseconds> tm1;+ tm.start();+ while (tm.stop() < TestMs) {+ tm1.start();+ for (size_t i = 0; i < batch; i++) {+ if constexpr (sizeof(T) == 4) {+ auto ev = ProB::template dequant<typename ProB::CfgDequantF32>(+ n, k, blocksize,+ {B_d + i * rawB.size(), S_d + i * scale.size(), blks, nullptr,+ asym ? dZP.data() + i * zp.size() : nullptr},+ DB_d + i * n * k, q);+ } else {+ auto ev = ProB::template dequant<typename ProB::CfgDequantF16>(+ n, k, blocksize,+ {B_d + i * rawB.size(), S_d + i * scale.size(), blks, nullptr,+ asym ? dZP.data() + i * zp.size() : nullptr},+ DB_d + i * n * k, q);+ }+ }+ q->wait();++ log.add(tm1.stop() / (float)batch);+ if (tm.stop() >= TestMs) {+ break;+ }+ }++ q->memcpy(dequant.data(), DB_d, dequant.size() * sizeof(T)).wait();+ buffer_error(ref.data(), dequant.data(), dequant.size(), T(0.1f));+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ printf(" %s Memory Bandwidth:%.3f\n", log.get_log_str(), flops);+ }++ template <typename T, bool asym = false>+ void dequant_next_s8(int n, int k, int blocksize, bool rescale) {+ int blks = updiv(k, blocksize);+ auto psize = (size_t)n * k + n * k / 2;+ int newblocksize = 256;+ int newblks = k / newblocksize;+ if (rescale) psize += n * blks * sizeof(T) + n * newblks * sizeof(T);+ if (asym) psize += n * blks;+ size_t batch = auto_batch(psize);+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case %s: %d %d %d, batch %d Device:%s\n", __FUNCTION__, n, k, blocksize, batch,+ dev->getName().c_str());+ avector<uint8_t> rawB(k * n / 2);+ using ST = std::conditional_t<std::is_same_v<T, utils::fp16>, sycl::half, float>;+ avector<T> scale(blks * n), scaleN(n * newblks);+ avector<float> dqs4(n * k), dqs8(n * k);+ avector<int8_t> dequant(n * k), ref(n * k);+ avector<int8_t> zp(n * blks);+ fill_buffer_randn(scale.data(), scale.size(), T(0.01f), T(0.03f));+ fill_buffer_randn(rawB.data(), rawB.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-8), int8_t(7));+ auto srcptr = (utils::int4x2*)rawB.data();+ for (int j = 0; j < n; j += 1) {+ for (size_t i = 0; i < newblks; i++) {+ float maxv = 0.f;+ int start_blk = i * newblocksize / blocksize;+ int end_blk = (i + 1) * newblocksize / blocksize;+ for (int ii = start_blk; ii < end_blk; ii += 1) {+ maxv = std::max((float)scale[ii + j * blks], maxv);+ }+ scaleN[j * newblks + i] = asym ? maxv * 16.f / 127.f : maxv * 8.f / 127.f;+ }+ }+#pragma omp parallel for+ for (int j = 0; j < n; j += 1) {+ for (int i = 0; i < k; i += 2) {+ auto tmp = srcptr[i / 2 + j * k / 2];+ auto noffset = i / blocksize + j * blks;+ float sn = (float)scaleN[j * newblks + i / newblocksize];+ float _scale = (float)scale[noffset] / sn;+ if constexpr (asym) {+ dqs4[i + j * k] = (static_cast<int8_t>(tmp.x) - 8 - zp[noffset]) * (float)scale[noffset];+ dqs4[i + 1 + j * k] = (static_cast<int8_t>(tmp.y) - 8 - zp[noffset]) * (float)scale[noffset];+ } else {+ dqs4[i + j * k] = (static_cast<int8_t>(tmp.x) - 8) * (float)scale[noffset];+ dqs4[i + 1 + j * k] = (static_cast<int8_t>(tmp.y) - 8) * (float)scale[noffset];+ }+ if (rescale) {+ if constexpr (asym) {+ ref[i + j * k] = std::round(static_cast<float>(static_cast<int8_t>(tmp.x) - 8 - zp[noffset]) * _scale);+ ref[i + 1 + j * k] = std::round(static_cast<float>(static_cast<int8_t>(tmp.y) - 8 - zp[noffset]) * _scale);+ } else {+ ref[i + j * k] = std::round(static_cast<float>(static_cast<int8_t>(tmp.x) - 8) * _scale);+ ref[i + 1 + j * k] = std::round(static_cast<float>(static_cast<int8_t>(tmp.y) - 8) * _scale);+ }+ dqs8[i + j * k] = ref[i + j * k] * sn;+ dqs8[i + 1 + j * k] = ref[i + 1 + j * k] * sn;+ } else {+ ref[i + j * k] = static_cast<int8_t>(tmp.x) - 8;+ ref[i + 1 + j * k] = static_cast<int8_t>(tmp.y) - 8;+ if constexpr (asym) {+ ref[i + j * k] -= zp[noffset];+ ref[i + 1 + j * k] -= zp[noffset];+ }+ }+ }+ }+ if (rescale) buffer_error(dqs4.data(), dqs8.data(), dqs8.size(), float(1));+ sycl_vector<ST> dS(scale.size() * batch, q), dSInv(scaleN.size() * batch, q);+ sycl_vector<uint8_t> dB(rawB.size() * batch, q);+ sycl_vector<int8_t> dequantB((size_t)n * k * batch, q), dZP(zp.size() * batch, q);+ for (size_t i = 0; i < batch; i++) {+ q->memcpy(dS.data() + i * scale.size(), scale.data(), scale.size() * sizeof(T));+ q->memcpy(dSInv.data() + i * scaleN.size(), scaleN.data(), scaleN.size() * sizeof(T));+ q->memcpy(dB.data() + i * rawB.size(), rawB.data(), rawB.size() * 1);+ q->memcpy(dZP.data() + i * zp.size(), zp.data(), zp.size() * 1);+ }+ q->wait();+ auto S_d = dS.data();+ auto SInv_d = rescale ? dSInv.data() : nullptr;+ auto B_d = dB.data();+ auto DB_d = dequantB.data();+ using LOG = timer_statistics_logger<TestMs * 2>;+ LOG log;+ using ProB = sycl_prologue_b::WeightS4T<ST>;+ utils::timer<std::chrono::milliseconds> tm;+ utils::timer<std::chrono::microseconds> tm1;+ tm.start();+ while (tm.stop() < TestMs) {+ tm1.start();+ for (size_t i = 0; i < batch; i++) {+ if (rescale) {+ auto ev = ProB::template dequantS8<typename ProB::CfgDequantS8Rescale>(+ n, k, blocksize,+ {B_d + i * rawB.size(), S_d + i * scale.size(), blks, nullptr,+ asym ? dZP.data() + i * zp.size() : nullptr, SInv_d ? SInv_d + i * n * newblks : nullptr},+ DB_d + i * n * k, q, newblocksize);+ } else {+ auto ev = ProB::template dequantS8<typename ProB::CfgDequantS8>(+ n, k, blocksize,+ {B_d + i * rawB.size(), nullptr, blks, nullptr, asym ? dZP.data() + i * zp.size() : nullptr, nullptr},+ DB_d + i * n * k, q);+ }+ }+ q->wait();++ log.add(tm1.stop() / (float)batch);+ if (tm.stop() >= TestMs) {+ break;+ }+ }++ q->memcpy(dequant.data(), DB_d, dequant.size()).wait();+ buffer_error(ref.data(), dequant.data(), dequant.size(), int8_t(1)); // Why?+ log.record();++ double flops = double(psize) / log.min_val / 1e6;+ printf(" %s Memory Bandwidth:%.3f\n", log.get_log_str(), flops);+ }+};+#ifdef BTLA_UT_SYCL+static Benchmark_S4 sBenchmark_S4;+#endif++class Benchmark_S2 {+ public:+ Benchmark_S2() {+ UT_START();+ gemv_next<float>(1, 4096, 4096, 32);+ gemv_next<float, true>(1, 4096, 4096, 32);+ gemv_next<utils::fp16>(1, 4096, 4096, 32);+ gemv_next<utils::fp16, true>(1, 4096, 4096, 32);++ dequant_next_s8<float, true>(4096, 4096, 32, 0);+ dequant_next_s8<float>(4096, 4096, 32, 0);+ dequant_next_s8<utils::fp16, true>(4096, 4096, 32, 0);+ dequant_next_s8<utils::fp16>(4096, 4096, 32, 0);+ dequant_next_s8<float, true>(4096, 4096, 32, 1);+ dequant_next_s8<float>(4096, 4096, 32, 1);+ dequant_next_s8<utils::fp16, true>(4096, 4096, 32, 1);+ dequant_next_s8<utils::fp16>(4096, 4096, 32, 1);++ dequant_next<float>(4096, 4096, 32);+ dequant_next<utils::fp16>(4096, 4096, 32);+ dequant_next<float, true>(4096, 4096, 32);+ dequant_next<utils::fp16, true>(4096, 4096, 32);+ }+ static constexpr size_t nbits = 2;++ template <typename T, bool asym = false>+ void gemv_next(int m, int n, int k, int blocksize) {+ int blks = k / blocksize;+ auto memsize = (size_t)(n * k * nbits / 8 + n * blks * sizeof(T)) + (m * k + m * n) * sizeof(T);+ if (asym) memsize += n * blks;+ auto batch = auto_batch(memsize);+ using LOG = timer_statistics_logger<TestMs * 2>;+ LOG log;+ auto dev = UT_Device::get();+ printf("Test Case %s: %d %d %d Device:%s\n", __FUNCTION__, n, k, blocksize, dev->getName().c_str());+ auto q = dev->getQueue();+ using ST = std::conditional_t<std::is_same_v<T, utils::fp16>, sycl::half, float>;+ avector<uint8_t> rawB(k * n / 4);+ avector<T> scale(blks * n), A(m * k), C(n * m);+ avector<float> Bf32(n * k), Af32(m * k), ref(n * m);+ avector<int8_t> zp(n * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-2), int8_t(1));+ fill_buffer_randn(A.data(), A.size(), T(-.5f), T(0.5f));+ fill_buffer_randn(scale.data(), scale.size(), T(0.01f), T(0.03f));+ fill_buffer_randn(rawB.data(), rawB.size(), uint8_t(0), uint8_t(255));+ auto srcptr = (utils::bit2x4*)rawB.data();+ Af32 = A.template to<float>();+ for (int j = 0; j < n; j += 1) {+ for (int i = 0; i < k; i += 4) {+ auto tmp = srcptr[i / 4 + j * k / 4];+ auto noffset = i / blocksize + j * blks;+ int8_t q0 = static_cast<int8_t>(tmp.a) - 2;+ int8_t q1 = static_cast<int8_t>(tmp.b) - 2;+ int8_t q2 = static_cast<int8_t>(tmp.c) - 2;+ int8_t q3 = static_cast<int8_t>(tmp.d) - 2;+ if constexpr (asym) {+ Bf32[(i + 0) * n + j] = static_cast<float>(q0 - zp[noffset]) * (float)scale[noffset];+ Bf32[(i + 1) * n + j] = static_cast<float>(q1 - zp[noffset]) * (float)scale[noffset];+ Bf32[(i + 2) * n + j] = static_cast<float>(q2 - zp[noffset]) * (float)scale[noffset];+ Bf32[(i + 3) * n + j] = static_cast<float>(q3 - zp[noffset]) * (float)scale[noffset];+ } else {+ Bf32[(i + 0) * n + j] = static_cast<float>(q0) * (float)scale[noffset];+ Bf32[(i + 1) * n + j] = static_cast<float>(q1) * (float)scale[noffset];+ Bf32[(i + 2) * n + j] = static_cast<float>(q2) * (float)scale[noffset];+ Bf32[(i + 3) * n + j] = static_cast<float>(q3) * (float)scale[noffset];+ }+ }+ }+ gemmref_fp32fp32fp32(m, n, k, Af32.data(), Bf32.data(), ref.data(), k, n, n);+ sycl_vector<ST> dS(scale.size() * batch, q), dA(A.size() * batch, q), dC((size_t)n * k * batch, q);+ sycl_vector<uint8_t> dB(rawB.size() * batch, q);+ sycl_vector<int8_t> dZP(zp.size() * batch, q);+ for (size_t i = 0; i < batch; i++) {+ q->memcpy(dS.data() + i * scale.size(), scale.data(), scale.size() * sizeof(T));+ q->memcpy(dB.data() + i * rawB.size(), rawB.data(), rawB.size() * 1);+ q->memcpy(dA.data() + i * A.size(), A.data(), A.size() * sizeof(T));+ q->memcpy(dZP.data() + i * zp.size(), zp.data(), zp.size() * 1);+ }+ q->wait();+ utils::timer<std::chrono::milliseconds> tm;+ utils::timer<std::chrono::microseconds> tm1;+ using ProB = sycl_prologue_b::WeightS2T<ST>;+ using Cfg = std::conditional_t<std::is_same_v<T, float>, typename ProB::CfgGemvF32, typename ProB::CfgGemvF16>;+ auto A_d = dA.data();+ auto B_d = dB.data();+ auto C_d = dC.data();+ auto S_d = dS.data();+ auto psize = (size_t)m * n * k * 2;+ tm.start();+ while (tm.stop() < TestMs) {+ tm1.start();+ for (size_t i = 0; i < batch; i++) {+ ProB::template gemv<Cfg, ST>(+ A_d + i * m * k,+ {B_d + i * n * k / 4, S_d + i * n * blks, blks, nullptr, asym ? dZP.data() + i * zp.size() : nullptr},+ C_d + i * m * n, n, k, blocksize, q);+ }+ q->wait();+ log.add(tm1.stop() / (float)batch);+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;++ double band = double(memsize) / log.min_val / 1e6;+ printf(" %s Flops:%.3f MemoryBandwidth:%.3fGB/s\n", log.get_log_str(), flops, band);+ q->memcpy(C.data(), C_d, C.size() * sizeof(T)).wait();+ auto Cf32 = C.template to<float>();+ buffer_error(ref.data(), Cf32.data(), ref.size(), float(0.1f));+ }++ template <typename T, bool asym = false>+ void dequant_next(int n, int k, int blocksize) {+ int blks = updiv(k, blocksize);+ auto psize = (size_t)n * k * sizeof(T) + n * k / 4 + n * k / blocksize * sizeof(T);+ if (asym) psize += n * blks;+ size_t batch = auto_batch(psize);+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case %s: %d %d %d Device:%s\n", __FUNCTION__, n, k, blocksize, dev->getName().c_str());+ avector<uint8_t> rawB(k * n / 4);+ using ST = std::conditional_t<std::is_same_v<T, utils::fp16>, sycl::half, float>;+ avector<T> scale(blks * n), dequant(n * k), ref(n * k);+ avector<int8_t> zp(n * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-2), int8_t(1));+ fill_buffer_randn(scale.data(), scale.size(), T(0.01f), T(0.03f));+ fill_buffer_randn(rawB.data(), rawB.size(), uint8_t(0), uint8_t(255));+ auto srcptr = (utils::bit2x4*)rawB.data();+ for (int j = 0; j < n; j += 1) {+ for (int i = 0; i < k; i += 4) {+ auto tmp = srcptr[i / 4 + j * k / 4];+ auto noffset = i / blocksize + j * blks;+ int8_t q0 = static_cast<int8_t>(tmp.a) - 2;+ int8_t q1 = static_cast<int8_t>(tmp.b) - 2;+ int8_t q2 = static_cast<int8_t>(tmp.c) - 2;+ int8_t q3 = static_cast<int8_t>(tmp.d) - 2;+ if constexpr (asym) {+ ref[i + 0 + j * k] = static_cast<float>(q0 - zp[noffset]) * (float)scale[noffset];+ ref[i + 1 + j * k] = static_cast<float>(q1 - zp[noffset]) * (float)scale[noffset];+ ref[i + 2 + j * k] = static_cast<float>(q2 - zp[noffset]) * (float)scale[noffset];+ ref[i + 3 + j * k] = static_cast<float>(q3 - zp[noffset]) * (float)scale[noffset];+ } else {+ ref[i + 0 + j * k] = static_cast<float>(q0) * (float)scale[noffset];+ ref[i + 1 + j * k] = static_cast<float>(q1) * (float)scale[noffset];+ ref[i + 2 + j * k] = static_cast<float>(q2) * (float)scale[noffset];+ ref[i + 3 + j * k] = static_cast<float>(q3) * (float)scale[noffset];+ }+ }+ }+ sycl_vector<ST> dS(scale.size() * batch, q), dequantB((size_t)n * k * batch, q);+ sycl_vector<uint8_t> dB(rawB.size() * batch, q);+ sycl_vector<int8_t> dZP(zp.size() * batch, q);+ for (size_t i = 0; i < batch; i++) {+ q->memcpy(dS.data() + i * scale.size(), scale.data(), scale.size() * sizeof(T)).wait();+ q->memcpy(dB.data() + i * rawB.size(), rawB.data(), rawB.size() * 1).wait();+ q->memcpy(dZP.data() + i * zp.size(), zp.data(), zp.size() * 1).wait();+ }++ auto S_d = dS.data();+ auto B_d = dB.data();+ auto DB_d = dequantB.data();+ using LOG = timer_statistics_logger<TestMs * 2>;+ LOG log;+ using ProB = sycl_prologue_b::WeightS2T<ST>;+ utils::timer<std::chrono::milliseconds> tm;+ utils::timer<std::chrono::microseconds> tm1;+ tm.start();+ while (tm.stop() < TestMs) {+ tm1.start();+ for (size_t i = 0; i < batch; i++) {+ if constexpr (sizeof(T) == 4) {+ auto ev = ProB::template dequant<typename ProB::CfgDequantF32>(+ n, k, blocksize,+ {B_d + i * rawB.size(), S_d + i * scale.size(), blks, nullptr,+ asym ? dZP.data() + i * zp.size() : nullptr},+ DB_d + i * n * k, q);+ } else {+ auto ev = ProB::template dequant<typename ProB::CfgDequantF16>(+ n, k, blocksize,+ {B_d + i * rawB.size(), S_d + i * scale.size(), blks, nullptr,+ asym ? dZP.data() + i * zp.size() : nullptr},+ DB_d + i * n * k, q);+ }+ }+ q->wait();++ log.add(tm1.stop() / (float)batch);+ if (tm.stop() >= TestMs) {+ break;+ }+ }++ q->memcpy(dequant.data(), DB_d, dequant.size() * sizeof(T)).wait();+ buffer_error(ref.data(), dequant.data(), dequant.size(), T(0.1f));+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ printf(" %s Memory Bandwidth:%.3f\n", log.get_log_str(), flops);+ }++ template <typename T, bool asym = false>+ void dequant_next_s8(int n, int k, int blocksize, bool rescale) {+ int blks = updiv(k, blocksize);+ auto psize = (size_t)n * k + n * k / 4;+ int newblocksize = 256;+ int newblks = k / newblocksize;+ if (rescale) psize += n * blks * sizeof(T) + n * newblks * sizeof(T);+ if (asym) psize += n * blks;+ size_t batch = auto_batch(psize);+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case %s: %d %d %d, batch %d Device:%s\n", __FUNCTION__, n, k, blocksize, batch,+ dev->getName().c_str());+ avector<uint8_t> rawB(k * n / 4);+ using ST = std::conditional_t<std::is_same_v<T, utils::fp16>, sycl::half, float>;+ avector<T> scale(blks * n), scaleN(n * newblks);+ avector<float> dqs2(n * k), dqs8(n * k);+ avector<int8_t> dequant(n * k), ref(n * k);+ avector<int8_t> zp(n * blks);+ fill_buffer_randn(scale.data(), scale.size(), T(0.01f), T(0.03f));+ fill_buffer_randn(rawB.data(), rawB.size(), uint8_t(0), uint8_t(255));+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-2), int8_t(1));+ auto srcptr = (utils::bit2x4*)rawB.data();+ for (int j = 0; j < n; j += 1) {+ for (size_t i = 0; i < newblks; i++) {+ float maxv = 0.f;+ int start_blk = i * newblocksize / blocksize;+ int end_blk = (i + 1) * newblocksize / blocksize;+ for (int ii = start_blk; ii < end_blk; ii += 1) {+ maxv = std::max((float)scale[ii + j * blks], maxv);+ }+ scaleN[j * newblks + i] = asym ? maxv * 4.f / 127.f : maxv * 2.f / 127.f;+ }+ }+#pragma omp parallel for+ for (int j = 0; j < n; j += 1) {+ for (int i = 0; i < k; i += 4) {+ auto tmp = srcptr[i / 4 + j * k / 4];+ auto noffset = i / blocksize + j * blks;+ float sn = (float)scaleN[j * newblks + i / newblocksize];+ float _scale = (float)scale[noffset] / sn;+ int8_t q0 = static_cast<int8_t>(tmp.a) - 2;+ int8_t q1 = static_cast<int8_t>(tmp.b) - 2;+ int8_t q2 = static_cast<int8_t>(tmp.c) - 2;+ int8_t q3 = static_cast<int8_t>(tmp.d) - 2;+ if constexpr (asym) {+ dqs2[i + 0 + j * k] = (q0 - zp[noffset]) * (float)scale[noffset];+ dqs2[i + 1 + j * k] = (q1 - zp[noffset]) * (float)scale[noffset];+ dqs2[i + 2 + j * k] = (q2 - zp[noffset]) * (float)scale[noffset];+ dqs2[i + 3 + j * k] = (q3 - zp[noffset]) * (float)scale[noffset];+ } else {+ dqs2[i + 0 + j * k] = q0 * (float)scale[noffset];+ dqs2[i + 1 + j * k] = q1 * (float)scale[noffset];+ dqs2[i + 2 + j * k] = q2 * (float)scale[noffset];+ dqs2[i + 3 + j * k] = q3 * (float)scale[noffset];+ }+ if (rescale) {+ if constexpr (asym) {+ ref[i + 0 + j * k] = std::round(static_cast<float>(q0 - zp[noffset]) * _scale);+ ref[i + 1 + j * k] = std::round(static_cast<float>(q1 - zp[noffset]) * _scale);+ ref[i + 2 + j * k] = std::round(static_cast<float>(q2 - zp[noffset]) * _scale);+ ref[i + 3 + j * k] = std::round(static_cast<float>(q3 - zp[noffset]) * _scale);+ } else {+ ref[i + 0 + j * k] = std::round(static_cast<float>(q0) * _scale);+ ref[i + 1 + j * k] = std::round(static_cast<float>(q1) * _scale);+ ref[i + 2 + j * k] = std::round(static_cast<float>(q2) * _scale);+ ref[i + 3 + j * k] = std::round(static_cast<float>(q3) * _scale);+ }+ dqs8[i + 0 + j * k] = ref[i + 0 + j * k] * sn;+ dqs8[i + 1 + j * k] = ref[i + 1 + j * k] * sn;+ dqs8[i + 2 + j * k] = ref[i + 2 + j * k] * sn;+ dqs8[i + 3 + j * k] = ref[i + 3 + j * k] * sn;+ } else {+ ref[i + 0 + j * k] = q0;+ ref[i + 1 + j * k] = q1;+ ref[i + 2 + j * k] = q2;+ ref[i + 3 + j * k] = q3;+ if constexpr (asym) {+ ref[i + 0 + j * k] -= zp[noffset];+ ref[i + 1 + j * k] -= zp[noffset];+ ref[i + 2 + j * k] -= zp[noffset];+ ref[i + 3 + j * k] -= zp[noffset];+ }+ }+ }+ }+ if (rescale) buffer_error(dqs2.data(), dqs8.data(), dqs8.size(), float(1));+ sycl_vector<ST> dS(scale.size() * batch, q), dSInv(scaleN.size() * batch, q);+ sycl_vector<uint8_t> dB(rawB.size() * batch, q);+ sycl_vector<int8_t> dequantB((size_t)n * k * batch, q), dZP(zp.size() * batch, q);+ for (size_t i = 0; i < batch; i++) {+ q->memcpy(dS.data() + i * scale.size(), scale.data(), scale.size() * sizeof(T));+ q->memcpy(dSInv.data() + i * scaleN.size(), scaleN.data(), scaleN.size() * sizeof(T));+ q->memcpy(dB.data() + i * rawB.size(), rawB.data(), rawB.size() * 1);+ q->memcpy(dZP.data() + i * zp.size(), zp.data(), zp.size() * 1);+ }+ q->wait();+ auto S_d = dS.data();+ auto SInv_d = rescale ? dSInv.data() : nullptr;+ auto B_d = dB.data();+ auto DB_d = dequantB.data();+ using LOG = timer_statistics_logger<TestMs * 2>;+ LOG log;+ using ProB = sycl_prologue_b::WeightS2T<ST>;+ utils::timer<std::chrono::milliseconds> tm;+ utils::timer<std::chrono::microseconds> tm1;+ tm.start();+ while (tm.stop() < TestMs) {+ tm1.start();+ for (size_t i = 0; i < batch; i++) {+ if (rescale) {+ auto ev = ProB::template dequantS8<typename ProB::CfgDequantS8Rescale>(+ n, k, blocksize,+ {B_d + i * rawB.size(), S_d + i * scale.size(), blks, nullptr,+ asym ? dZP.data() + i * zp.size() : nullptr, SInv_d ? SInv_d + i * n * newblks : nullptr},+ DB_d + i * n * k, q, newblocksize);+ } else {+ auto ev = ProB::template dequantS8<typename ProB::CfgDequantS8>(+ n, k, blocksize,+ {B_d + i * rawB.size(), nullptr, blks, nullptr, asym ? dZP.data() + i * zp.size() : nullptr, nullptr},+ DB_d + i * n * k, q);+ }+ }+ q->wait();++ log.add(tm1.stop() / (float)batch);+ if (tm.stop() >= TestMs) {+ break;+ }+ }++ q->memcpy(dequant.data(), DB_d, dequant.size()).wait();+ buffer_error(ref.data(), dequant.data(), dequant.size(), int8_t(1));+ log.record();++ double flops = double(psize) / log.min_val / 1e6;+ printf(" %s Memory Bandwidth:%.3f\n", log.get_log_str(), flops);+ }+};+#ifdef BTLA_UT_SYCL+#endif+static Benchmark_S2 sBenchmark_S2;++class Benchmark_S8 {+ public:+ Benchmark_S8() {+ UT_START();+ gemv_next<float, true>(1, 4096, 4096, 32);+ gemv_next<float>(1, 4096, 4096, 32);+ // gemv_next<float>(1, 4096, 768, 32);+ gemv_next<utils::fp16, true>(1, 4096, 4096, 32);+ gemv_next<utils::fp16>(1, 4096, 4096, 32);+ // gemv_next<utils::fp16>(1, 4096, 768, 32);++ dequant_next<float, true>(4096, 4096, 32);+ dequant_next<float, false>(4096, 4096, 32);+ // dequant_next<float>(4096, 768, 32);+ dequant_next<utils::fp16, true>(4096, 4096, 32);+ dequant_next<utils::fp16, false>(4096, 4096, 32);+ // dequant_next<utils::fp16>(4096, 768, 32);+ }+ static constexpr size_t nbits = 8;++ template <typename T, bool asym = false>+ void gemv_next(int m, int n, int k, int blocksize) {+ int blks = k / blocksize;+ auto memsize = (size_t)(n * k * nbits / 8 + n * blks * sizeof(T)) + (m * k + m * n) * sizeof(T);+ auto batch = auto_batch(memsize);+ using LOG = timer_statistics_logger<TestMs * 2>;+ LOG log;+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ using ST = std::conditional_t<std::is_same_v<T, utils::fp16>, sycl::half, float>;+ avector<int8_t> rawB(k * n);+ avector<T> scale(blks * n), A(m * k), C(n * m);+ avector<float> Bf32(n * k), Af32(m * k), ref(n * m);+ avector<int8_t> zp(n * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(A.data(), A.size(), T(-.5f), T(0.5f));+ fill_buffer_randn(scale.data(), scale.size(), T(0.01f), T(0.03f));+ fill_buffer_randn(rawB.data(), rawB.size(), int8_t(-128), int8_t(127));+ auto srcptr = (int8_t*)rawB.data();+ Af32 = A.template to<float>();++ for (int j = 0; j < n; j += 1) {+ for (int i = 0; i < k; i += 2) {+ auto tmp = srcptr[i + j * k];+ auto tmp1 = srcptr[i + 1 + j * k];+ auto noffset = i / blocksize + j * blks;+ if constexpr (asym) {+ Bf32[i * n + j] = static_cast<float>((int)tmp - zp[noffset]) * (float)scale[noffset];+ Bf32[(i + 1) * n + j] = static_cast<float>((int)tmp1 - zp[noffset]) * (float)scale[noffset];+ } else {+ Bf32[i * n + j] = static_cast<float>(tmp) * (float)scale[noffset];+ Bf32[(i + 1) * n + j] = static_cast<float>(tmp1) * (float)scale[noffset];+ }+ }+ }+ gemmref_fp32fp32fp32(m, n, k, Af32.data(), Bf32.data(), ref.data(), k, n, n);+ sycl_vector<ST> dS(scale.size() * batch, q), dA(A.size() * batch, q), dC((size_t)n * k * batch, q);+ sycl_vector<int8_t> dB(rawB.size() * batch, q);+ sycl_vector<int8_t> dZP(zp.size() * batch, q);+ for (size_t i = 0; i < batch; i++) {+ q->memcpy(dS.data() + i * scale.size(), scale.data(), scale.size() * sizeof(T));+ q->memcpy(dZP.data() + i * zp.size(), zp.data(), zp.size() * 1);+ q->memcpy(dB.data() + i * rawB.size(), rawB.data(), rawB.size() * 1);+ q->memcpy(dA.data() + i * A.size(), A.data(), A.size() * sizeof(T));+ }+ q->wait();+ utils::timer<std::chrono::milliseconds> tm;+ utils::timer<std::chrono::microseconds> tm1;+ using ProB = sycl_prologue_b::WeightS8T<ST>;+ using Cfg = std::conditional_t<std::is_same_v<T, float>, typename ProB::CfgGemvF32, typename ProB::CfgGemvF16>;+ auto A_d = dA.data();+ auto B_d = dB.data();+ auto C_d = dC.data();+ auto S_d = dS.data();+ auto psize = (size_t)m * n * k * 2;+ tm.start();+ while (tm.stop() < TestMs) {+ tm1.start();+ for (size_t i = 0; i < batch; i++) {+ auto ev = ProB::template gemv<Cfg, ST>(+ A_d + i * m * k,+ {B_d + i * n * k, S_d + i * n * blks, blks, nullptr, asym ? dZP.data() + i * zp.size() : nullptr},+ C_d + i * m * n, n, k, blocksize, q);+ // ev.wait();+ // log.add(event_helper::execute_time(ev) * 1000);+ }+ q->wait();+ log.add(tm1.stop() / (float)batch);+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;++ double band = double(memsize) / log.min_val / 1e6;+ printf(" %s Flops:%.3f MemoryBandwidth:%.3fGB/s\n", log.get_log_str(), flops, band);+ q->memcpy(C.data(), C_d, C.size() * sizeof(T)).wait();+ auto Cf32 = C.template to<float>();+ buffer_error(ref.data(), Cf32.data(), ref.size(), float(0.3f));+ }++ template <typename T, bool asym = false>+ void dequant_next(int n, int k, int blocksize) {+ int blks = updiv(k, blocksize);+ auto psize = (size_t)n * k * sizeof(T) + n * k * nbits / 8 + n * k / blocksize * sizeof(T);+ size_t batch = auto_batch(psize);+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case %s: %d %d %d Device:%s\n", __FUNCTION__, n, k, blocksize, dev->getName().c_str());+ avector<int8_t> rawB(k * n);+ using ST = std::conditional_t<std::is_same_v<T, utils::fp16>, sycl::half, float>;+ avector<T> scale(blks * n), dequant(n * k), ref(n * k);+ avector<int8_t> zp(n * blks);+ fill_buffer_randn(zp.data(), zp.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(scale.data(), scale.size(), T(0.01f), T(0.03f));+ fill_buffer_randn(rawB.data(), rawB.size(), int8_t(-128), int8_t(127));+ auto srcptr = (int8_t*)rawB.data();+ for (int j = 0; j < n; j += 1) {+ for (int i = 0; i < k; i += 2) {+ auto tmp = srcptr[i + j * k];+ auto tmp1 = srcptr[i + 1 + j * k];+ auto noffset = i / blocksize + j * blks;+ if constexpr (asym) {+ ref[i + j * k] = static_cast<float>((int)tmp - zp[noffset]) * (float)scale[noffset];+ ref[i + 1 + j * k] = static_cast<float>((int)tmp1 - zp[noffset]) * (float)scale[noffset];+ } else {+ ref[i + j * k] = static_cast<float>(tmp) * (float)scale[noffset];+ ref[i + 1 + j * k] = static_cast<float>(tmp1) * (float)scale[noffset];+ }+ }+ }++ sycl_vector<ST> dS(scale.size() * batch, q), dequantB((size_t)n * k * batch, q);+ sycl_vector<int8_t> dB(rawB.size() * batch, q);+ sycl_vector<int8_t> dZP(zp.size() * batch, q);+ for (size_t i = 0; i < batch; i++) {+ q->memcpy(dZP.data() + i * zp.size(), zp.data(), zp.size() * 1).wait();+ q->memcpy(dS.data() + i * scale.size(), scale.data(), scale.size() * sizeof(T)).wait();+ q->memcpy(dB.data() + i * rawB.size(), rawB.data(), rawB.size() * 1).wait();+ }++ auto S_d = dS.data();+ auto B_d = dB.data();+ auto DB_d = dequantB.data();+ using LOG = timer_statistics_logger<TestMs * 2>;+ LOG log;+ using ProB = sycl_prologue_b::WeightS8T<ST>;+ utils::timer<std::chrono::milliseconds> tm;+ utils::timer<std::chrono::microseconds> tm1;+ tm.start();+ while (tm.stop() < TestMs) {+ tm1.start();+ for (size_t i = 0; i < batch; i++) {+ if constexpr (sizeof(T) == 4) {+ auto ev = ProB::template dequant<typename ProB::CfgDequantF32>(+ n, k, blocksize,+ {B_d + i * rawB.size(), S_d + i * scale.size(), blks, nullptr,+ asym ? dZP.data() + i * zp.size() : nullptr},+ DB_d + i * n * k, q);+ } else {+ auto ev = ProB::template dequant<typename ProB::CfgDequantF16>(+ n, k, blocksize,+ {B_d + i * rawB.size(), S_d + i * scale.size(), blks, nullptr,+ asym ? dZP.data() + i * zp.size() : nullptr},+ DB_d + i * n * k, q);+ }+ }+ q->wait();++ log.add(tm1.stop() / (float)batch);+ if (tm.stop() >= TestMs) {+ break;+ }+ }++ q->memcpy(dequant.data(), DB_d, dequant.size() * sizeof(T)).wait();+ buffer_error(ref.data(), dequant.data(), dequant.size(), T(0.1f));+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ printf(" %s Memory Bandwidth:%.3f\n", log.get_log_str(), flops);+ }+};+#ifdef BTLA_UT_SYCL+static Benchmark_S8 sBenchmark_S8;+#endif++class Benchmark_F8 {+ public:+ Benchmark_F8() {+ UT_START();+ gemv_next<float>(1, 4096, 4096, 32);+ gemv_next<float, false>(1, 4096, 4096, 32);+ gemv_next<utils::fp16>(1, 4096, 4096, 32);+ // gemv_next<utils::fp16, false>(1, 4096, 4096, 32);+ // gemv_next<float>(1, 4096, 768, 32);+ // gemv_next<utils::fp16>(1, 16384, 4096, 32);+ // gemv_next<utils::fp16>(1, 4096, 768, 32);++ // dequant_next<float>(256, 768, 128, true);+ // dequant_next<float>(256, 768, 32, true);+ // dequant_next<float>(256, 768, 16, true);+ // dequant_next<utils::fp16>(256, 768, 128, true);+ // dequant_next<utils::fp16>(256, 768, 32, true);+ // dequant_next<utils::fp16>(256, 768, 16, true);+ dequant_next<float>(4096, 4096, 32);+ dequant_next<float, false>(4096, 4096, 32);+ dequant_next<utils::fp16>(4096, 4096, 32);+ dequant_next<utils::fp16, false>(4096, 4096, 32);+ }+ static constexpr size_t nbits = 8;++ template <typename T, bool IsE4M3 = true>+ void gemv_next(int m, int n, int k, int blocksize) {+ int blks = k / blocksize;+ auto memsize = (size_t)(n * k * nbits / 8 + n * blks * sizeof(T)) + (m * k + m * n) * sizeof(T);+ auto batch = auto_batch(memsize);+ auto dev = UT_Device::get();+ printf("Test Case %s: %d %d %d %d, %s batch %d Device:%s\n", __FUNCTION__, m, n, k, blocksize,+ IsE4M3 ? "E4M3" : "E5M2", batch, dev->getName().c_str());+ using LOG = timer_statistics_logger<TestMs * 2>;+ LOG log;+ auto q = dev->getQueue();+ using ST = std::conditional_t<std::is_same_v<T, utils::fp16>, sycl::half, float>;+ avector<uint8_t> rawB(k * n);+ avector<T> scale(blks * n), A(m * k), C(n * m);+ avector<float> Bf32(n * k), Af32(m * k), ref(n * m);+ fill_buffer_randn(A.data(), A.size(), T(-.5f), T(0.5f));+ fill_buffer_randn(scale.data(), scale.size(), T(0.01f), T(0.03f));+ fill_buffer_randn(rawB.data(), rawB.size(), uint8_t(0), uint8_t(255));++ sanitize_fp8_buffer<IsE4M3>(rawB.data(), rawB.size());+#pragma omp parallel for+ for (int i = 0; i < n; i++) {+ for (int j = 0; j < k; j++) {+ auto noffset = i * blks + j / blocksize;+ Bf32[i + j * n] = decode_fp8_byte<IsE4M3>(rawB[i * k + j]) * (float)scale[noffset];+ }+ }+ Af32 = A.template to<float>();++ gemmref_fp32fp32fp32(m, n, k, Af32.data(), Bf32.data(), ref.data(), k, n, n);+ sycl_vector<ST> dS(scale.size() * batch, q), dA(A.size() * batch, q), dC((size_t)n * k * batch, q);+ sycl_vector<uint8_t> dB(rawB.size() * batch, q);+ for (size_t i = 0; i < batch; i++) {+ q->memcpy(dS.data() + i * scale.size(), scale.data(), scale.size() * sizeof(T));+ q->memcpy(dB.data() + i * rawB.size(), rawB.data(), rawB.size() * 1);+ q->memcpy(dA.data() + i * A.size(), A.data(), A.size() * sizeof(T));+ }+ q->wait();+ utils::timer<std::chrono::milliseconds> tm;+ utils::timer<std::chrono::microseconds> tm1;+ using ProB = sycl_prologue_b::WeightF8T<ST, IsE4M3>;+ auto A_d = dA.data();+ auto B_d = dB.data();+ auto C_d = dC.data();+ auto S_d = dS.data();+ auto psize = (size_t)m * n * k * 2;+ tm.start();+ while (tm.stop() < TestMs) {+ tm1.start();+ for (size_t i = 0; i < batch; i++) {+ auto ev = ProB::template gemv<ST>(A_d + i * m * k, {B_d + i * n * k, S_d + i * n * blks, blks}, C_d + i * m * n,+ n, k, blocksize, q);+ }+ q->wait();+ log.add(tm1.stop() / (float)batch);+ }+ log.record();+ double flops = double(psize) / log.min_val / 1e6;++ double band = double(memsize) / log.min_val / 1e6;+ printf(" %s Flops:%.3f MemoryBandwidth:%.3fGB/s\n", log.get_log_str(), flops, band);+ q->memcpy(C.data(), C_d, C.size() * sizeof(T)).wait();+ auto Cf32 = C.template to<float>();+ buffer_error(ref.data(), Cf32.data(), ref.size(), 0.3f, 0.1f);+ }++ template <typename T, bool IsE4M3 = true>+ void dequant_next(int n, int k, int blocksize) {+ int blks = updiv(k, blocksize);+ auto psize = (size_t)n * k * sizeof(T) + n * k * nbits / 8 + n * k / blocksize * sizeof(T);+ size_t batch = auto_batch(psize);+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case %s: %d %d %d, %s batch %d Device:%s\n", __FUNCTION__, n, k, blocksize, IsE4M3 ? "E4M3" : "E5M2",+ batch, dev->getName().c_str());+ avector<uint8_t> rawB(k * n);+ using ST = std::conditional_t<std::is_same_v<T, utils::fp16>, sycl::half, float>;+ avector<T> scale(blks * n), dequant(n * k), ref(n * k);+ fill_buffer_randn(scale.data(), scale.size(), T(0.01f), T(0.03f));+ fill_buffer_randn(rawB.data(), rawB.size(), uint8_t(0), uint8_t(255));+ sanitize_fp8_buffer<IsE4M3>(rawB.data(), rawB.size());+ for (int i = 0; i < n; i++) {+ for (int j = 0; j < k; j++) {+ auto noffset = i * blks + j / blocksize;+ ref[i * k + j] = decode_fp8_byte<IsE4M3>(rawB[i * k + j]) * (float)scale[noffset];+ }+ }++ sycl_vector<ST> dS(scale.size() * batch, q), dequantB((size_t)n * k * batch, q);+ sycl_vector<uint8_t> dB(rawB.size() * batch, q);+ for (size_t i = 0; i < batch; i++) {+ q->memcpy(dS.data() + i * scale.size(), scale.data(), scale.size() * sizeof(T));+ q->memcpy(dB.data() + i * rawB.size(), rawB.data(), rawB.size() * 1);+ }+ q->wait();++ auto S_d = dS.data();+ auto B_d = dB.data();+ auto DB_d = dequantB.data();+ using LOG = timer_statistics_logger<TestMs * 2>;+ LOG log;+ using ProB = sycl_prologue_b::WeightF8T<ST, IsE4M3>;+ utils::timer<std::chrono::milliseconds> tm;+ utils::timer<std::chrono::microseconds> tm1;+ tm.start();+ while (tm.stop() < TestMs) {+ tm1.start();+ for (size_t i = 0; i < batch; i++) {+ if constexpr (sizeof(T) == 4) {+ auto ev = ProB::template dequant<typename ProB::Cfg>(+ n, k, blocksize, {B_d + i * rawB.size(), S_d + i * scale.size(), blks}, DB_d + i * n * k, q);+ } else {+ auto ev = ProB::template dequant<typename ProB::Cfg>(+ n, k, blocksize, {B_d + i * rawB.size(), S_d + i * scale.size(), blks}, DB_d + i * n * k, q);+ }+ }+ q->wait();+ log.add(tm1.stop() / (float)batch);+ if (tm.stop() >= TestMs) {+ break;+ }+ }+ q->memcpy(dequant.data(), DB_d, dequant.size() * sizeof(T)).wait();+ buffer_error(ref.data(), dequant.data(), dequant.size(), T(0.3f), T(0.1f));+ log.record();+ double flops = double(psize) / log.min_val / 1e6;+ printf(" %s Memory Bandwidth:%.3f\n", log.get_log_str(), flops);+ }+};+#ifdef BTLA_UT_SYCL+static Benchmark_F8 sBenchmark_F8;+#endif++class Benchmark_SDPAFp16Fp16 {+ public:+ Benchmark_SDPAFp16Fp16() {+ UT_START();+ // benchmark_all(512, 1024, 20, 20, 128, 128, true);+ benchmark_all(512, 512, 16, 16, 128, 128, true);+ benchmark_all(512, 512, 16 * 32, 16 * 32, 128, 128, false);+ // benchmark_all(1, 1024, 768, true);+ // benchmark_all(1, 4096, 4096);+ // benchmark_all(4096, 4096, 4096);+ }++ using AType = sycl::half;+ using BType = sycl::half;+ using CType = sycl::half;++ void benchmark(int head_num, int head_kv_num, int seq_cur, int seq_kv, int head_idim, int head_odim, float scale,+ AType* q_p, BType* k_p, CType* v_p, CType* out, CType* mask, bool verify) {+ using LOG = timer_statistics_logger<TestMs * 2>;+ LOG log;+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ float timems = TestMs;+ utils::timer<std::chrono::milliseconds> tm;+ // auto A_d = A;+ // auto B_d = B;+ // auto C_d = C;+ // auto psize = (size_t)m * n * k * 2;+ // using CFG = xmx::HGemmCfg;+ q->wait();+ size_t constexpr sg_size = 16;+ size_t constexpr TM = 16, TN = 16, TK = 16;+ size_t constexpr SGM = 1, SGN = 2, SGN2 = 8;+ size_t constexpr TileSKV = TN * SGN;+ size_t constexpr TileSQ = TM * SGM;+ size_t constexpr TileHO = TN * SGN2;+ size_t constexpr SG_SQ = 1;+ int constexpr UnrollK = 2;+ int sg_col = head_odim / TileHO;+ int sg_count = sg_col * SG_SQ;+ size_t wg_size = sg_size * sg_col * SG_SQ;+ int constexpr WG_SQ = TileSQ * SG_SQ;+ int sq_wg_count = seq_cur / WG_SQ;+ size_t wg_repeat = head_kv_num * sq_wg_count;+ static int constexpr PrefetchDis = 3;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ CType hscale = scale;+ auto ker = [&](sycl::handler& cgh) {+ // sycl::stream outs(65536, 256, cgh);+ cgh.parallel_for(sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}), [=](sycl::nd_item<2>+ it) [[sycl::reqd_sub_group_size(+ sg_size)]] {+ auto g = it.get_group();+ auto sg = it.get_sub_group();+ int g_id = it.get_group(0);+ int g_sq = g_id % sq_wg_count;+ int g_hkv = g_id / sq_wg_count;+ // int g_hkv = g_id % head_kv_num;+ // int g_sq = g_id / head_kv_num;+ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;+ int sg_sq = g_sq * WG_SQ + sggid_row * TileSQ;+ if (sg_sq >= seq_cur || g_hkv >= head_kv_num) return;+ int sgId = sg.get_local_id()[0];+ auto g_q = q_p + g_hkv * head_idim;+ auto g_k = k_p + g_hkv * head_idim;+ auto g_v = v_p + g_hkv * head_odim + sggid_col * TileHO;+ auto g_o = out + g_hkv * head_odim + sggid_col * TileHO;+ auto pQ = syclex::annotated_ptr{+ g_q, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};+ auto pK = syclex::annotated_ptr{+ g_k, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};+ auto pV = syclex::annotated_ptr{+ g_v, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};+ auto pO = syclex::annotated_ptr{+ g_o, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_through, syclex::cache_level::L1, syclex::cache_level::L3>>}};++ using SA = sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, AType, use::a, TM, TK,+ layout::row_major>;+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, AType, use::a, TM, TK, layout::row_major>+ sub_Q[SGM], subSA[SGM * SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, BType, use::b, TK, TN, layout::col_major>+ sub_K[SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, CType, use::accumulator, TM, TN>+ sub_S[SGM * SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, BType, use::b, TK, TN, layout::row_major>+ sub_V[SGN2];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, CType, use::accumulator, TM, TN>+ sub_O[SGM * SGN2], gsub_O[SGM * SGN2];+ CType g_m[SGM];+ CType g_l[SGM];+#pragma unroll+ for (int i = 0; i < SGM; i++) {+ g_m[i] = CType(0);+ g_l[i] = CType(0);+ }+ int ldq = head_num * head_idim;+ int ldk = head_kv_num * head_idim;+ int ldv = head_kv_num * head_odim;+ int ldo = head_num * head_odim;+#pragma unroll+ for (int i = 0; i < SGM * SGN2; i++) joint_matrix_fill(sg, gsub_O[i], 0);+ sycl::group_barrier(sg);+ // iskv: N loop for 1st mm, K loop for 2nd mm+ for (int iskv = 0; iskv < seq_kv; iskv += TileSKV) {+#pragma unroll+ for (int i = 0; i < SGM * SGN; i++) joint_matrix_fill(sg, sub_S[i], 0);+ for (int ik = 0; ik < head_idim; ik += TK * UnrollK) {+#pragma unroll+ for (int ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_K[in], pK, ldk, seq_kv, head_idim, iskv + in * TN, ik + ikk * TK);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_Q[im], pQ, ldq, seq_cur, head_idim, sg_sq + im * TM, ik + ikk * TK);+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_mad(sg, sub_S[im * SGN + in], sub_Q[im], sub_K[in], sub_S[im * SGN + in]);+ }+ }+ }+ // if (iskv + TileSKV <= seq_kv) {+ // for (int in = sgGroupId; in < SGN; in += sg_count)+ // joint_matrix_prefetch<TN, TK * UnrollK>(sg, g_k + (in * TN) * ldk + ik + TK * UnrollK * PrefetchDis,+ // ldk, layout::row_major,+ // syclex::properties{syclex::prefetch_hint_L1});+ // for (int in = sgGroupId; in < UnrollK; in += sg_count)+ // joint_matrix_prefetch<TM, TK>(sg, g_q + sg_sq * ldq + ik + TK * UnrollK * PrefetchDis + in * TK, ldq,+ // layout::row_major, syclex::properties{syclex::prefetch_hint_L1});+ // }+ }+ sycl::group_barrier(sg);++ CType maxS[SGM];+#pragma unroll+ for (int im = 0; im < SGM; im++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ auto wi_data_c = sycl::ext::oneapi::detail::get_wi_data(sg, sub_S[im * SGN + in]);+#pragma unroll+ for (int imm = 0; imm < TM; imm++) {+ auto element = wi_data_c[imm];+ CType tmp = element * hscale + mask[(sg_sq + im * TM + imm) * seq_kv + iskv + in * TN + sgId];+ CType _max = sycl::reduce_over_group(sg, tmp, sycl::maximum<CType>());+ if (imm == sgId) maxS[im] = std::max(maxS[im], _max);+ wi_data_c[imm] = tmp;+ }+ }+ }+ sycl::group_barrier(sg);++ CType sumExp[SGM];+#pragma unroll+ for (int i = 0; i < SGM; i++) {+ sumExp[i] = CType(0);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ auto wi_data_c = sycl::ext::oneapi::detail::get_wi_data(sg, sub_S[im * SGN + in]);+#pragma unroll+ for (int imm = 0; imm < TM; imm++) {+ auto element = wi_data_c[imm];+ CType tmp = sycl::exp(element - sycl::select_from_group(sg, maxS[im], imm));+ CType sumtmp = sycl::reduce_over_group(sg, tmp, sycl::plus<CType>());+ if (imm == sgId) sumExp[im] += sumtmp;+ wi_data_c[imm] = tmp;+ }+ }+ }+ CType blk_scale0[SGM], blk_scale1[SGM];+ CType maxS_[SGM], sumExp_[SGM];++#pragma unroll+ for (int i = 0; i < SGM; i++) {+ maxS_[i] = std::max(g_m[i], maxS[i]);+ blk_scale0[i] = sycl::exp(g_m[i] - maxS_[i]) * g_l[i];+ blk_scale1[i] = sycl::exp(maxS[i] - maxS_[i]);+ sumExp_[i] = blk_scale0[i] + blk_scale1[i] * sumExp[i];+ blk_scale0[i] = blk_scale0[i] / sumExp_[i];+ blk_scale1[i] = blk_scale1[i] / sumExp_[i];+ g_m[i] = maxS_[i];+ g_l[i] = sumExp_[i];+ }+ sycl::group_barrier(sg);++#pragma unroll+ for (int i = 0; i < SGM * SGN2; i++) joint_matrix_fill(sg, sub_O[i], 0);+#pragma unroll+ for (int ik2 = 0; ik2 < SGN; ik2++) {+#pragma unroll+ for (int in = 0; in < SGN2; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_V[in], pV, head_kv_num * head_odim, seq_kv, head_odim, iskv + ik2 * TK, in * TN);+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ joint_matrix_mad(sg, sub_O[im * SGN2 + in], *(SA*)&sub_S[im * SGN + ik2], sub_V[in],+ sub_O[im * SGN2 + in]);+ }+ }+ }+ sycl::group_barrier(sg);++ for (int im = 0; im < SGM; im++) {+ for (int in = 0; in < SGN2; in++) {+ auto wi_data_c0 = sycl::ext::oneapi::detail::get_wi_data(sg, gsub_O[im * SGN2 + in]);+ auto wi_data_c1 = sycl::ext::oneapi::detail::get_wi_data(sg, sub_O[im * SGN2 + in]);+ for (int imm = 0; imm < TM; imm++) {+ auto element0 = wi_data_c0[imm];+ auto element1 = wi_data_c1[imm];+ wi_data_c0[imm] = element0 * sycl::select_from_group(sg, blk_scale0[im], imm) ++ element1 * sycl::select_from_group(sg, blk_scale1[im], imm);+ }+ }+ }+ sycl::group_barrier(sg);+ // if (iskv + TileSKV * (PrefetchDis + 1) <= seq_kv) {+ // for (int im = sgGroupId; im < SGN; im += sg_count)+ // joint_matrix_prefetch<TM, TK>(sg, pK + (im * TM) * lda + ik + TK * UnrollK * PrefetchDis,+ // lda, layout::row_major,+ // syclex::properties{syclex::prefetch_hint_L1});+ // for (int in = sggid_row; in < SGN; in += sg_row)+ // if ((g_idn + in * TN) <= n)+ // joint_matrix_prefetch<TN, TK * UnrollK>(sg, Bwg_d + (in * TN) * ldb + ik + TK * UnrollK *+ // PrefetchDis,+ // ldb, layout::row_major,+ // syclex::properties{syclex::prefetch_hint_L1});+ // }+ }+ sycl::group_barrier(sg);++#pragma unroll+ for (int im = 0; im < SGM; im++) {+#pragma unroll+ for (int in = 0; in < SGN2; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_store_checked(+ sg, gsub_O[im * SGN2 + in], pO, head_num * head_odim, layout::row_major, seq_cur, head_odim,+ sg_sq + im * TM, in * TN);+ }+ }+ }); // parallel for+ };+ auto ker2 = [&](sycl::handler& cgh) {+ // sycl::stream outs(65536, 256, cgh);+ cgh.parallel_for(sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}), [=](sycl::nd_item<2>+ it) [[sycl::reqd_sub_group_size(+ sg_size)]] {+ auto g = it.get_group();+ auto sg = it.get_sub_group();+ int g_id = it.get_group(0);+ // int g_sq = g_id % sq_wg_count;+ // int g_hkv = g_id / sq_wg_count;+ int g_hkv = g_id % head_kv_num;+ int g_sq = g_id / head_kv_num;+ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;+ int sg_sq = g_sq * WG_SQ + sggid_row * TileSQ;+ if (sg_sq >= seq_cur || g_hkv >= head_kv_num) return;+ int sgId = sg.get_local_id()[0];+ auto g_q = q_p + g_hkv * head_idim;+ auto g_k = k_p + g_hkv * head_idim;+ auto g_v = v_p + g_hkv * head_odim + sggid_col * TileHO;+ auto g_o = out + g_hkv * head_odim + sggid_col * TileHO;+ auto pQ = syclex::annotated_ptr{+ g_q, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};+ auto pK = syclex::annotated_ptr{+ g_k, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};+ auto pV = syclex::annotated_ptr{+ g_v, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};+ auto pO = syclex::annotated_ptr{+ g_o, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_through, syclex::cache_level::L1, syclex::cache_level::L3>>}};++ using SA = sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, AType, use::a, TM, TK,+ layout::row_major>;+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, AType, use::a, TM, TK, layout::row_major>+ sub_Q[SGM];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, BType, use::b, TK, TN, layout::col_major>+ sub_K[SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, CType, use::accumulator, TM, TN>+ sub_S[SGM * SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, BType, use::b, TK, TN, layout::row_major>+ sub_V[SGN2];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, CType, use::accumulator, TM, TN>+ sub_O[SGM * SGN2], gsub_O[SGM * SGN2];++ int ldq = head_num * head_idim;+ int ldk = head_kv_num * head_idim;+ int ldv = head_kv_num * head_odim;+ int ldo = head_num * head_odim;+#pragma unroll+ for (int i = 0; i < SGM * SGN2; i++) joint_matrix_fill(sg, gsub_O[i], 0);++ sycl::group_barrier(sg);+ // iskv: N loop for 1st mm, K loop for 2nd mm+ for (int iskv = 0; iskv < seq_kv; iskv += TileSKV) {+ // if (iskv + TileSKV * (PrefetchDis + 1) <= seq_kv) {+ // for (int in = sgGroupId; in < SGN * 4; in += sg_count) {+ // int _in = in % SGN;+ // int _ih = in / SGN;+ // joint_matrix_prefetch<TN, TK * UnrollK>(+ // sg, g_k + (iskv + TileSKV * PrefetchDis + _in * TN) * ldk + _ih * TK * UnrollK, ldk,+ // layout::row_major, syclex::properties{syclex::prefetch_hint_L1});+ // }+ // }+ // for (int in = sgGroupId; in < SGN2; in += sg_count) {+ // joint_matrix_prefetch<TN, TK * SGN>(sg, g_v + (iskv + TileSKV * 1) * ldq + in * TK * SGN, ldq,+ // layout::row_major, syclex::properties{syclex::prefetch_hint_L1});+ // }+#pragma unroll+ for (int i = 0; i < SGM * SGN; i++) joint_matrix_fill(sg, sub_S[i], 0);+ sycl::group_barrier(sg);+ for (int ik = 0; ik < head_idim; ik += TK * UnrollK) {+ // if (ik + TK * UnrollK * (PrefetchDis + 1) <= head_idim) {+ // for (int in = sgGroupId; in < SGN; in += sg_count) {+ // joint_matrix_prefetch<TN, TK * UnrollK>(+ // sg, g_k + (iskv + in * TN) * ldk + ik + TK * UnrollK * PrefetchDis, ldk, layout::row_major,+ // syclex::properties{syclex::prefetch_hint_L1});+ // }+ // for (int in = sgGroupId; in < SGM; in += sg_count) {+ // joint_matrix_prefetch<TM, TK * UnrollK>(+ // sg, g_q + (sg_sq + in * TM) * ldq + ik + TK * UnrollK * PrefetchDis, ldq, layout::row_major,+ // syclex::properties{syclex::prefetch_hint_L1});+ // }+ // } else if (iskv + TileSKV < seq_kv) {+ // for (int in = sgGroupId; in < SGN; in += sg_count) {+ // joint_matrix_prefetch<TN, TK * UnrollK>(+ // sg, g_k + (iskv + TileSKV + in * TN) * ldk + ik + TK * UnrollK * PrefetchDis - head_idim, ldk,+ // layout::row_major, syclex::properties{syclex::prefetch_hint_L1});+ // }+ // }++#pragma unroll+ for (int ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_K[in], pK, ldk, seq_kv, head_idim, iskv + in * TN, ik + ikk * TK);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_Q[im], pQ, ldq, seq_cur, head_idim, sg_sq + im * TM, ik + ikk * TK);+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_mad(sg, sub_S[im * SGN + in], sub_Q[im], sub_K[in], sub_S[im * SGN + in]);+ }+ }+ }+ }+ sycl::group_barrier(sg);++#pragma unroll+ for (int i = 0; i < SGM * SGN2; i++) joint_matrix_fill(sg, sub_O[i], 0);+#pragma unroll+ for (int ik2 = 0; ik2 < SGN; ik2++) {+#pragma unroll+ for (int in = 0; in < SGN2; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_V[in], pV, head_kv_num * head_odim, seq_kv, head_odim, iskv + ik2 * TK, in * TN);+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ joint_matrix_mad(sg, sub_O[im * SGN2 + in], *(SA*)&sub_S[im * SGN + ik2], sub_V[in],+ sub_O[im * SGN2 + in]);+ }+ }+ }+ sycl::group_barrier(sg);++ for (int im = 0; im < SGM; im++) {+ for (int in = 0; in < SGN2; in++) {+ auto wi_data_c0 = sycl::ext::oneapi::detail::get_wi_data(sg, gsub_O[im * SGN2 + in]);+ auto wi_data_c1 = sycl::ext::oneapi::detail::get_wi_data(sg, sub_O[im * SGN2 + in]);+ for (int imm = 0; imm < TM; imm++) {+ auto element0 = wi_data_c0[imm];+ auto element1 = wi_data_c1[imm];+ wi_data_c0[imm] = element0 + element1;+ }+ }+ }+ sycl::group_barrier(sg);+ }++#pragma unroll+ for (int im = 0; im < SGM; im++) {+#pragma unroll+ for (int in = 0; in < SGN2; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_store_checked(+ sg, gsub_O[im * SGN2 + in], pO, head_num * head_odim, layout::row_major, seq_cur, head_odim,+ sg_sq + im * TM, in * TN);+ }+ }+ }); // parallel for+ };++ q->wait();+ for (size_t i = 0; i < 1; i++) {+ q->submit(ker2);+ }+ q->wait();+ if (verify) {+ return;+ }+ tm.start();+ utils::timer<std::chrono::microseconds> tm1;+ while (tm.stop() < timems) {+ tm1.start();+ for (size_t i = 0; i < 10; i++) {+ q->submit(ker2);+ }+ q->wait();+ log.add(tm1.stop() / 10);+ if (tm.stop() >= timems) {+ break;+ }+ }+ log.record();+ size_t group = 1;+ size_t OPS = group * head_kv_num * seq_cur * head_idim * seq_kv * 2; // q*k+ OPS += group * head_kv_num * seq_cur * seq_kv * 2; // # sfmx+ OPS += group * head_kv_num * seq_cur * head_odim * seq_kv * 2; // # s*v+ size_t MEM = group * head_kv_num * seq_cur * head_idim * sizeof(AType); // # q+ MEM += head_kv_num * seq_kv * head_idim * sizeof(AType); // # k+ MEM += head_kv_num * seq_kv * head_odim * sizeof(AType); // # v++ double flops = double(OPS) / log.min_val / 1e6;+ double mms = double(MEM) / log.min_val / 1e6;+ printf(" %s Flops:%.3f GOPS Bandwidth:%.3f GB/s\n", log.get_log_str(), flops, mms);+ }++ void benchmark_all(int seq, int seq_kv, int hn_q, int hn_kv, int h_dim, int h_vdim, bool verify = false) {+ auto batch = verify ? 1 : 32;+ // printf("%d %d %d %d %s %s %s\n", m, n, k, batch, bestla_dtype_str(BTLA_DTYPE::F16),+ // bestla_dtype_str(BTLA_DTYPE::F16), bestla_dtype_str(BTLA_DTYPE::F16));+ float testtime = float(TestMs);+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ float scale = 1 / sqrt(h_dim);+ sycl_vector<AType> dQ(size_t(seq) * hn_q * h_dim * batch, q);+ sycl_vector<BType> dK(size_t(seq_kv) * hn_kv * h_dim * batch, q);+ sycl_vector<CType> dV(size_t(seq_kv) * hn_kv * h_vdim * batch, q);+ sycl_vector<CType> dO(size_t(seq) * hn_q * h_vdim * batch, q);+ sycl_vector<CType> dMask(seq * seq_kv * batch, q);+ avector<utils::fp16> V(size_t(seq_kv) * hn_kv * h_vdim), O(size_t(seq) * hn_q * h_vdim), Mask(seq * seq_kv);+ avector<utils::fp16> Q(size_t(seq) * hn_q * h_dim), Ref16(size_t(seq) * hn_q * h_vdim),+ K(size_t(seq_kv) * hn_kv * h_dim);+ avector<float> Ref(size_t(seq) * hn_q * h_vdim), Of32(size_t(seq) * hn_q * h_vdim);+ if (verify) {+ fill_buffer_randn(Q.data(), Q.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ fill_buffer_randn(K.data(), K.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ fill_buffer_randn(V.data(), V.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ // for (int i = 0; i < seq_kv / 2; i++) {+ // K[i * hn_kv * h_dim + (i % h_dim)] = float(K[i * hn_kv * h_dim + (i % h_dim)]) + 10.f;+ // V[i * hn_kv * h_vdim + (i % h_vdim)] = float(V[i * hn_kv * h_vdim + (i % h_vdim)]) + 10.f;+ // }+ fill_buffer_randn(Mask.data(), Mask.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ for (size_t i = 0; i < batch; i++) {+ q->memcpy(dQ.data() + i * Q.size(), Q.data(), Q.size() * sizeof(Q[0]));+ q->memcpy(dK.data() + i * K.size(), K.data(), K.size() * sizeof(K[0]));+ q->memcpy(dV.data() + i * V.size(), V.data(), V.size() * sizeof(V[0]));+ q->memcpy(dMask.data() + i * Mask.size(), Mask.data(), Mask.size() * sizeof(Mask[0]));+ }+ q->wait();+ }++ benchmark(hn_q, hn_kv, seq, seq_kv, h_dim, h_vdim, scale, dQ.data(), dK.data(), dV.data(), dO.data(), dMask.data(),+ verify);+ if (verify) {+ auto Qf32 = Q.to<float>();+ auto Kf32 = K.to<float>();+ sdpa_ref(hn_q, hn_kv, seq, seq_kv, h_dim, h_vdim, scale, Qf32.data(), Kf32.data(), V.to<float>().data(),+ Ref.data(), Mask.to<float>().data(), true);+ q->memcpy(O.data(), dO.data(), O.size() * sizeof(O[0])).wait();+ Of32 = O.to<float>();+ buffer_error(Ref.data(), Of32.data(), Of32.size(), 1.f);+ print_buffer(Ref.data(), 16);+ print_buffer(O.data(), 16);+ }+ }+};+#ifdef BTLA_UT_SYCL+static Benchmark_SDPAFp16Fp16 sBenchmark_SDPAFp16Fp16;+#endif+} // namespace sycl_ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_gemm.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_gemm.cppnew file mode 100644index 000000000..b9229a3e7--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_gemm.cpp@@ -0,0 +1,3159 @@+#include "bestla_ut.h"+#include "sycl_ut.h"+#include "../sycl/sycl_wrapper.h"+#include "../sycl/fp8_lut.h"+#include "bestla_prologue_b.h"+#include "kernel_wrapper.h"+#include <sycl/ext/intel/experimental/grf_size_properties.hpp>+#undef BTLA_UT_SYCL+namespace bestla {+using namespace ut;+using namespace utils;+using namespace sycl_utils;+using namespace sycl_gemm;+namespace syclex = sycl::ext::oneapi::experimental;+namespace syclintelex = sycl::ext::intel::experimental;+namespace sycl_ut {+namespace {+template <bool IsE4M3>+inline uint8_t sanitize_fp8_byte(uint8_t raw) {+ if constexpr (IsE4M3) {+ uint8_t exp = (raw >> 3) & 0xF;+ uint8_t mant = raw & 0x7;+ if (exp == 0xF && mant == 0x7) {+ raw = (raw & 0xF8) | 0x6; // avoid NaN payload+ }+ } else {+ uint8_t exp = (raw >> 2) & 0x1F;+ if (exp == 0x1F) {+ raw = (raw & 0x83) | (0x1E << 2); // drop Inf/NaN exponent+ }+ }+ return raw;+}++template <bool IsE4M3>+inline void sanitize_fp8_buffer(uint8_t* data, size_t size) {+ for (size_t i = 0; i < size; ++i) {+ data[i] = sanitize_fp8_byte<IsE4M3>(data[i]);+ }+}++template <bool IsE4M3>+inline float decode_fp8_byte(uint8_t raw) {+ const uint8_t mag = raw & 0x7F;+ const float v = IsE4M3 ? sycl_prologue_b::fp8_lut::lut_e4m3_128[mag] : sycl_prologue_b::fp8_lut::lut_e5m2_128[mag];+ return (raw & 0x80) ? -v : v;+}+} // namespace++class UT_SyclS4IGemm {+ public:+ UT_SyclS4IGemm() {+ UT_START();+ // ut_testT2();+ // ut_testT1(552, 552, 96);+ ut_testT3(4096, 4096, 4096, 64);+ ut_testT3(4096, 4096, 4096, 128);+ ut_testT3_half(4096, 4096, 4096, 64);+ ut_testT3_half(4096, 4096, 4096, 128);+ // ut_testT3(4096, 4096, 4096, 256);+ // ut_testT3(4096, 4096, 4096, 512);+ // ut_testT3(4096, 4096, 4096, 1024);+ // ut_testT3(4096, 4096, 4096, 4096);+ // ut_testT4(4096, 4096, 4096, 128);+ // ut_testT1(4096, 4096, 4096);+ // ut_testT1(4096, 11008, 4096);+ // ut_testT1(4096, 16384, 4096);+ }++ void ut_testT4(int m, int n, int k, int blocksize) {+ int constexpr runs = 100;+ int constexpr sg_size = 16;+ int constexpr sg_row = 8; // half threads of a XVE+ int constexpr sg_col = 4; // half threads of a XVE+ int constexpr wg_size = sg_size * sg_row * sg_col; // half threads of a XVE+ int constexpr TM = 8;+ int constexpr TN = 16;+ int constexpr TK = 32;+ int constexpr SGM = 4;+ int constexpr SGN = 4;+ int constexpr UnrollK = 2;+ // A large GRF kernel+ // SYCL_PROGRAM_COMPILE_OPTIONS="-ze-opt-large-register-file"+ int constexpr PrefetchDis = 3;+ int constexpr G_NROW = 3;+ int constexpr BM = TM * sg_row * SGM;+ int constexpr BM_STRIDE = BM + 8;+ int constexpr BN = TN * sg_col * SGN;+ int constexpr BN_STRIDE = BN;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ namespace syclex = sycl::ext::oneapi::experimental;+ namespace syclintelex = sycl::ext::intel::experimental;++ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case: %d %d %d Device:%s\n", m, n, k, dev->getName().c_str());+ avector<float> matAfp(m * k), scaleA(m), matB(k * n), matC(m * n), ref(m * n);+ avector<int8_t> matA(m * k);+ fill_buffer_randn(scaleA.data(), scaleA.size(), 0.001f, 0.005f);+ fill_buffer_randn(matA.data(), matA.size(), int8_t(-128), int8_t(127));+ for (int i = 0; i < m; i++) {+ for (int j = 0; j < k; j += 1) {+ matAfp[i * k + j] = static_cast<float>(matA[i * k + j]) * scaleA[i];+ }+ }++ int blks = k / blocksize;+ avector<float> B_scale(size_t(blks) * n);+ avector<int8_t> B_s8(k * n), B_s8NT(k * n);+ fill_buffer_randn(B_s8.data(), B_s8.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(B_scale.data(), B_scale.size(), 0.001f, 0.005f);+ for (int i = 0; i < n; i++) {+ for (int j = 0; j < k; j += 1) {+ auto noffset = i * blks + j / blocksize;+ matB[i * k + j] = static_cast<float>(B_s8[i * k + j]) * B_scale[noffset];+ }+ }+ avector<float> matBNT(k * n);+ kernel::wrapper::Transpose2D<float>::forward<BTLA_ISA::NoSIMD>(matB.data(), matBNT.data(), n, k, k, n);+ kernel::wrapper::Transpose2D<int8_t>::forward<BTLA_ISA::NoSIMD>(B_s8.data(), B_s8NT.data(), n, k, k, n);+ gemmref_fp32fp32fp32(m, n, k, matAfp.data(), matBNT.data(), ref.data(), k, n, n);+ avector<int> testRef(m * n), O32(m * n);+ gemmref_s8s8s32(m, n, k, matA.data(), B_s8NT.data(), testRef.data(), k, n, n);+ sycl_vector<float> dB(matB.size(), q), dC(matC.size(), q), dB_scale(B_scale.size(), q), dA_scale(m, q);+ sycl_vector<int8_t> dA(B_s8.size(), q), dBs8(B_s8.size(), q);+ sycl_vector<int> dO(m * n, q);+ sycl_vector<int> dws0(m * n, q);+ q->memcpy(dA.data(), matA.data(), matA.size() * 1).wait();+ q->memcpy(dBs8.data(), B_s8.data(), B_s8.size() * 1).wait();+ q->memcpy(dB_scale.data(), B_scale.data(), B_scale.size() * 4).wait();+ q->memcpy(dA_scale.data(), scaleA.data(), scaleA.size() * 4).wait();+ auto A_d = dA.data();+ auto B_d = dBs8.data();+ auto A_scale_d = dA_scale.data();+ auto B_scale_d = dB_scale.data();+ auto C_d = dC.data();+ auto C_d_s32 = dO.data();+ auto ws0 = dws0.data();++ size_t g_ncnt = (n + BN - 1) / BN;+ size_t g_mcnt = (m + BM - 1) / BM;+ size_t wg_repeat = g_mcnt * g_ncnt;+ size_t m_tail = g_mcnt % G_NROW;+ size_t g_m_aligned = g_mcnt - m_tail;+ size_t aligned_wg = g_m_aligned * g_ncnt;++ auto ker = [&](sycl::handler& cgh) {+ // sycl::stream out(65536, 256, cgh);+ // sycl::local_accessor<int, 1> slm_b(sycl::range(BN * BM), cgh);+ // sycl::local_accessor<float, 1> slm_bf(sycl::range(BN * BM), cgh);+ cgh.parallel_for(+ sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(sg_size)]]+ {+ auto sg = it.get_sub_group();+ int g_id = it.get_group(0);+ size_t ws_off = (size_t)g_id * BN * BM;+ int g_n = g_id % g_ncnt;+ int g_m = g_id / g_ncnt;+ if (g_id <= aligned_wg) {+ int g_m_ = g_id % G_NROW;+ g_id /= G_NROW;+ g_n = g_id % g_ncnt;+ g_id /= g_ncnt;+ g_m = g_id * G_NROW + g_m_;+ }+ if (g_n >= g_ncnt || g_m >= g_mcnt) return;++ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;++ int sgId = sg.get_local_id()[0];+ int sg_idn = sggid_col * TN * SGN;+ int sg_idm = sggid_row * TM * SGM;+ int g_idn = sggid_col * TN * SGN + g_n * BN;+ int g_idm = sggid_row * TM * SGM + g_m * BM;+ // auto pA =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(A_d);+ auto pSA = syclex::annotated_ptr{+ A_scale_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L3>>}};+ auto pSB = syclex::annotated_ptr{+ B_scale_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L3>>}};+ auto pA = syclex::annotated_ptr{+ A_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L3>>}};+ // auto pB =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(B_d);+ auto pB = syclex::annotated_ptr{+ B_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};+ // auto pC = sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(+ // C_d_s32);+ auto pC = syclex::annotated_ptr{+ C_d, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_back, syclex::cache_level::L1, syclex::cache_level::L3>>}};+ auto pCt = syclex::annotated_ptr{+ ws0 + ws_off,+ syclex::properties{syclintelex::write_hint<+ syclintelex::cache_control<syclintelex::cache_mode::write_back, syclex::cache_level::L3>>}};+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int8_t, use::a, TM, TK,+ layout::row_major>+ sub_a[SGM];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int8_t, use::b, TK, TN,+ layout::col_major>+ sub_b[SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int32_t, use::accumulator, TM, TN>+ sub_c[SGM * SGN];++ if (TK * UnrollK * (PrefetchDis + 1) <= k)+ for (size_t i = 0; i < PrefetchDis; i++) {+ for (int im = sggid_col; im < SGM; im += sg_col)+ if ((g_idm + (im + 1) * TM) <= m)+ joint_matrix_prefetch<TM, TK * UnrollK>(sg, A_d + (g_idm + im * TM) * k * 1 + TK * UnrollK * i, k,+ layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});++ for (int in = sggid_row; in < SGN; in += sg_row)+ if ((g_idn + (in + 1) * TN) <= n)+ joint_matrix_prefetch<TN, TK * UnrollK>(sg, B_d + (g_idn + in * TN) * k + TK * UnrollK * i, k,+ layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ }++ for (size_t ib = 0; ib < blks; ib++) {+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) joint_matrix_fill(sg, sub_c[im * SGN + in], 0);+#pragma unroll(1)+ for (size_t ik = ib * blocksize; ik < (ib + 1) * blocksize; ik += TK * UnrollK) {+#pragma unroll+ for (size_t ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_b[in], pB, k, n, k, (g_idn + in * TN) * 1, ik + ikk * TK);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_a[im], pA, k, m, k, (g_idm + im * TM) * 1, ik + TK * ikk);+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_mad(sg, sub_c[im * SGN + in], sub_a[im], sub_b[in], sub_c[im * SGN + in]);+ }+ }+ }+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++)+ joint_matrix_prefetch<TM, TN>(sg, C_d + (g_idm + im * TM) * n + g_idn + in * TN, n,+ layout::row_major, syclex::properties{syclex::prefetch_hint_L3});+ if (ik + TK * UnrollK * (PrefetchDis + 1) <= k) {+ for (int im = sggid_col; im < SGM; im += sg_col)+ if ((g_idm + (im + 1) * TM) <= m)+ joint_matrix_prefetch<TM, TK * UnrollK>(+ sg, A_d + (g_idm + im * TM) * k * 1 + ik + TK * UnrollK * PrefetchDis, k, layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ for (int in = sggid_row; in < SGN; in += sg_row)+ if ((g_idn + (in + 1) * TN) <= n)+ joint_matrix_prefetch<TN, TK * UnrollK>(+ sg, B_d + (g_idn + in * TN) * k + (ik + TK * UnrollK * PrefetchDis), k, layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ }+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_store(sg, sub_c[im * SGN + in], pCt + (im * TM + sg_idm) * BN + sg_idn + in * TN, BN,+ layout::row_major);+ }+ }++#pragma unroll+ for (int im = 0; im < SGM; im++) {+#pragma unroll+ for (size_t imm = 0; imm < TM; imm++) {+ auto tmp = *(sycl::vec<int, SGN>*)&ws0[(im * TM + imm + sg_idm) * BN + sg_idn + sgId * SGN + ws_off];+ auto tmp2 = *(sycl::vec<float, SGN>*)&C_d[(g_idm + im * TM + imm) * n + g_idn + sgId * SGN];+ // sycl::vec<float, SGN> tmp2;+ // float scaleA = pSA[g_idm + im * TM + imm];+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ float scaleB = pSB[(g_idn + sgId * SGN + in) * blks + ib]; // * scaleA;+ tmp2[in] = tmp2[in] + static_cast<float>(tmp[in]) * scaleB;+ }+ *(sycl::vec<float, SGN>*)&C_d[(g_idm + im * TM + imm) * n + g_idn + sgId * SGN] = tmp2;+ }+ }+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+#pragma unroll+ for (size_t imm = 0; imm < TM; imm++) {+ auto tmp2 = *(sycl::vec<float, SGN>*)&C_d[(g_idm + im * TM + imm) * n + g_idn + sgId * SGN];+ float scaleA = pSA[g_idm + im * TM + imm];+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ tmp2[in] = tmp2[in] * scaleA;+ }+ *(sycl::vec<float, SGN>*)&C_d[(g_idm + im * TM + imm) * n + g_idn + sgId * SGN] = tmp2;+ }+ }+ }); // parallel for+ };+ q->wait();+ for (size_t i = 0; i < runs; i++) {+ q->submit(ker);+ }+ q->wait();+ bestla::utils::timer<bestla::utils::microseconds> tm;+ tm.start();+ for (size_t i = 0; i < runs; i++) {+ q->submit(ker);+ }+ q->wait();+ auto t_ms = tm.stop() / runs;+ double ops = (double)m * n * k * 2;+ printf("Time %f us, %f GFLOPS\n", t_ms, ops / t_ms / 1e3);+ q->memcpy(matC.data(), C_d, matC.size() * 4).wait();+ // q->memcpy(O32.data(), C_d_s32, matC.size() * 4).wait();+ // buffer_error(testRef.data(), O32.data(), testRef.size(), 0);+ buffer_error(ref.data(), matC.data(), ref.size(), 0.01f);+ }++ void ut_testT3_half(int m, int n, int k, int blocksize) {+ int constexpr runs = 100;+ int constexpr sg_size = 16;+ int constexpr sg_row = 8; // half threads of a XVE+ int constexpr sg_col = 4; // half threads of a XVE+ int constexpr wg_size = sg_size * sg_row * sg_col; // half threads of a XVE+ int constexpr TM = 8;+ int constexpr TN = 16;+ int constexpr TK = 32;+ int constexpr SGM = 4;+ int constexpr SGN = 4;+ int constexpr UnrollK = 2;+ // A large GRF kernel+ // SYCL_PROGRAM_COMPILE_OPTIONS="-ze-opt-large-register-file"+ int constexpr PrefetchDis = 3;+ int constexpr G_NROW = 3;+ int constexpr BM = TM * sg_row * SGM;+ int constexpr BM_STRIDE = BM + 8;+ int constexpr BN = TN * sg_col * SGN;+ int constexpr BN_STRIDE = BN;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ namespace syclex = sycl::ext::oneapi::experimental;+ namespace syclintelex = sycl::ext::intel::experimental;++ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ int blks = k / blocksize;+ printf("Test Case: %d %d %d %d Device:%s\n", m, n, k, blocksize, dev->getName().c_str());+ avector<float> matAfp(m * k), matB(k * n), ref(m * n);+ avector<bestla::utils::fp16> B_scale(size_t(blks) * n), scaleA(m), matC(m * n);+ avector<int8_t> matA(m * k);+ fill_buffer_randn(scaleA.data(), scaleA.size(), bestla::utils::fp16(0.001f), bestla::utils::fp16(0.005f));+ fill_buffer_randn(matA.data(), matA.size(), int8_t(-128), int8_t(127));+ for (int i = 0; i < m; i++) {+ for (int j = 0; j < k; j += 1) {+ matAfp[i * k + j] = static_cast<float>(matA[i * k + j]) * float(scaleA[i]);+ }+ }++ avector<int8_t> B_s8(k * n), B_s8NT(k * n);+ fill_buffer_randn(B_s8.data(), B_s8.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(B_scale.data(), B_scale.size(), bestla::utils::fp16(0.001f), bestla::utils::fp16(0.005f));+ for (int i = 0; i < n; i++) {+ for (int j = 0; j < k; j += 1) {+ auto noffset = i * blks + j / blocksize;+ matB[i * k + j] = static_cast<float>(B_s8[i * k + j]) * float(B_scale[noffset]);+ }+ }+ avector<float> matBNT(k * n);+ kernel::wrapper::Transpose2D<float>::forward<BTLA_ISA::NoSIMD>(matB.data(), matBNT.data(), n, k, k, n);+ kernel::wrapper::Transpose2D<int8_t>::forward<BTLA_ISA::NoSIMD>(B_s8.data(), B_s8NT.data(), n, k, k, n);+ gemmref_fp32fp32fp32(m, n, k, matAfp.data(), matBNT.data(), ref.data(), k, n, n);+ avector<bestla::utils::fp16> ref_half(ref.size());+ for (size_t i = 0; i < ref.size(); i++) {+ ref_half[i] = ref[i];+ }++ avector<int> testRef(m * n), O32(m * n);+ gemmref_s8s8s32(m, n, k, matA.data(), B_s8NT.data(), testRef.data(), k, n, n);+ sycl_vector<sycl::half> dC(matC.size(), q), dB_scale(B_scale.size(), q), dA_scale(m, q);+ sycl_vector<int8_t> dA(B_s8.size(), q), dBs8(B_s8.size(), q);+ q->memcpy(dA.data(), matA.data(), matA.size() * 1).wait();+ q->memcpy(dBs8.data(), B_s8.data(), B_s8.size() * 1).wait();+ q->memcpy(dB_scale.data(), B_scale.data(), B_scale.size() * 2).wait();+ q->memcpy(dA_scale.data(), scaleA.data(), scaleA.size() * 2).wait();+ auto A_d = dA.data();+ auto B_d = dBs8.data();+ auto A_scale_d = dA_scale.data();+ auto B_scale_d = dB_scale.data();+ auto C_d = dC.data();++ size_t g_ncnt = (n + BN - 1) / BN;+ size_t g_mcnt = (m + BM - 1) / BM;+ size_t wg_repeat = g_mcnt * g_ncnt;+ size_t m_tail = g_mcnt % G_NROW;+ size_t g_m_aligned = g_mcnt - m_tail;+ size_t aligned_wg = g_m_aligned * g_ncnt;++ auto ker = [&](sycl::handler& cgh) {+ // sycl::stream out(65536, 256, cgh);+ // sycl::local_accessor<int, 1> slm_b(sycl::range(BN * BM), cgh);+ cgh.parallel_for(+ sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(sg_size)]]+ {+ auto sg = it.get_sub_group();+ int g_id = it.get_group(0);+ int g_n = g_id % g_ncnt;+ int g_m = g_id / g_ncnt;+ if (g_id <= aligned_wg) {+ int g_m_ = g_id % G_NROW;+ g_id /= G_NROW;+ g_n = g_id % g_ncnt;+ g_id /= g_ncnt;+ g_m = g_id * G_NROW + g_m_;+ }+ if (g_n >= g_ncnt || g_m >= g_mcnt) return;++ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;++ int sgId = sg.get_local_id()[0];+ int g_idn = sggid_col * TN * SGN + g_n * BN;+ int g_idm = sggid_row * TM * SGM + g_m * BM;+ // auto pA =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(A_d);+ // auto pB =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(B_d);+ // auto pC = sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(+ // C_d_s32);+ auto pSA =+ sycl::address_space_cast<sycl::access::address_space::global_space, sycl::access::decorated::yes>(+ A_scale_d);+ // auto pSB =+ // sycl::address_space_cast<sycl::access::address_space::global_space, sycl::access::decorated::yes>(+ // B_scale_d);+ // auto pSA = syclex::annotated_ptr{+ // A_scale_d, syclex::properties{syclintelex::read_hint<+ // syclintelex::cache_control<syclintelex::cache_mode::streaming,+ // syclex::cache_level::L1>, syclintelex::cache_control<syclintelex::cache_mode::cached,+ // syclex::cache_level::L3>>}};+ auto pSB = syclex::annotated_ptr{+ B_scale_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L3>>}};+ auto pA = syclex::annotated_ptr{+ A_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};++ auto pB = syclex::annotated_ptr{+ B_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};++ auto pC = syclex::annotated_ptr{+ C_d, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_through, syclex::cache_level::L1, syclex::cache_level::L3>>}};+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int8_t, use::a, TM, TK,+ layout::row_major>+ sub_a[SGM];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int8_t, use::b, TK, TN,+ layout::col_major>+ sub_b[SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int32_t, use::accumulator, TM, TN>+ sub_c[SGM * SGN];+ // sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, sycl::half, use::accumulator, TM,+ // TN>+ // sub_c_fp[SGM * SGN];+ sycl::half acc_c[SGM * SGN * TM];+ for (size_t i = 0; i < SGM * SGN * TM; i++) {+ acc_c[i] = sycl::half(0);+ }++ if (TK * UnrollK * (PrefetchDis + 1) <= k)+ for (size_t i = 0; i < PrefetchDis; i++) {+ for (int im = sggid_col; im < SGM; im += sg_col)+ if ((g_idm + (im + 1) * TM) <= m)+ joint_matrix_prefetch<TM, TK * UnrollK>(sg, A_d + (g_idm + im * TM) * k * 1 + TK * UnrollK * i, k,+ layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});++ for (int in = sggid_row; in < SGN; in += sg_row)+ if ((g_idn + (in + 1) * TN) <= n)+ joint_matrix_prefetch<TN, TK * UnrollK>(sg, B_d + (g_idn + in * TN) * k + TK * UnrollK * i, k,+ layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ }++ for (size_t ib = 0; ib < blks; ib++) {+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) joint_matrix_fill(sg, sub_c[im * SGN + in], 0);+#pragma unroll(1)+ for (size_t ik = ib * blocksize; ik < (ib + 1) * blocksize; ik += TK * UnrollK) {+#pragma unroll+ for (size_t ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_b[in], pB, k, n, k, (g_idn + in * TN) * 1, ik + ikk * TK);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_a[im], pA, k, m, k, (g_idm + im * TM) * 1, ik + TK * ikk);+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_mad(sg, sub_c[im * SGN + in], sub_a[im], sub_b[in], sub_c[im * SGN + in]);+ }+ }+ }+ if (ik + TK * UnrollK * (PrefetchDis + 1) <= k) {+ for (int im = sggid_col; im < SGM; im += sg_col)+ if ((g_idm + (im + 1) * TM) <= m)+ joint_matrix_prefetch<TM, TK * UnrollK>(+ sg, A_d + (g_idm + im * TM) * k * 1 + ik + TK * UnrollK * PrefetchDis, k, layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ for (int in = sggid_row; in < SGN; in += sg_row)+ if ((g_idn + (in + 1) * TN) <= n)+ joint_matrix_prefetch<TN, TK * UnrollK>(+ sg, B_d + (g_idn + in * TN) * k + (ik + TK * UnrollK * PrefetchDis), k, layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ }+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ auto wi_data_c = sycl::ext::oneapi::detail::get_wi_data(sg, sub_c[im * SGN + in]);+ for (size_t imm = 0; imm < TM; imm++) {+ auto element = wi_data_c[imm];+ auto scaleB = pSB[(g_idn + in * TN + sgId) * blks + ib];+ acc_c[(im * SGN + in) * TM + imm] += static_cast<sycl::half>(element) * scaleB;+ }+ }+ }+ }++#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ for (size_t imm = 0; imm < TM; imm++) {+ auto scaleA = pSA[g_idm + im * TM + imm];+ acc_c[(im * SGN + in) * TM + imm] *= scaleA;+ pC[(g_idm + im * TM + imm) * n + g_idn + in * TN + sgId] = acc_c[(im * SGN + in) * TM + imm];+ }+ // joint_matrix_store(sg, sub_c[im * SGN + in], pC + (g_idm + im * TM) * n + g_idn + in *TN,+ // n,layout::row_major);+ // sycl::ext::intel::experimental::matrix::joint_matrix_store_checked(+ // sg, sub_c_fp[im * SGN + in], pC, n, layout::row_major, m, n, g_idm + im * TM, g_idn + in * TN);+ }+ }); // parallel for+ };+ q->wait();+ for (size_t i = 0; i < runs; i++) {+ // q->submit(ker);+ Launcher<xmx::IKblockGemmDQCfg<sycl::half>, xmx::IKblockGemmDQCore>::run(+ q, {A_d, B_d, C_d, m, n, k, k, k, n, nullptr, A_scale_d, B_scale_d, blocksize});+ }+ q->wait();+ bestla::utils::timer<bestla::utils::microseconds> tm;+ tm.start();+ for (size_t i = 0; i < runs; i++) {+ // q->submit(ker);+ Launcher<xmx::IKblockGemmDQCfg<sycl::half>, xmx::IKblockGemmDQCore>::run(+ q, {A_d, B_d, C_d, m, n, k, k, k, n, nullptr, A_scale_d, B_scale_d, blocksize});+ }+ q->wait();+ auto t_ms = tm.stop() / runs;+ double ops = (double)m * n * k * 2;+ printf("Time %f us, %f GFLOPS\n", t_ms, ops / t_ms / 1e3);+ q->memcpy(matC.data(), C_d, matC.size() * 2).wait();+ // q->memcpy(O32.data(), C_d_s32, matC.size() * 4).wait();+ // buffer_error(testRef.data(), O32.data(), testRef.size(), 0);+ buffer_error(ref_half.data(), matC.data(), ref_half.size(), bestla::utils::fp16(0.01f));+ }++ template <typename T1, typename T2>+ struct KernelFunctor {+ T1* mPA;+ T2 mProp;+ KernelFunctor(T1* PA, T2 Prop) : mPA(PA), mProp(Prop) {}++ void operator()(sycl::id<1> i) const { mPA[i] += 2; }+ auto get(syclex::properties_tag) const { return mProp; }+ };++ void ut_testT3(int m, int n, int k, int blocksize) {+ int constexpr runs = 100;+ int constexpr sg_size = 16;+ int constexpr sg_row = 4; // half threads of a XVE+ int constexpr sg_col = 4; // half threads of a XVE+ int constexpr wg_size = sg_size * sg_row * sg_col; // half threads of a XVE+ int constexpr TM = 8;+ int constexpr TN = 16;+ int constexpr TK = 32;+ int constexpr SGM = 4;+ int constexpr SGN = 2;+ int constexpr UnrollK = 2;+ // A large GRF kernel+ // SYCL_PROGRAM_COMPILE_OPTIONS="-ze-opt-large-register-file"+ int constexpr PrefetchDis = 3;+ int constexpr G_NROW = 3;+ int constexpr BM = TM * sg_row * SGM;+ int constexpr BM_STRIDE = BM + 8;+ int constexpr BN = TN * sg_col * SGN;+ int constexpr BN_STRIDE = BN;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;++ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case: %d %d %d Block:%d Device:%s\n", m, n, k, blocksize, dev->getName().c_str());+ avector<float> matAfp(m * k), scaleA(m), matB(k * n), matC(m * n), ref(m * n);+ avector<int8_t> matA(m * k);+ fill_buffer_randn(scaleA.data(), scaleA.size(), 0.001f, 0.005f);+ fill_buffer_randn(matA.data(), matA.size(), int8_t(-128), int8_t(127));+ for (int i = 0; i < m; i++) {+ for (int j = 0; j < k; j += 1) {+ matAfp[i * k + j] = static_cast<float>(matA[i * k + j]) * scaleA[i];+ }+ }++ int blks = k / blocksize;+ avector<float> B_scale(size_t(blks) * n);+ avector<int8_t> B_s8(k * n), B_s8NT(k * n);+ fill_buffer_randn(B_s8.data(), B_s8.size(), int8_t(-8), int8_t(7));+ fill_buffer_randn(B_scale.data(), B_scale.size(), 0.001f, 0.005f);+ for (int i = 0; i < n; i++) {+ for (int j = 0; j < k; j += 1) {+ auto noffset = i * blks + j / blocksize;+ matB[i * k + j] = static_cast<float>(B_s8[i * k + j]) * B_scale[noffset];+ }+ }+ avector<float> matBNT(k * n), bias(n);+ fill_buffer_randn(bias.data(), bias.size(), 0.1f, 0.5f);++ kernel::wrapper::Transpose2D<float>::forward<BTLA_ISA::NoSIMD>(matB.data(), matBNT.data(), n, k, k, n);+ kernel::wrapper::Transpose2D<int8_t>::forward<BTLA_ISA::NoSIMD>(B_s8.data(), B_s8NT.data(), n, k, k, n);+ gemmref_fp32fp32fp32(m, n, k, matAfp.data(), matBNT.data(), ref.data(), k, n, n);+ for (int i = 0; i < m; i++) {+ for (int j = 0; j < n; j += 1) {+ ref[i * n + j] += bias[j];+ }+ }++ avector<int> testRef(m * n), O32(m * n);+ gemmref_s8s8s32(m, n, k, matA.data(), B_s8NT.data(), testRef.data(), k, n, n);+ sycl_vector<float> dB(matB.size(), q), dC(matC.size(), q), dB_scale(B_scale.size(), q), dA_scale(m, q), dBias(n, q);+ sycl_vector<int8_t> dA(B_s8.size(), q), dBs8(B_s8.size(), q);+ sycl_vector<int> dO(m * n, q);+ q->memcpy(dA.data(), matA.data(), matA.size() * 1).wait();+ q->memcpy(dBs8.data(), B_s8.data(), B_s8.size() * 1).wait();+ q->memcpy(dB_scale.data(), B_scale.data(), B_scale.size() * 4).wait();+ q->memcpy(dA_scale.data(), scaleA.data(), scaleA.size() * 4).wait();+ q->memcpy(dBias.data(), bias.data(), bias.size() * 4).wait();+ auto A_d = dA.data();+ auto B_d = dBs8.data();+ auto A_scale_d = dA_scale.data();+ auto B_scale_d = dB_scale.data();+ auto C_d = dC.data();+ auto C_d_s32 = dO.data();++ size_t g_ncnt = (n + BN - 1) / BN;+ size_t g_mcnt = (m + BM - 1) / BM;+ size_t wg_repeat = g_mcnt * g_ncnt;+ size_t m_tail = g_mcnt % G_NROW;+ size_t g_m_aligned = g_mcnt - m_tail;+ size_t aligned_wg = g_m_aligned * g_ncnt;+ // syclex::properties prop{syclintelex::grf_size<256>};+ // auto e = q->submit(+ // [&](sycl::handler& cgh) { cgh.parallel_for<class SYCLKernelSpecifiedGRF>(32, KernelFunctor(A_d, prop)); });+ // e.wait();++ const auto ker = [=](sycl::handler& cgh) {+ // sycl::stream out(65536, 256, cgh);+ // sycl::local_accessor<int, 1> slm_b(sycl::range(BN * BM), cgh);+ cgh.parallel_for(+ sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(sg_size)]] [[sycl_grf_size]]+ {+ syclex::properties volatile kernel_properties{syclintelex::grf_size<256>};+ (void)kernel_properties;+ auto sg = it.get_sub_group();+ int g_id = it.get_group(0);+ int g_n = g_id % g_ncnt;+ int g_m = g_id / g_ncnt;+ if (g_id < aligned_wg) {+ int g_m_ = g_id % G_NROW;+ g_id /= G_NROW;+ g_n = g_id % g_ncnt;+ g_id /= g_ncnt;+ g_m = g_id * G_NROW + g_m_;+ }+ if (g_n >= g_ncnt || g_m >= g_mcnt) return;++ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;++ int sgId = sg.get_local_id()[0];+ int g_idn = sggid_col * TN * SGN + g_n * BN;+ int g_idm = sggid_row * TM * SGM + g_m * BM;+ // auto pA =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(A_d);+ // auto pB =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(B_d);+ // auto pC = sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(+ // C_d_s32);+ auto pSA =+ sycl::address_space_cast<sycl::access::address_space::global_space, sycl::access::decorated::yes>(+ A_scale_d);+ // auto pSB =+ // sycl::address_space_cast<sycl::access::address_space::global_space, sycl::access::decorated::yes>(+ // B_scale_d);+ // auto pSA = syclex::annotated_ptr{+ // A_scale_d, syclex::properties{syclintelex::read_hint<+ // syclintelex::cache_control<syclintelex::cache_mode::streaming,+ // syclex::cache_level::L1>, syclintelex::cache_control<syclintelex::cache_mode::cached,+ // syclex::cache_level::L3>>}};+ auto pSB = syclex::annotated_ptr{+ B_scale_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L3>>}};+ auto pA = syclex::annotated_ptr{+ A_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};++ auto pB = syclex::annotated_ptr{+ B_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};++ auto pC = syclex::annotated_ptr{+ C_d, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_through, syclex::cache_level::L1, syclex::cache_level::L3>>}};+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int8_t, use::a, TM, TK,+ layout::row_major>+ sub_a[SGM];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int8_t, use::b, TK, TN,+ layout::col_major>+ sub_b[SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int32_t, use::accumulator, TM, TN>+ sub_c[SGM * SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, float, use::accumulator, TM, TN>+ sub_c_fp[SGM * SGN];+ if (TK * UnrollK * (PrefetchDis + 1) <= k)+ for (size_t i = 0; i < PrefetchDis; i++) {+ for (int im = sggid_col; im < SGM; im += sg_col)+ if ((g_idm + (im + 1) * TM) <= m)+ joint_matrix_prefetch<TM, TK * UnrollK>(sg, A_d + (g_idm + im * TM) * k * 1 + TK * UnrollK * i, k,+ layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});++ for (int in = sggid_row; in < SGN; in += sg_row)+ if ((g_idn + (in + 1) * TN) <= n)+ joint_matrix_prefetch<TN, TK * UnrollK>(sg, B_d + (g_idn + in * TN) * k + TK * UnrollK * i, k,+ layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ }+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) joint_matrix_fill(sg, sub_c_fp[im * SGN + in], 0.f);+ sycl::group_barrier(sg);+ for (size_t ib = 0; ib < blks; ib++) {+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) joint_matrix_fill(sg, sub_c[im * SGN + in], 0);+#pragma unroll(1)+ for (int ik = ib * blocksize; ik < (ib + 1) * blocksize; ik += TK * UnrollK) {+#pragma unroll+ for (int ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_b[in], pB, k, n, k, (g_idn + in * TN) * 1, ik + ikk * TK);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_a[im], pA, k, m, k, (g_idm + im * TM) * 1, ik + TK * ikk);+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_mad(sg, sub_c[im * SGN + in], sub_a[im], sub_b[in], sub_c[im * SGN + in]);+ }+ }+ }+ if (ik + TK * UnrollK * (PrefetchDis + 1) <= k) {+ for (int im = sggid_col; im < SGM; im += sg_col)+ if ((g_idm + (im + 1) * TM) <= m)+ joint_matrix_prefetch<TM, TK * UnrollK>(+ sg, A_d + (g_idm + im * TM) * k * 1 + ik + TK * UnrollK * PrefetchDis, k, layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ for (int in = sggid_row; in < SGN; in += sg_row)+ if ((g_idn + (in + 1) * TN) <= n)+ joint_matrix_prefetch<TN, TK * UnrollK>(+ sg, B_d + (g_idn + in * TN) * k + (ik + TK * UnrollK * PrefetchDis), k, layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ }+ }+ sycl::group_barrier(sg);++#pragma unroll+ for (int im = 0; im < SGM; im++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ auto wi_data_c = sycl::ext::oneapi::detail::get_wi_data(sg, sub_c[im * SGN + in]);+ auto wi_data_c1 = sycl::ext::oneapi::detail::get_wi_data(sg, sub_c_fp[im * SGN + in]);+ for (int i = 0; i < wi_data_c.length(); i++) {+ auto element = wi_data_c[i];+ auto element1 = wi_data_c1[i];+ auto [row, col] = wi_data_c[i].get_coord();+ float scaleB = pSB[(g_idn + in * TN + col) * blks + ib];+ element1 = element1 + static_cast<float>(element) * scaleB;+ wi_data_c1[i] = element1;+ }+ }+ }+ sycl::group_barrier(sg);+ }+ sycl::group_barrier(sg);++#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ auto wi_data_c = sycl::ext::oneapi::detail::get_wi_data(sg, sub_c_fp[im * SGN + in]);+ for (int i = 0; i < wi_data_c.length(); i++) {+ auto element = wi_data_c[i];+ auto [row, col] = wi_data_c[i].get_coord();+ float scaleA = pSA[g_idm + im * TM + row];+ wi_data_c[i] = element * scaleA;+ }+ // joint_matrix_store(sg, sub_c[im * SGN + in], pC + (g_idm + im * TM) * n + g_idn + in * TN, n,+ // layout::row_major);+ sycl::ext::intel::experimental::matrix::joint_matrix_store_checked(+ sg, sub_c_fp[im * SGN + in], pC, n, layout::row_major, m, n, g_idm + im * TM, g_idn + in * TN);+ }+ }); // parallel for+ };++ q->wait();+ for (size_t i = 0; i < runs; i++) {+ // q->submit(ker);+ Launcher<xmx::IKblockGemmDQCfg<float>, xmx::IKblockGemmDQCore>::run(+ q, {A_d, B_d, C_d, m, n, k, k, k, n, dBias.data(), A_scale_d, B_scale_d, blocksize});+ }+ q->wait();+ bestla::utils::timer<bestla::utils::microseconds> tm;+ tm.start();+ for (size_t i = 0; i < runs; i++) {+ // q->submit(ker);+ Launcher<xmx::IKblockGemmDQCfg<float>, xmx::IKblockGemmDQCore>::run(+ q, {A_d, B_d, C_d, m, n, k, k, k, n, dBias.data(), A_scale_d, B_scale_d, blocksize});+ }+ q->wait();+ auto t_ms = tm.stop() / runs;+ double ops = (double)m * n * k * 2;+ printf("Time %f us, %f GFLOPS\n", t_ms, ops / t_ms / 1e3);+ q->memcpy(matC.data(), C_d, matC.size() * 4).wait();+ // q->memcpy(O32.data(), C_d_s32, matC.size() * 4).wait();+ // buffer_error(testRef.data(), O32.data(), testRef.size(), 0);+ buffer_error(ref.data(), matC.data(), ref.size(), 0.01f);+ }++ void ut_testT2() {+ int constexpr sg_size = 16;+ int constexpr TM = 8;+ int constexpr TN = 16;+ int constexpr TK = 32;+ int m = TM, n = TN, k = TK * 2;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ namespace syclex = sycl::ext::oneapi::experimental;+ namespace syclintelex = sycl::ext::intel::experimental;+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case: %d %d %d Device:%s\n", m, n, k, dev->getName().c_str());+ avector<int8_t> matA(m * k), matB(k * n), matBT(k * n), matBT4(k * n / 2);+ avector<int32_t> matC(m * n), ref(m * n);+ fill_buffer_randn(matA.data(), matA.size(), int8_t(-128), int8_t(127));+ fill_buffer_randn(matB.data(), matB.size(), int8_t(-8), int8_t(7));+ gemmref_s8s8s32(m, n, k, matA.data(), matB.data(), ref.data(), k, n, n);+ bestla::kernel::wrapper::Transpose2D<int8_t>::forward_auto(matB.data(), matBT.data(), k, n, n, k);+ for (size_t i = 0; i < n; i++) {+ for (size_t j = 0; j < k; j += 64) {+ for (size_t ik = 0; ik < 32; ik += 1) {+ auto v1 = uint8_t(matBT[i * k + j + ik] + 8);+ auto v2 = uint8_t(matBT[i * k + j + ik + 32] + 8);+ *(uint8_t*)&matBT4[(i * k + j) / 2 + ik] = v1 | (v2 << 4);+ }+ }+ }++ bestla::sycl_vector<int8_t> dA(matA.size(), q), dB4(matBT4.size(), q);+ bestla::sycl_vector<int32_t> dC(matC.size(), q);+ q->memcpy(dA.data(), matA.data(), matA.size() * 1).wait();+ q->memcpy(dB4.data(), matBT4.data(), matBT4.size() * 1).wait();+ auto A_d = dA.data();+ auto B4_d = dB4.data();+ auto C_d = dC.data();++ auto ker = [&](sycl::handler& cgh) {+ sycl::stream out(65536, 256, cgh);+ cgh.parallel_for(+ sycl::nd_range<2>({1, sg_size}, {1, sg_size}), [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(sg_size)]]+ {+ auto sg = it.get_sub_group();+ int g_id = it.get_group(0);++ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sgId = sg.get_local_id()[0];+ int g_idn = 0;+ int g_idm = 0;+ auto pA =+ sycl::address_space_cast<sycl::access::address_space::global_space, sycl::access::decorated::yes>(A_d);+ // auto pA = syclex::annotated_ptr{+ // A_d, syclex::properties{syclintelex::read_hint<+ // syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L1>,+ // syclintelex::cache_control<syclintelex::cache_mode::uncached, syclex::cache_level::L3>>}};+ auto pB4 =+ sycl::address_space_cast<sycl::access::address_space::global_space, sycl::access::decorated::yes>(B4_d);+ // auto pB = syclex::annotated_ptr{+ // B_d, syclex::properties{syclintelex::read_hint<syclintelex::cache_control<+ // syclintelex::cache_mode::cached, syclex::cache_level::L1, syclex::cache_level::L3>>}};+ // auto pC =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(C_d);+ auto pC = syclex::annotated_ptr{+ C_d, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_through, syclex::cache_level::L1, syclex::cache_level::L3>>}};+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int8_t, use::a, TM, TK,+ layout::row_major>+ sub_a, sub_a1;+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int8_t, use::b, TK, TN,+ layout::col_major>+ sub_b, sub_b1;+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int32_t, use::accumulator, TM, TN>+ sub_c;++ joint_matrix_fill(sg, sub_c, 0);+ joint_matrix_load(sg, sub_b, pB4, k / 2);+ joint_matrix_copy(sg, sub_b, sub_b1);+ // int a = sycl::ext::oneapi::detail::get_wi_data(sg, sub_b)[0];+ // a = *(uint8_t*)&a;+ // out << sgId << " " << a << "\n";+ joint_matrix_apply(sg, sub_b, [=](int8_t& val) {+ auto t = *(uint8_t*)&val;+ val = static_cast<int8_t>(t & 0xf) - 8;+ });+ // a = sycl::ext::oneapi::detail::get_wi_data(sg, sub_b)[0];+ // out << sgId << " " << a << "\n";++ // a = sycl::ext::oneapi::detail::get_wi_data(sg, sub_b1)[0];+ // a = *(uint8_t*)&a;+ // out << sgId << " " << a << "\n";+ joint_matrix_apply(sg, sub_b1, [=](int8_t& val) {+ auto t = *(uint8_t*)&val;+ val = static_cast<int8_t>(t >> 4) - 8;+ });+ // a = sycl::ext::oneapi::detail::get_wi_data(sg, sub_b1)[0];+ // out << sgId << " " << a << "\n";+ joint_matrix_load(sg, sub_a, pA, k);+ joint_matrix_load(sg, sub_a1, pA + TK, k);++ joint_matrix_mad(sg, sub_c, sub_a, sub_b, sub_c);+ joint_matrix_mad(sg, sub_c, sub_a1, sub_b1, sub_c);++ // joint_matrix_apply(sg, sub_c, [=](Tc& x) { x *= ALPHA; });+ joint_matrix_store(sg, sub_c, pC, n, layout::row_major);+ }); // parallel for+ };+ q->wait();+ bestla::utils::timer<bestla::utils::microseconds> tm;+ tm.start();+ for (size_t i = 0; i < 1; i++) {+ q->submit(ker);+ }+ q->wait();+ auto t_ms = tm.stop() / 1;+ double ops = (double)m * n * k * 2;+ printf("Time %f us, %f GFLOPS\n", t_ms, ops / t_ms / 1e3);+ q->memcpy(matC.data(), C_d, matC.size() * 4).wait();+ buffer_error(ref.data(), matC.data(), ref.size(), 0);+ }++ void ut_testT1(int m, int n, int k) {+ int constexpr runs = 100;+ int constexpr sg_size = 16;+ int constexpr sg_row = 2; // half threads of a XVE+ int constexpr sg_col = 16; // half threads of a XVE+ int constexpr wg_size = sg_size * sg_row * sg_col; // half threads of a XVE+ int constexpr TM = 8;+ int constexpr TN = 16;+ int constexpr TK = 32;+ int constexpr SGM = 16;+ int constexpr SGN = 1;+ int constexpr UnrollK = 2;+ // A large GRF kernel+ // SYCL_PROGRAM_COMPILE_OPTIONS="-ze-opt-large-register-file"+ int constexpr PrefetchDis = 3;+ int constexpr G_NROW = 3;+ int constexpr BM = TM * sg_row * SGM;+ int constexpr BM_STRIDE = BM + 8;+ int constexpr BN = TN * sg_col * SGN;+ int constexpr BN_STRIDE = BN;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ namespace syclex = sycl::ext::oneapi::experimental;+ namespace syclintelex = sycl::ext::intel::experimental;+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case: %d %d %d Device:%s\n", m, n, k, dev->getName().c_str());+ avector<int8_t> matA(m * k), matB(k * n), matBT(k * n), matBT4(k * n / 2);+ avector<int32_t> matC(m * n), ref(m * n);+ fill_buffer_randn(matA.data(), matA.size(), int8_t(-128), int8_t(127));+ fill_buffer_randn(matB.data(), matB.size(), int8_t(-8), int8_t(7));+ gemmref_s8s8s32(m, n, k, matA.data(), matB.data(), ref.data(), k, n, n);+ bestla::kernel::wrapper::Transpose2D<int8_t>::forward_auto(matB.data(), matBT.data(), k, n, n, k);+ for (size_t i = 0; i < n; i++) {+ for (size_t j = 0; j < k; j += 64) {+ for (size_t ik = 0; ik < 32; ik += 1) {+ auto v1 = uint8_t(matBT[i * k + j + ik] + 8);+ auto v2 = uint8_t(matBT[i * k + j + ik + 32] + 8);+ *(uint8_t*)&matBT4[(i * k + j) / 2 + ik] = v1 | (v2 << 4);+ }+ }+ }+ bestla::sycl_vector<int8_t> dA(matA.size(), q), dB(matBT4.size(), q);+ bestla::sycl_vector<int32_t> dC(matC.size(), q);+ q->memcpy(dA.data(), matA.data(), matA.size() * 1).wait();+ q->memcpy(dB.data(), matBT4.data(), matBT4.size() * 1).wait();+ auto A_d = dA.data();+ auto B_d = dB.data();+ auto C_d = dC.data();+ size_t g_ncnt = (n + BN - 1) / BN;+ size_t g_mcnt = (m + BM - 1) / BM;+ size_t wg_repeat = g_mcnt * g_ncnt;+ size_t m_tail = g_mcnt % G_NROW;+ size_t g_m_aligned = g_mcnt - m_tail;+ size_t aligned_wg = g_m_aligned * g_ncnt;++ auto ker = [&](sycl::handler& cgh) {+ // sycl::stream out(65536, 256, cgh);+ // sycl::local_accessor<int8_t, 2> slm_b(sycl::range(BN_STRIDE, TK * UnrollK), cgh);+ cgh.parallel_for(+ sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(sg_size)]]+ {+ auto sg = it.get_sub_group();+ int g_id = it.get_group(0);+ int g_n = g_id % g_ncnt;+ int g_m = g_id / g_ncnt;+ if (g_id <= aligned_wg) {+ int g_m_ = g_id % G_NROW;+ g_id /= G_NROW;+ g_n = g_id % g_ncnt;+ g_id /= g_ncnt;+ g_m = g_id * G_NROW + g_m_;+ }+ if (g_n >= g_ncnt || g_m >= g_mcnt) return;++ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;++ int sgId = sg.get_local_id()[0];+ int g_idn = sggid_col * TN * SGN + g_n * BN;+ int g_idm = sggid_row * TM * SGM + g_m * BM;+ // auto pA =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(A_d);+ auto pA = syclex::annotated_ptr{+ A_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L3>>}};+ // auto pB =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(B_d);+ auto pB = syclex::annotated_ptr{+ B_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L3>>}};+ // auto pC =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(C_d);+ auto pC = syclex::annotated_ptr{+ C_d, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_through, syclex::cache_level::L1, syclex::cache_level::L3>>}};+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int8_t, use::a, TM, TK,+ layout::row_major>+ sub_a[SGM];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int8_t, use::b, TK, TN,+ layout::col_major>+ sub_b[SGN * UnrollK];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int32_t, use::accumulator, TM, TN>+ sub_c[SGM * SGN];+ if (TK * UnrollK * (PrefetchDis + 1) <= k)+ for (size_t i = 0; i < PrefetchDis; i++) {+ for (int im = sggid_col; im < SGM; im += sg_col)+ if ((g_idm + (im + 1) * TM) <= m)+ joint_matrix_prefetch<TM, TK * UnrollK>(sg, A_d + (g_idm + im * TM) * k * 1 + TK * UnrollK * i, k,+ layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});++ for (int in = sggid_row; in < SGN; in += sg_row)+ if ((g_idn + (in + 1) * TN) <= n)+ joint_matrix_prefetch<TN, TK * UnrollK>(sg, B_d + (g_idn + in * TN) * k / 2 + TK * UnrollK * i / 2,+ k / 2, layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ }++#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) joint_matrix_fill(sg, sub_c[im * SGN + in], 0);++#pragma unroll(1)+ for (size_t ik = 0; ik < k; ik += TK * UnrollK) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_b[in * UnrollK + 0], pB, k / 2, n, k / 2, (g_idn + in * TN) * 1, ik / 2);+ joint_matrix_copy(sg, sub_b[in * UnrollK + 0], sub_b[in * UnrollK + 1]);+ joint_matrix_apply(sg, sub_b[in * UnrollK + 0], [=](int8_t& val) {+ auto t = *(uint8_t*)&val;+ val = static_cast<int8_t>(t & (uint8_t)0xf) - (int8_t)8;+ });+ joint_matrix_apply(sg, sub_b[in * UnrollK + 1], [=](int8_t& val) {+ auto t = *(uint8_t*)&val;+ val = static_cast<int8_t>(t >> (uint8_t)4) - (int8_t)8;+ // val = val; // >> (uint8_t)4;+ });+ // joint_matrix_apply(sg, sub_b[in * UnrollK + 0], sub_b[in * UnrollK + 1],+ // [=](int8_t& val, int8_t& val1) {+ // auto t = *(uint8_t*)&val;+ // val = static_cast<int8_t>(t & (uint8_t)0xf) - (int8_t)8;+ // val1 = static_cast<int8_t>(t >> (uint8_t)4) - (int8_t)8;+ // });++ // auto wi_data_c = sycl::ext::oneapi::detail::get_wi_data(sg, sub_b[in * UnrollK + 0]);+ // auto wi_data_c1 = sycl::ext::oneapi::detail::get_wi_data(sg, sub_b[in * UnrollK + 1]);+ // for (int i = 0; i < wi_data_c.length(); i++) {+ // auto element = wi_data_c[i];+ // auto element1 = wi_data_c1[i];+ // auto t = *(uint8_t*)&element;+ // element = static_cast<int8_t>(t & (uint8_t)0xf) - (int8_t)8;+ // element1 = static_cast<int8_t>(t >> (uint8_t)4) - (int8_t)8;+ // wi_data_c[i] = element;+ // wi_data_c1[i] = element1;+ // }+ }+#pragma unroll+ for (size_t ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_a[im], pA, k, m, k, (g_idm + im * TM) * 1, ik + TK * ikk);+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_mad(sg, sub_c[im * SGN + in], sub_a[im], sub_b[in * UnrollK + ikk],+ sub_c[im * SGN + in]);+ }+ }+ }+ if (ik + TK * UnrollK * (PrefetchDis + 1) <= k) {+ for (int im = sggid_col; im < SGM; im += sg_col)+ if ((g_idm + (im + 1) * TM) <= m)+ joint_matrix_prefetch<TM, TK * UnrollK>(+ sg, A_d + (g_idm + im * TM) * k * 1 + ik + TK * UnrollK * PrefetchDis, k, layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ for (int in = sggid_row; in < SGN; in += sg_row)+ if ((g_idn + (in + 1) * TN) <= n)+ joint_matrix_prefetch<TN, TK * UnrollK>(+ sg, B_d + (g_idn + in * TN) * k / 2 + (ik + TK * UnrollK * PrefetchDis) / 2, k / 2,+ layout::row_major, syclex::properties{syclex::prefetch_hint_L1});+ }+ }+ // joint_matrix_apply(sg, sub_c, [=](Tc& x) { x *= ALPHA; });+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++)+ // joint_matrix_store(sg, sub_c[im * SGN + in], pC + (g_idm + im * TM) * n + g_idn + in * TN, n,+ // layout::row_major);+ sycl::ext::intel::experimental::matrix::joint_matrix_store_checked(+ sg, sub_c[im * SGN + in], pC, n, layout::row_major, m, n, g_idm + im * TM, g_idn + in * TN);+ }); // parallel for+ };+ q->wait();+ for (size_t i = 0; i < runs; i++) {+ q->submit(ker);+ }+ q->wait();+ bestla::utils::timer<bestla::utils::microseconds> tm;+ tm.start();+ for (size_t i = 0; i < runs; i++) {+ q->submit(ker);+ }+ q->wait();+ auto t_ms = tm.stop() / runs;+ double ops = (double)m * n * k * 2;+ printf("Time %f us, %f GFLOPS\n", t_ms, ops / t_ms / 1e3);+ q->memcpy(matC.data(), C_d, matC.size() * 4).wait();+ buffer_error(ref.data(), matC.data(), ref.size(), 0);+ }++ void ut_testT2(int m, int n, int k) {+ int constexpr runs = 100;+ int constexpr sg_size = 16;+ int constexpr sg_row = 4; // half threads of a XVE+ int constexpr sg_col = 4; // half threads of a XVE+ int constexpr wg_size = sg_size * sg_row * sg_col; // half threads of a XVE+ int constexpr TM = 8;+ int constexpr TN = 16;+ int constexpr TK = 32;+ int constexpr SGM = 4;+ int constexpr SGN = 4;+ int constexpr UnrollK = 2;+ // A large GRF kernel+ // SYCL_PROGRAM_COMPILE_OPTIONS="-ze-opt-large-register-file"+ int constexpr PrefetchDis = 1;+ int constexpr G_NROW = 1;+ int constexpr BM = TM * sg_row * SGM;+ int constexpr BM_STRIDE = BM + 8;+ int constexpr BN = TN * sg_col * SGN;+ int constexpr BN_STRIDE = BN;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ namespace syclex = sycl::ext::oneapi::experimental;+ namespace syclintelex = sycl::ext::intel::experimental;+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case: %d %d %d Device:%s\n", m, n, k, dev->getName().c_str());+ avector<int8_t> matA(m * k), matB(k * n), matBT(k * n), matBT4(k * n / 2);+ avector<int32_t> matC(m * n), ref(m * n);+ fill_buffer_randn(matA.data(), matA.size(), int8_t(-128), int8_t(127));+ fill_buffer_randn(matB.data(), matB.size(), int8_t(-8), int8_t(7));+ gemmref_s8s8s32(m, n, k, matA.data(), matB.data(), ref.data(), k, n, n);+ bestla::kernel::wrapper::Transpose2D<int8_t>::forward_auto(matB.data(), matBT.data(), k, n, n, k);+ for (size_t i = 0; i < n; i++) {+ for (size_t j = 0; j < k; j += 64) {+ for (size_t ik = 0; ik < 32; ik += 1) {+ auto v1 = uint8_t(matBT[i * k + j + ik] + 8);+ auto v2 = uint8_t(matBT[i * k + j + ik + 32] + 8);+ *(uint8_t*)&matBT4[(i * k + j) / 2 + ik] = v1 | (v2 << 4);+ }+ }+ }+ bestla::sycl_vector<int8_t> dA(matA.size(), q), dB(matBT4.size(), q);+ bestla::sycl_vector<int32_t> dC(matC.size(), q);+ q->memcpy(dA.data(), matA.data(), matA.size() * 1).wait();+ q->memcpy(dB.data(), matBT4.data(), matBT4.size() * 1).wait();+ auto A_d = dA.data();+ auto B_d = dB.data();+ auto C_d = dC.data();+ size_t g_ncnt = (n + BN - 1) / BN;+ size_t g_mcnt = (m + BM - 1) / BM;+ size_t wg_repeat = g_mcnt * g_ncnt;+ size_t m_tail = g_mcnt % G_NROW;+ size_t g_m_aligned = g_mcnt - m_tail;+ size_t aligned_wg = g_m_aligned * g_ncnt;++ auto ker = [&](sycl::handler& cgh) {+ // sycl::stream out(65536, 256, cgh);+ sycl::local_accessor<int8_t, 2> slm_b(sycl::range(BN_STRIDE, TK * UnrollK), cgh);+ cgh.parallel_for(+ sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(sg_size)]]+ {+ auto sg = it.get_sub_group();+ int g_id = it.get_group(0);+ int g_n = g_id % g_ncnt;+ int g_m = g_id / g_ncnt;+ if (g_id <= aligned_wg) {+ int g_m_ = g_id % G_NROW;+ g_id /= G_NROW;+ g_n = g_id % g_ncnt;+ g_id /= g_ncnt;+ g_m = g_id * G_NROW + g_m_;+ }+ if (g_n >= g_ncnt || g_m >= g_mcnt) return;++ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;++ int sgId = sg.get_local_id()[0];+ int g_idn = sggid_col * TN * SGN + g_n * BN;+ int g_idm = sggid_row * TM * SGM + g_m * BM;+ // auto pA =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(A_d);+ auto pA = syclex::annotated_ptr{+ A_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::uncached, syclex::cache_level::L3>>}};+ // auto pBl =+ // sycl::address_space_cast<sycl::access::address_space::local_space,+ // sycl::access::decorated::yes>(slm_b);+ auto pB = syclex::annotated_ptr{+ B_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L3>>}};+ // auto pC =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(C_d);+ auto pC = syclex::annotated_ptr{+ C_d, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_through, syclex::cache_level::L1, syclex::cache_level::L3>>}};+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int8_t, use::a, TM, TK,+ layout::row_major>+ sub_a[SGM];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int8_t, use::b, TK, TN,+ layout::col_major>+ sub_b[SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int32_t, use::accumulator, TM, TN>+ sub_c[SGM * SGN];+ if (TK * UnrollK * (PrefetchDis + 1) <= k)+ for (size_t i = 0; i < PrefetchDis; i++) {+ for (int im = sggid_col; im < SGM; im += sg_col)+ if ((g_idm + (im + 1) * TM) <= m)+ joint_matrix_prefetch<TM, TK * UnrollK>(sg, A_d + (g_idm + im * TM) * k * 1 + TK * UnrollK * i, k,+ layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});++ for (int in = sggid_row; in < SGN; in += sg_row)+ if ((g_idn + (in + 1) * TN) <= n)+ joint_matrix_prefetch<TN, TK * UnrollK>(sg, B_d + (g_idn + in * TN) * k / 2 + TK * UnrollK * i / 2,+ k / 2, layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ }++#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) joint_matrix_fill(sg, sub_c[im * SGN + in], 0);++#pragma unroll(1)+ for (size_t ik = 0; ik < k; ik += TK * UnrollK) {+ // #pragma unroll+ // for (int in = 0; in < SGN; in++) {+ // sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ // sg, sub_b[in * UnrollK + 0], pBl, TK * UnrollK, n, k / 2, (g_idn + in * TN) * 1, ik+ // / 2);+ // joint_matrix_copy(sg, sub_b[in * UnrollK + 0], sub_b[in * UnrollK + 1]);+ // joint_matrix_apply(sg, sub_b[in * UnrollK + 0], [=](int8_t& val) {+ // auto t = *(uint8_t*)&val;+ // val = static_cast<int8_t>(t & 0xf) - 8;+ // });+ // joint_matrix_apply(sg, sub_b[in * UnrollK + 1], [=](int8_t& val) {+ // auto t = *(uint8_t*)&val;+ // val = static_cast<int8_t>(t >> 4) - 8;+ // });+ // }+ it.barrier(sycl::access::fence_space::local_space);+#pragma unroll+ for (size_t ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_load(sg, sub_b[in],+ slm_b.get_multi_ptr<sycl::access::decorated::no>() + in * TN * TK + BN * TK * ikk,+ TK);+ // joint_matrix_load(sg, sub_b[in], pB + ik + TK * ikk + (g_idn + in * TN) * k * 1, k);+ // sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ // sg, sub_b[in], pB, k, n, k, (g_idn + in * TN) * 1, ik + TK * ikk);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_a[im], pA, k, m, k, (g_idm + im * TM) * 1, ik + TK * ikk);+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_mad(sg, sub_c[im * SGN + in], sub_a[im], sub_b[in * UnrollK + ikk],+ sub_c[im * SGN + in]);+ }+ }+ it.barrier(sycl::access::fence_space::local_space);+ }+ // if (ik + TK * UnrollK * (PrefetchDis + 1) <= k) {+ // for (int im = sggid_col; im < SGM; im += sg_col)+ // if ((g_idm + (im + 1) * TM) <= m)+ // joint_matrix_prefetch<TM, TK * UnrollK>(+ // sg, A_d + (g_idm + im * TM) * k * 1 + ik + TK * UnrollK * PrefetchDis, k, layout::row_major,+ // syclex::properties{syclex::prefetch_hint_L1});+ // for (int in = sggid_row; in < SGN; in += sg_row)+ // if ((g_idn + (in + 1) * TN) <= n)+ // joint_matrix_prefetch<TN, TK * UnrollK>(+ // sg, B_d + (g_idn + in * TN) * k / 2 + (ik + TK * UnrollK * PrefetchDis) / 2, k / 2,+ // layout::row_major, syclex::properties{syclex::prefetch_hint_L1});+ // }+ }+ // joint_matrix_apply(sg, sub_c, [=](Tc& x) { x *= ALPHA; });+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++)+ // joint_matrix_store(sg, sub_c[im * SGN + in], pC + (g_idm + im * TM) * n + g_idn + in * TN, n,+ // layout::row_major);+ sycl::ext::intel::experimental::matrix::joint_matrix_store_checked(+ sg, sub_c[im * SGN + in], pC, n, layout::row_major, m, n, g_idm + im * TM, g_idn + in * TN);+ }); // parallel for+ };+ q->wait();+ for (size_t i = 0; i < runs; i++) {+ q->submit(ker);+ }+ q->wait();+ bestla::utils::timer<bestla::utils::microseconds> tm;+ tm.start();+ for (size_t i = 0; i < runs; i++) {+ q->submit(ker);+ }+ q->wait();+ auto t_ms = tm.stop() / runs;+ double ops = (double)m * n * k * 2;+ printf("Time %f us, %f GFLOPS\n", t_ms, ops / t_ms / 1e3);+ q->memcpy(matC.data(), C_d, matC.size() * 4).wait();+ buffer_error(ref.data(), matC.data(), ref.size(), 0);+ }+};+#ifdef BTLA_UT_SYCL+static UT_SyclS4IGemm sUT_SyclS4IGemm;+#endif++class UT_SyclIGemm {+ public:+ UT_SyclIGemm() {+ UT_START();+ ut_testT1(123, 128, 128);+ ut_dq<float>(123, 128, 128);+ ut_dq<float>(4096, 4096, 4096);+ ut_dq<utils::fp16, sycl::half>(4096, 4096, 4096);+ ut_dq<utils::bf16, sycl::ext::oneapi::bfloat16>(4096, 4096, 4096);+ ut_testT1(233, 256, 768);+ ut_testT1(4096, 4096, 4096);++ // ut_testT1(552, 552, 96);+ // ut_testT1(4096, 4096, 4096);+ // ut_testT1(4096, 11008, 4096);+ // ut_testT1(4096, 16384, 4096);+ }++ void ut_testT1(int m, int n, int k) {+ int constexpr runs = 100;+ int constexpr sg_size = 16;+ int constexpr sg_row = 8; // half threads of a XVE+ int constexpr sg_col = 4; // half threads of a XVE+ int constexpr wg_size = sg_size * sg_row * sg_col; // half threads of a XVE+ int constexpr TM = 8;+ int constexpr TN = 16;+ int constexpr TK = 32;+ int constexpr SGM = 4;+ int constexpr SGN = 4;+ int constexpr UnrollK = 2;+ // A large GRF kernel+ // SYCL_PROGRAM_COMPILE_OPTIONS="-ze-opt-large-register-file"+ int constexpr PrefetchDis = 3;+ int constexpr G_NROW = 3;+ int constexpr G_NCOL = -1;+ int constexpr BM = TM * sg_row * SGM;+ int constexpr BM_STRIDE = BM + 8;+ int constexpr BN = TN * sg_col * SGN;+ int constexpr BN_STRIDE = BN;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ namespace syclex = sycl::ext::oneapi::experimental;+ namespace syclintelex = sycl::ext::intel::experimental;+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case: %d %d %d Device:%s\n", m, n, k, dev->getName().c_str());+ avector<int8_t> matA(m * k), matB(k * n), matBT(k * n);+ avector<int32_t> matC(m * n), ref(m * n);+ fill_buffer_randn(matA.data(), matA.size(), int8_t(-128), int8_t(127));+ fill_buffer_randn(matB.data(), matB.size(), int8_t(-128), int8_t(127));+ gemmref_s8s8s32(m, n, k, matA.data(), matB.data(), ref.data(), k, n, n);+ bestla::kernel::wrapper::Transpose2D<int8_t>::forward_auto(matB.data(), matBT.data(), k, n, n, k);+ bestla::sycl_vector<int8_t> dA(matA.size(), q), dB(matB.size(), q);+ bestla::sycl_vector<int32_t> dC(matC.size(), q);+ q->memcpy(dA.data(), matA.data(), matA.size() * 1).wait();+ q->memcpy(dB.data(), matBT.data(), matBT.size() * 1).wait();+ auto A_d = dA.data();+ auto B_d = dB.data();+ auto C_d = dC.data();+ size_t g_ncnt = (n + BN - 1) / BN;+ size_t g_mcnt = (m + BM - 1) / BM;+ size_t wg_repeat = g_mcnt * g_ncnt;+ size_t m_tail = g_mcnt % G_NROW;+ size_t g_m_aligned = g_mcnt - m_tail;+ size_t aligned_wg = g_m_aligned * g_ncnt;++ auto ker = [&](sycl::handler& cgh) {+ // sycl::stream out(65536, 256, cgh);+ // sycl::local_accessor<int8_t, 2> slm_b(sycl::range(BN_STRIDE, TK * UnrollK), cgh);+ cgh.parallel_for(+ sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(sg_size)]]+ {+ auto sg = it.get_sub_group();+ int g_id = it.get_group(0);+ int g_n = g_id % g_ncnt;+ int g_m = g_id / g_ncnt;+ if (g_id <= aligned_wg) {+ int g_m_ = g_id % G_NROW;+ g_id /= G_NROW;+ g_n = g_id % g_ncnt;+ g_id /= g_ncnt;+ g_m = g_id * G_NROW + g_m_;+ }+ if (g_n >= g_ncnt || g_m >= g_mcnt) return;++ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;++ int sgId = sg.get_local_id()[0];+ int g_idn = sggid_col * TN * SGN + g_n * BN;+ int g_idm = sggid_row * TM * SGM + g_m * BM;+ // auto pA =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(A_d);+ auto pA = syclex::annotated_ptr{+ A_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L3>>}};+ // auto pB =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(B_d);+ auto pB = syclex::annotated_ptr{+ B_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};+ // auto pC =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(C_d);+ auto pC = syclex::annotated_ptr{+ C_d, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_through, syclex::cache_level::L1, syclex::cache_level::L3>>}};+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int8_t, use::a, TM, TK,+ layout::row_major>+ sub_a[SGM];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int8_t, use::b, TK, TN,+ layout::col_major>+ sub_b[SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, int32_t, use::accumulator, TM, TN>+ sub_c[SGM * SGN];+ if (TK * UnrollK * (PrefetchDis + 1) <= k)+ for (size_t i = 0; i < PrefetchDis; i++) {+ for (int im = sggid_col; im < SGM; im += sg_col)+ if ((g_idm + (im + 1) * TM) <= m)+ joint_matrix_prefetch<TM, TK * UnrollK>(sg, A_d + (g_idm + im * TM) * k * 1 + TK * UnrollK * i, k,+ layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});++ for (int in = sggid_row; in < SGN; in += sg_row)+ if ((g_idn + (in + 1) * TN) <= n)+ joint_matrix_prefetch<TN, TK * UnrollK>(sg, B_d + (g_idn + in * TN) * k * 1 + TK * UnrollK * i, k,+ layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ }++#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) joint_matrix_fill(sg, sub_c[im * SGN + in], 0);++#pragma unroll(1)+ for (size_t ik = 0; ik < k; ik += TK * UnrollK) {+#pragma unroll+ for (size_t ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ // joint_matrix_load(sg, sub_b[in], slm_b.get_multi_ptr<sycl::access::decorated::no>(), TK);+ // joint_matrix_load(sg, sub_b[in], pB + ik + TK * ikk + (g_idn + in * TN) * k * 1, k);+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_b[in], pB, k, n, k, (g_idn + in * TN) * 1, ik + TK * ikk);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ // joint_matrix_load(sg, sub_a[im], pA + (g_idm + im * TM) * k * 1 + ik + TK * ikk, k);+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(+ sg, sub_a[im], pA, k, m, k, (g_idm + im * TM) * 1, ik + TK * ikk);+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_mad(sg, sub_c[im * SGN + in], sub_a[im], sub_b[in], sub_c[im * SGN + in]);+ }+ }+ }+ if (ik + TK * UnrollK * (PrefetchDis + 1) <= k) {+ for (int im = sggid_col; im < SGM; im += sg_col)+ if ((g_idm + (im + 1) * TM) <= m)+ joint_matrix_prefetch<TM, TK * UnrollK>(+ sg, A_d + (g_idm + im * TM) * k * 1 + ik + TK * UnrollK * PrefetchDis, k, layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ for (int in = sggid_row; in < SGN; in += sg_row)+ if ((g_idn + (in + 1) * TN) <= n)+ joint_matrix_prefetch<TN, TK * UnrollK>(+ sg, B_d + (g_idn + in * TN) * k * 1 + ik + TK * UnrollK * PrefetchDis, k, layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ }+ }+ // joint_matrix_apply(sg, sub_c, [=](Tc& x) { x *= ALPHA; });+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++)+ // joint_matrix_store(sg, sub_c[im * SGN + in], pC + (g_idm + im * TM) * n + g_idn + in * TN, n,+ // layout::row_major);+ sycl::ext::intel::experimental::matrix::joint_matrix_store_checked(+ sg, sub_c[im * SGN + in], pC, n, layout::row_major, m, n, g_idm + im * TM, g_idn + in * TN);+ }); // parallel for+ };+ q->wait();+ for (size_t i = 0; i < runs; i++) {+ // q->submit(ker);+ Launcher<xmx::IGemmCfg, xmx::GemmCore>::run(q, {A_d, B_d, C_d, m, n, k, k, k, n});+ }+ q->wait();+ bestla::utils::timer<bestla::utils::microseconds> tm;+ tm.start();+ for (size_t i = 0; i < runs; i++) {+ // q->submit(ker);+ Launcher<xmx::IGemmCfg, xmx::GemmCore>::run(q, {A_d, B_d, C_d, m, n, k, k, k, n});+ }+ q->wait();+ auto t_ms = tm.stop() / runs;+ double ops = (double)m * n * k * 2;+ printf("Time %f us, %f GFLOPS\n", t_ms, ops / t_ms / 1e3);+ q->memcpy(matC.data(), C_d, matC.size() * 4).wait();+ buffer_error(ref.data(), matC.data(), ref.size(), 0);+ }++ template <typename T, typename ST = T>+ void ut_dq(int m, int n, int k) {+ int constexpr runs = 100;+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case: %d %d %d Device:%s\n", m, n, k, dev->getName().c_str());+ avector<int8_t> matA(m * k), matB(k * n), matBT(k * n);+ avector<int32_t> ref(m * n);+ avector<T> matC(m * n), ref0(m * n), scaleA(m), scaleB(n), bias(n);+ fill_buffer_randn(matA.data(), matA.size(), int8_t(-128), int8_t(127));+ fill_buffer_randn(matB.data(), matB.size(), int8_t(-128), int8_t(127));+ fill_buffer_randn(scaleA.data(), scaleA.size(), T(0.001), T(0.004));+ fill_buffer_randn(scaleB.data(), scaleB.size(), T(0.001), T(0.004));+ fill_buffer_randn(bias.data(), bias.size(), T(0.1), T(0.3));+ gemmref_s8s8s32(m, n, k, matA.data(), matB.data(), ref.data(), k, n, n);+ for (size_t i = 0; i < m; i++) {+ for (size_t j = 0; j < n; j++) {+ ref0[i * n + j] = (float)ref[i * n + j] * (float)scaleA[i] * (float)scaleB[j] + (float)bias[j];+ }+ }++ bestla::kernel::wrapper::Transpose2D<int8_t>::forward_auto(matB.data(), matBT.data(), k, n, n, k);+ bestla::sycl_vector<int8_t> dA(matA.size(), q), dB(matB.size(), q);+ bestla::sycl_vector<T> dscaleA(m, q), dscaleB(n, q), dbias(n, q), dC(matC.size(), q);+ q->memcpy(dA.data(), matA.data(), matA.size() * 1).wait();+ q->memcpy(dB.data(), matBT.data(), matBT.size() * 1).wait();+ q->memcpy(dscaleA.data(), scaleA.data(), scaleA.size() * sizeof(T)).wait();+ q->memcpy(dscaleB.data(), scaleB.data(), scaleB.size() * sizeof(T)).wait();+ q->memcpy(dbias.data(), bias.data(), bias.size() * sizeof(T)).wait();+ auto A_d = dA.data();+ auto B_d = dB.data();+ auto C_d = dC.data();++ q->wait();+ for (size_t i = 0; i < runs; i++) {+ Launcher<xmx::IGemmDQCfg<ST>, xmx::IGemmDQCore>::run(+ q, {A_d, B_d, C_d, m, n, k, k, k, n, dbias.data(), dscaleA.data(), dscaleB.data()});+ }+ q->wait();+ bestla::utils::timer<bestla::utils::microseconds> tm;+ tm.start();+ for (size_t i = 0; i < runs; i++) {+ Launcher<xmx::IGemmDQCfg<ST>, xmx::IGemmDQCore>::run(+ q, {A_d, B_d, C_d, m, n, k, k, k, n, dbias.data(), dscaleA.data(), dscaleB.data()});+ }+ q->wait();+ auto t_ms = tm.stop() / runs;+ double ops = (double)m * n * k * 2;+ printf("Time %f us, %f GFLOPS\n", t_ms, ops / t_ms / 1e3);+ q->memcpy(matC.data(), C_d, matC.size() * sizeof(T)).wait();+ buffer_error(ref0.data(), matC.data(), ref0.size(), T(0.2));+ }+};+#ifdef BTLA_UT_SYCL+static UT_SyclIGemm sUT_SyclIGemm;+#endif++class UT_SyclHGemm {+ public:+ UT_SyclHGemm() {+ UT_START();+ ut_test(256, 128, 2048);+ // ut_testT(256, 256, 1024 * 4);+ // ut_testT1(552, 552, 96);+ // ut_testT1(1, 1024, 768);+ // ut_testT1(300, 1024, 768);++ // ut_testT1(4096, 4096, 4096);+ // ut_fp32(4096, 4096, 4096);+ // ut_testT1(4096, 11008, 4096);+ // ut_testT1(4096, 16384, 4096);+ }+ using DT = sycl::half;+ void ut_test(int m, int n, int k) {+ int constexpr runs = 100;+ int constexpr sg_size = 16;+ int constexpr sg_row = 8; // half threads of a XVE+ int constexpr sg_col = 4; // half threads of a XVE+ int constexpr wg_size = sg_size * sg_row * sg_col; // half threads of a XVE+ int constexpr TM = 16;+ int constexpr TN = 16;+ int constexpr TK = 16;+ int constexpr SGM = 2;+ int constexpr SGN = 2;+ int constexpr UnrollK = 2;+ int constexpr wg_repeat = 20 * 16;+ int constexpr BM = TM * sg_row * SGM;+ int constexpr BM_STRIDE = BM + 8;+ int constexpr BN = TN * sg_col * SGN;+ int constexpr BN_STRIDE = BN + 8;+ m = BM;+ n = BN;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ namespace syclex = sycl::ext::oneapi::experimental;+ namespace syclintelex = sycl::ext::intel::experimental;+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case: %d %d %d Device:%s\n", m, n, k, dev->getName().c_str());+ avector<utils::fp16> matA(m * k), matB(k * n), matC(m * n), ref(m * n);+ fill_buffer_randn(matA.data(), matA.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ fill_buffer_randn(matB.data(), matB.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ gemmref_fp16fp16fp16(m, n, k, matA.data(), matB.data(), ref.data(), k, n, n);+ sycl_vector<DT> dA(matA.size(), q), dB(matB.size(), q), dC(matC.size(), q);+ q->memcpy(dA.data(), matA.data(), matA.size() * 2).wait();+ q->memcpy(dB.data(), matB.data(), matB.size() * 2).wait();+ auto A_d = dA.data();+ auto B_d = dB.data();+ auto C_d = dC.data();++ auto ker = [&](sycl::handler& cgh) {+ sycl::local_accessor<float, 1> slm_a(sycl::range(BM_STRIDE * TK), cgh);+ sycl::local_accessor<float, 1> slm_b(sycl::range(BN_STRIDE * TK), cgh);+ cgh.parallel_for(+ sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(sg_size)]]+ {+ auto sg = it.get_sub_group();+ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;+ int sgId = sg.get_local_id()[0];+ int g_idn = sggid_col * TN * SGN;+ int g_idm = sggid_row * TM * SGM;+ // auto pA =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::no>(A_d);+ // auto pB =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::no>(B_d);+ auto pC =+ sycl::address_space_cast<sycl::access::address_space::global_space, sycl::access::decorated::no>(C_d);+ auto pA = syclex::annotated_ptr{+ A_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};+ auto pB = syclex::annotated_ptr{+ B_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::a, TM, TK,+ layout::row_major>+ sub_a[SGM];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::b, TK, TN,+ layout::row_major>+ sub_b[SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::accumulator, TM, TN>+ sub_c[SGM * SGN];+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) joint_matrix_fill(sg, sub_c[im * SGN + in], DT(0.f));+#pragma unroll(1)+ for (size_t ik = 0; ik < k; ik += TK * UnrollK) {+#pragma unroll+ for (size_t ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_load(sg, sub_b[in], pB + (ik + TK * ikk) * n + g_idn + in * TN, n);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ joint_matrix_load(sg, sub_a[im], pA + (g_idm + im * TM) * k + ik + TK * ikk, k);+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_mad(sg, sub_c[im * SGN + in], sub_a[im], sub_b[in], sub_c[im * SGN + in]);+ }+ }+ }+ }+ // joint_matrix_apply(sg, sub_c, [=](Tc& x) { x *= ALPHA; });+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++)+ joint_matrix_store(sg, sub_c[im * SGN + in], pC + (g_idm + im * TM) * n + g_idn + in * TN, n,+ layout::row_major);+ }); // parallel for+ };+ q->wait();+ for (size_t i = 0; i < runs; i++) {+ q->submit(ker);+ }+ q->wait();+ bestla::utils::timer<bestla::utils::microseconds> tm;+ tm.start();+ for (size_t i = 0; i < runs; i++) {+ q->submit(ker);+ }+ q->wait();+ auto t_ms = tm.stop() / runs;+ double ops = (double)m * n * k * 2 * wg_repeat;+ printf("Time %f ms, %f GFLOPS\n", t_ms, ops / t_ms / 1e3);+ q->memcpy(matC.data(), C_d, matC.size() * 2).wait();+ buffer_error(ref.data(), matC.data(), ref.size(), utils::fp16(0.2f));+ }++ template <typename CFG>+ struct Prefetcher {+ using DT = CFG::DT;+ static int constexpr SGM = CFG::SGM;+ static int constexpr SGN = CFG::SGN;+ static int constexpr TM = CFG::TM;+ static int constexpr TN = CFG::TN;+ static int constexpr TK = CFG::TK;+ static int constexpr UnrollK = CFG::UnrollK;+ static int constexpr PrefetchDis = CFG::PrefetchDis;++ static inline void next(sycl::sub_group sg, int ik, int m, int n, int k, int g_idm, int g_idn, int sggid_col,+ int sg_col, int sggid_row, int sg_row, int lda, int ldb, const DT* Awg_d, const DT* Bwg_d) {+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ if (ik + TK * UnrollK * (PrefetchDis + 1) <= k) {+ for (int im = sggid_col; im < SGM; im += sg_col)+ if ((g_idm + im * TM) <= m)+ joint_matrix_prefetch<TM, TK * UnrollK>(sg, Awg_d + (im * TM) * lda + ik + TK * UnrollK * PrefetchDis, lda,+ layout::row_major, syclex::properties{syclex::prefetch_hint_L1});+ for (int in = sggid_row; in < SGN; in += sg_row)+ if ((g_idn + in * TN) <= n)+ joint_matrix_prefetch<TN, TK * UnrollK>(sg, Bwg_d + (in * TN) * ldb + ik + TK * UnrollK * PrefetchDis, ldb,+ layout::row_major, syclex::properties{syclex::prefetch_hint_L1});+ }+ }+ };++ template <typename TP, typename CFG>+ struct KernelFunctor {+ using DT = CFG::DT;+ using DT1 = CFG::DT_ACC;++ static int constexpr sg_size = CFG::sg_size;+ static int constexpr sg_row = CFG::sg_row;+ static int constexpr sg_col = CFG::sg_col;+ static int constexpr TM = CFG::TM;+ static int constexpr TN = CFG::TN;+ static int constexpr TK = CFG::TK;+ static int constexpr SGM = CFG::SGM;+ static int constexpr SGN = CFG::SGN;+ static int constexpr UnrollK = CFG::UnrollK;+ static int constexpr PrefetchDis = CFG::PrefetchDis;+ static int constexpr G_NROW = CFG::G_NROW;++ static int constexpr wg_size = sg_size * sg_row * sg_col;+ static int constexpr BM = TM * sg_row * SGM;+ static int constexpr BM_STRIDE = BM;+ static int constexpr BN = TN * sg_col * SGN;+ static int constexpr BN_STRIDE = BN;++ DT *A_d, *B_d;+ DT1* C_d;+ int m, n, k, lda, ldb, ldc;+ TP mProp;++ int g_ncnt, g_mcnt;+ size_t wg_repeat, aligned_wg;+ KernelFunctor(TP Prop, CFG _cfg, DT* A, DT* B, DT* C, int _m, int _n, int _k, int _lda = 0, int _ldb = 0,+ int _ldc = 0)+ : mProp(Prop) {+ A_d = A;+ B_d = B;+ C_d = C;+ m = _m;+ n = _n;+ k = _k;+ lda = _lda ? _lda : k;+ ldb = _ldb ? _ldb : k;+ ldc = _ldc ? _ldc : n;+ g_ncnt = (n + BN - 1) / BN;+ g_mcnt = (m + BM - 1) / BM;+ wg_repeat = g_mcnt * g_ncnt;+ size_t m_tail = g_mcnt % G_NROW;+ size_t g_m_aligned = g_mcnt - m_tail;+ aligned_wg = g_m_aligned * g_ncnt;+ }++ sycl::nd_range<2> get_range() { return sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}); }++ void operator() [[sycl::reqd_sub_group_size(sg_size)]] (sycl::nd_item<2> it) const {+ auto sg = it.get_sub_group();+ int g_id = it.get_group(0);+ int g_n = g_id % g_ncnt;+ int g_m = g_id / g_ncnt;+ if (g_id <= aligned_wg) {+ int g_m_ = g_id % G_NROW;+ g_id /= G_NROW;+ g_n = g_id % g_ncnt;+ g_id /= g_ncnt;+ g_m = g_id * G_NROW + g_m_;+ }+ if (g_n >= g_ncnt || g_m >= g_mcnt) return;+ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;+ int sgId = sg.get_local_id()[0];+ int g_idn = sggid_col * TN * SGN + g_n * BN;+ int g_idm = sggid_row * TM * SGM + g_m * BM;+ auto Awg_d = A_d + (size_t)g_idm * lda;+ auto Bwg_d = B_d + (size_t)g_idn * ldb;+ auto Cwg_d = C_d + (size_t)g_idm * ldc + g_idn;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ auto pA = syclex::annotated_ptr{+ Awg_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L3>>}};++ auto pB = syclex::annotated_ptr{+ Bwg_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};++ auto pC = syclex::annotated_ptr{+ Cwg_d, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_through, syclex::cache_level::L1, syclex::cache_level::L3>>}};+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::a, TM, TK, layout::row_major>+ sub_a[SGM];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::b, TK, TN, layout::col_major>+ sub_b[SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT1, use::accumulator, TM, TN>+ sub_c[SGM * SGN];++ Prefetcher<CFG>::next(sg, 0, m, n, k, g_idm, g_idn, sggid_col, sg_col, sggid_row, sg_row, lda, ldb, Awg_d, Bwg_d);++#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) joint_matrix_fill(sg, sub_c[im * SGN + in], DT(0.f));++#pragma unroll(1)+ for (size_t ik = 0; ik < k; ik += TK * UnrollK) {+#pragma unroll+ for (size_t ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(sg, sub_b[in], pB, ldb, n, k, in * TN,+ ik + TK * ikk);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(sg, sub_a[im], pA, lda, m, k, im * TM,+ ik + TK * ikk);+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_mad(sg, sub_c[im * SGN + in], sub_a[im], sub_b[in], sub_c[im * SGN + in]);+ }+ }+ }+ Prefetcher<CFG>::next(sg, ik, m, n, k, g_idm, g_idn, sggid_col, sg_col, sggid_row, sg_row, lda, ldb, Awg_d,+ Bwg_d);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++)+ sycl::ext::intel::experimental::matrix::joint_matrix_store_checked(sg, sub_c[im * SGN + in], pC, ldc,+ layout::row_major, m, n, im * TM, in * TN);+ }++ auto get(syclex::properties_tag) const { return mProp; }+ };++ struct HGemmCfg {+ static int constexpr sg_size = 16;+ static int constexpr sg_row = 8;+ static int constexpr sg_col = 4;+ static int constexpr TM = 16;+ static int constexpr TN = 16;+ static int constexpr TK = 16;+ static int constexpr SGM = 2;+ static int constexpr SGN = 4;+ static int constexpr UnrollK = 2;+ static int constexpr PrefetchDis = 3;+ static int constexpr G_NROW = 3;+ using DT = sycl::half;+ using DT_ACC = sycl::half;+ };+ void ut_testT1(int m, int n, int k) {+ int constexpr runs = 100;+ int constexpr sg_size = 16;+ int constexpr sg_row = 8; // half threads of a XVE+ int constexpr sg_col = 4; // half threads of a XVE+ int constexpr wg_size = sg_size * sg_row * sg_col; // half threads of a XVE+ int constexpr TM = 16;+ int constexpr TN = 16;+ int constexpr TK = 16;+ int constexpr SGM = 2;+ int constexpr SGN = 4;+ int constexpr UnrollK = 2;+ // A large GRF kernel+ // SYCL_PROGRAM_COMPILE_OPTIONS="-ze-opt-large-register-file"+ int constexpr PrefetchDis = 3;+ int constexpr G_NROW = 3;+ int constexpr BM = TM * sg_row * SGM;+ int constexpr BM_STRIDE = BM + 8;+ int constexpr BN = TN * sg_col * SGN;+ int constexpr BN_STRIDE = BN;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case: %d %d %d Device:%s\n", m, n, k, dev->getName().c_str());+ avector<utils::fp16> matA(m * k), matB(k * n), matBT(k * n), matC(m * n), ref(m * n), bias(n);+ fill_buffer_randn(matA.data(), matA.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ fill_buffer_randn(matB.data(), matB.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ fill_buffer_randn(bias.data(), bias.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ gemmref_fp16fp16fp16(m, n, k, matA.data(), matB.data(), ref.data(), k, n, n);+ for (size_t i = 0; i < m; i++) {+ for (size_t j = 0; j < n; j++) {+ ref[i * n + j] = (float)ref[i * n + j] + (float)bias[j];+ }+ }+ bestla::kernel::wrapper::Transpose2D<utils::fp16>::forward_auto(matB.data(), matBT.data(), k, n, n, k);+ bestla::sycl_vector<DT> dA(matA.size(), q), dB(matB.size(), q), dC(matC.size(), q), dBias(bias.size(), q);+ q->memcpy(dA.data(), matA.data(), matA.size() * 2).wait();+ q->memcpy(dB.data(), matBT.data(), matBT.size() * 2).wait();+ q->memcpy(dBias.data(), bias.data(), bias.size() * 2).wait();+ auto A_d = dA.data();+ auto B_d = dB.data();+ auto C_d = dC.data();+ size_t g_ncnt = (n + BN - 1) / BN;+ size_t g_mcnt = (m + BM - 1) / BM;+ size_t wg_repeat = g_mcnt * g_ncnt;+ size_t m_tail = g_mcnt % G_NROW;+ size_t g_m_aligned = g_mcnt - m_tail;+ size_t aligned_wg = g_m_aligned * g_ncnt;+ auto ker = [&](sycl::handler& cgh) {+ // sycl::local_accessor<DT, 2> slm_b(sycl::range(BN_STRIDE, TK * UnrollK), cgh);+ cgh.parallel_for(+ sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(sg_size)]]+ {+ auto sg = it.get_sub_group();+ int g_id = it.get_group(0);+ int g_n = g_id % g_ncnt;+ int g_m = g_id / g_ncnt;+ if (g_id <= aligned_wg) {+ int g_m_ = g_id % G_NROW;+ g_id /= G_NROW;+ g_n = g_id % g_ncnt;+ g_id /= g_ncnt;+ g_m = g_id * G_NROW + g_m_;+ }+ if (g_n >= g_ncnt || g_m >= g_mcnt) return;+ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;+ int sgId = sg.get_local_id()[0];+ int g_idn = sggid_col * TN * SGN + g_n * BN;+ int g_idm = sggid_row * TM * SGM + g_m * BM;+ // auto pA =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(A_d);+ auto pA = syclex::annotated_ptr{+ A_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L3>>}};+ // auto pB =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(B_d);+ auto pB = syclex::annotated_ptr{+ B_d, syclex::properties{syclintelex::read_hint<+ syclintelex::cache_control<syclintelex::cache_mode::cached, syclex::cache_level::L1>,+ syclintelex::cache_control<syclintelex::cache_mode::streaming, syclex::cache_level::L3>>}};+ // auto pC =+ // sycl::address_space_cast<sycl::access::address_space::global_space,+ // sycl::access::decorated::yes>(C_d);+ auto pC = syclex::annotated_ptr{+ C_d, syclex::properties{syclintelex::write_hint<syclintelex::cache_control<+ syclintelex::cache_mode::write_through, syclex::cache_level::L1, syclex::cache_level::L3>>}};+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::a, TM, TK,+ layout::row_major>+ sub_a[SGM];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::b, TK, TN,+ layout::col_major>+ sub_b[SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::accumulator, TM, TN>+ sub_c[SGM * SGN];+ if (TK * UnrollK * (PrefetchDis + 1) <= k)+ for (size_t i = 0; i < PrefetchDis; i++) {+ for (int im = sggid_col; im < SGM; im += sg_col)+ if ((g_idm + im * TM) <= m)+ joint_matrix_prefetch<TM, TK * UnrollK>(sg, A_d + (g_idm + im * TM) * k * 1 + TK * UnrollK * i, k,+ layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ for (int in = sggid_row; in < SGN; in += sg_row)+ if ((g_idn + in * TN) <= n)+ joint_matrix_prefetch<TN, TK * UnrollK>(sg, B_d + (g_idn + in * TN) * k * 1 + TK * UnrollK * i, k,+ layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ }++#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) joint_matrix_fill(sg, sub_c[im * SGN + in], DT(0.f));++#pragma unroll(1)+ for (size_t ik = 0; ik < k; ik += TK * UnrollK) {+#pragma unroll+ for (size_t ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ // joint_matrix_load(sg, sub_b[in], slm_b.get_multi_ptr<sycl::access::decorated::no>(), TK);+ // joint_matrix_load(sg, sub_b[in], pB + ik + TK * ikk + (g_idn + in * TN) * k * 1, k);+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(sg, sub_b[in], pB, k, n, k,+ g_idn + in * TN, ik + TK * ikk);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ // joint_matrix_load(sg, sub_a[im], pA + (g_idm + im * TM) * k * 1 + ik + TK * ikk, k);+ sycl::ext::intel::experimental::matrix::joint_matrix_load_checked(sg, sub_a[im], pA, k, m, k,+ g_idm + im * TM, ik + TK * ikk);+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_mad(sg, sub_c[im * SGN + in], sub_a[im], sub_b[in], sub_c[im * SGN + in]);+ }+ }+ }+ if (ik + TK * UnrollK * (PrefetchDis + 1) <= k) {+ for (int im = sggid_col; im < SGM; im += sg_col)+ if ((g_idm + im * TM) <= m)+ joint_matrix_prefetch<TM, TK * UnrollK>(+ sg, A_d + (g_idm + im * TM) * k * 1 + ik + TK * UnrollK * PrefetchDis, k, layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ for (int in = sggid_row; in < SGN; in += sg_row)+ if ((g_idn + in * TN) <= n)+ joint_matrix_prefetch<TN, TK * UnrollK>(+ sg, B_d + (g_idn + in * TN) * k * 1 + ik + TK * UnrollK * PrefetchDis, k, layout::row_major,+ syclex::properties{syclex::prefetch_hint_L1});+ }+ }+ // joint_matrix_apply(sg, sub_c, [=](Tc& x) { x *= ALPHA; });+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++)+ // joint_matrix_store(sg, sub_c[im * SGN + in], pC + (g_idm + im * TM) * n + g_idn + in * TN, n,+ // layout::row_major);+ sycl::ext::intel::experimental::matrix::joint_matrix_store_checked(+ sg, sub_c[im * SGN + in], pC, n, layout::row_major, m, n, g_idm + im * TM, g_idn + in * TN);+ }); // parallel for+ };+ auto ker1 = [&](sycl::handler& cgh) {+ syclex::properties prop{syclintelex::grf_size<256>};+ KernelFunctor largeker(prop, HGemmCfg(), A_d, B_d, C_d, m, n, k);+ cgh.parallel_for(largeker.get_range(), largeker);+ };+ using CFG = xmx::HGemmCfg;+ q->wait();+ for (size_t i = 0; i < runs; i++) {+ // q->submit(ker1);+ Launcher<CFG, xmx::GemmCore>::run(q, {A_d, B_d, C_d, m, n, k, k, k, n, dBias.data()});+ }+ q->wait();+ bestla::utils::timer<bestla::utils::microseconds> tm;+ tm.start();+ for (size_t i = 0; i < runs; i++) {+ // q->submit(ker1);+ Launcher<CFG, xmx::GemmCore>::run(q, {A_d, B_d, C_d, m, n, k, k, k, n, dBias.data()});+ }+ q->wait();+ auto t_ms = tm.stop() / runs;+ double ops = (double)m * n * k * 2;+ printf("Time %f us, %f GFLOPS\n", t_ms, ops / t_ms / 1e3);+ q->memcpy(matC.data(), C_d, matC.size() * 2).wait();+ buffer_error(ref.data(), matC.data(), ref.size(), utils::fp16(0.3f));+ }++ void ut_fp32(int m, int n, int k) {+ int constexpr runs = 100;+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case: %d %d %d Device:%s\n", m, n, k, dev->getName().c_str());+ avector<utils::fp16> matA(m * k), matB(k * n), matBT(k * n);+ avector<float> matC(m * n), ref(m * n), bias(n);+ fill_buffer_randn(matA.data(), matA.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ fill_buffer_randn(matB.data(), matB.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ fill_buffer_randn(bias.data(), bias.size(), -0.5f, 0.5f);+ gemmref_fp16fp16fp32(m, n, k, matA.data(), matB.data(), ref.data(), k, n, n);+ for (size_t i = 0; i < m; i++) {+ for (size_t j = 0; j < n; j++) {+ ref[i * n + j] += bias[j];+ }+ }++ bestla::kernel::wrapper::Transpose2D<utils::fp16>::forward_auto(matB.data(), matBT.data(), k, n, n, k);+ bestla::sycl_vector<DT> dA(matA.size(), q), dB(matB.size(), q);+ sycl_vector<float> dC(matC.size(), q), dBias(bias.size(), q);+ q->memcpy(dA.data(), matA.data(), matA.size() * 2).wait();+ q->memcpy(dB.data(), matBT.data(), matBT.size() * 2).wait();+ q->memcpy(dBias.data(), bias.data(), bias.size() * 4).wait();+ auto A_d = dA.data();+ auto B_d = dB.data();+ auto C_d = dC.data();++ using CFG = xmx::HGemmAfp32Cfg;+ q->wait();+ for (size_t i = 0; i < runs; i++) {+ // q->submit(ker1);+ Launcher<CFG, xmx::GemmCore>::run(q, {A_d, B_d, C_d, m, n, k, k, k, n, dBias.data()});+ }+ q->wait();+ bestla::utils::timer<bestla::utils::microseconds> tm;+ tm.start();+ for (size_t i = 0; i < runs; i++) {+ // q->submit(ker1);+ Launcher<CFG, xmx::GemmCore>::run(q, {A_d, B_d, C_d, m, n, k, k, k, n, dBias.data()});+ }+ q->wait();+ auto t_ms = tm.stop() / runs;+ double ops = (double)m * n * k * 2;+ printf("Time %f us, %f GFLOPS\n", t_ms, ops / t_ms / 1e3);+ q->memcpy(matC.data(), C_d, matC.size() * 4).wait();+ buffer_error(ref.data(), matC.data(), ref.size(), float(0.3f));+ }++ void ut_testT(int m, int n, int k) {+ int constexpr runs = 100;+ int constexpr sg_size = 16;+ int constexpr sg_row = 8; // half threads of a XVE+ int constexpr sg_col = 4; // half threads of a XVE+ int constexpr wg_size = sg_size * sg_row * sg_col; // half threads of a XVE+ int constexpr TM = 16;+ int constexpr TN = 16;+ int constexpr TK = 16;+ int constexpr SGM = 2;+ int constexpr SGN = 2;+ int constexpr UnrollK = 2;+ int constexpr wg_repeat = 20 * 32;+ int constexpr BM = TM * sg_row * SGM;+ int constexpr BM_STRIDE = BM + 8;+ int constexpr BN = TN * sg_col * SGN;+ int constexpr BN_STRIDE = BN + 8;+ m = BM;+ n = BN;+ using sycl::ext::oneapi::experimental::matrix::layout;+ using sycl::ext::oneapi::experimental::matrix::use;+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case: %d %d %d Device:%s\n", m, n, k, dev->getName().c_str());+ avector<utils::fp16> matA(m * k), matB(k * n), matBT(k * n), matC(m * n), ref(m * n);+ fill_buffer_randn(matA.data(), matA.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ fill_buffer_randn(matB.data(), matB.size(), utils::fp16(-0.5f), utils::fp16(0.5f));+ gemmref_fp16fp16fp16(m, n, k, matA.data(), matB.data(), ref.data(), k, n, n);+ bestla::kernel::wrapper::Transpose2D<utils::fp16>::forward_auto(matB.data(), matBT.data(), k, n, n, k);+ bestla::sycl_vector<DT> dA(matA.size(), q), dB(matB.size(), q), dC(matC.size(), q);+ q->memcpy(dA.data(), matA.data(), matA.size() * 2).wait();+ q->memcpy(dB.data(), matBT.data(), matBT.size() * 2).wait();+ auto A_d = dA.data();+ auto B_d = dB.data();+ auto C_d = dC.data();++ auto ker = [&](sycl::handler& cgh) {+ sycl::local_accessor<float, 1> slm_a(sycl::range(BM_STRIDE * TK), cgh);+ sycl::local_accessor<float, 1> slm_b(sycl::range(BN_STRIDE * TK), cgh);+ cgh.parallel_for(+ sycl::nd_range<2>({wg_repeat, wg_size}, {1, wg_size}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(sg_size)]]+ {+ auto sg = it.get_sub_group();+ int sgSize = sg.get_local_range()[0];+ int sgGroupId = sg.get_group_id()[0];+ int sggid_col = sgGroupId % sg_col;+ int sggid_row = sgGroupId / sg_col;+ int sgId = sg.get_local_id()[0];+ int g_idn = sggid_col * TN * SGN;+ int g_idm = sggid_row * TM * SGM;+ auto pA =+ sycl::address_space_cast<sycl::access::address_space::global_space, sycl::access::decorated::no>(A_d);+ auto pB =+ sycl::address_space_cast<sycl::access::address_space::global_space, sycl::access::decorated::no>(B_d);+ auto pC =+ sycl::address_space_cast<sycl::access::address_space::global_space, sycl::access::decorated::no>(C_d);+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::a, TM, TK,+ layout::row_major>+ sub_a[SGM];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::b, TK, TN,+ layout::col_major>+ sub_b[SGN];+ sycl::ext::oneapi::experimental::matrix::joint_matrix<sycl::sub_group, DT, use::accumulator, TM, TN>+ sub_c[SGM * SGN];+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++) joint_matrix_fill(sg, sub_c[im * SGN + in], DT(0.f));+#pragma unroll(1)+ for (size_t ik = 0; ik < k; ik += TK * UnrollK) {+#pragma unroll+ for (size_t ikk = 0; ikk < UnrollK; ikk++) {+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_load(sg, sub_b[in], pB + ik + TK * ikk + (g_idn + in * TN) * k, k);+ }+#pragma unroll+ for (int im = 0; im < SGM; im++) {+ joint_matrix_load(sg, sub_a[im], pA + (g_idm + im * TM) * k + ik + TK * ikk, k);+#pragma unroll+ for (int in = 0; in < SGN; in++) {+ joint_matrix_mad(sg, sub_c[im * SGN + in], sub_a[im], sub_b[in], sub_c[im * SGN + in]);+ }+ }+ }+ }+ // joint_matrix_apply(sg, sub_c, [=](Tc& x) { x *= ALPHA; });+#pragma unroll+ for (int im = 0; im < SGM; im++)+#pragma unroll+ for (int in = 0; in < SGN; in++)+ joint_matrix_store(sg, sub_c[im * SGN + in], pC + (g_idm + im * TM) * n + g_idn + in * TN, n,+ layout::row_major);+ }); // parallel for+ };+ q->wait();+ for (size_t i = 0; i < runs; i++) {+ q->submit(ker);+ }+ q->wait();+ bestla::utils::timer<bestla::utils::microseconds> tm;+ tm.start();+ for (size_t i = 0; i < runs; i++) {+ q->submit(ker);+ }+ q->wait();+ auto t_ms = tm.stop() / runs;+ double ops = (double)m * n * k * 2 * wg_repeat;+ printf("Time %f ms, %f GFLOPS\n", t_ms, ops / t_ms / 1e3);+ q->memcpy(matC.data(), C_d, matC.size() * 2).wait();+ buffer_error(ref.data(), matC.data(), ref.size(), utils::fp16(0.2f));+ }+};+#ifdef BTLA_UT_SYCL+#endif+static UT_SyclHGemm sUT_SyclHGemm;++class UT_SyclHGemmBF16 {+ public:+ UT_SyclHGemmBF16() {+ UT_START();+ ut_xmx(1, 1024, 768);+ ut_xmx(300, 1024, 1024);+ ut_xmx(1033, 1024, 1024);+ ut_xmx(4096, 4096, 4096);+ ut_xmx_fp32(4096, 4096, 4096);+ }++ void ut_xmx(int m, int n, int k) {+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case: %d %d %d Device:%s\n", m, n, k, dev->getName().c_str());+ avector<utils::bf16> matA(m * k), matB(k * n), matBT(k * n), matC(m * n), ref(m * n);+ fill_buffer_randn(matA.data(), matA.size(), utils::bf16(-0.5f), utils::bf16(0.5f));+ fill_buffer_randn(matB.data(), matB.size(), utils::bf16(-0.5f), utils::bf16(0.5f));+ gemmref_bf16bf16bf16(m, n, k, matA.data(), matB.data(), ref.data(), k, n, n);+ bestla::kernel::wrapper::Transpose2D<utils::bf16>::forward_auto(matB.data(), matBT.data(), k, n, n, k);++ sycl_vector<sycl::ext::oneapi::bfloat16> dA(matA.size(), q), dB(matB.size(), q), dC(matC.size(), q);+ q->memcpy(dA.data(), matA.data(), matA.size() * 2).wait();+ q->memcpy(dB.data(), matBT.data(), matBT.size() * 2).wait();+ auto A_d = dA.data();+ auto B_d = dB.data();+ auto C_d = dC.data();+ Launcher<xmx::HGemmBf16Cfg, xmx::GemmCore>::run(q, {A_d, B_d, C_d, m, n, k, k, k, n});+ q->memcpy(matC.data(), C_d, matC.size() * 2).wait();+ buffer_error(ref.data(), matC.data(), ref.size(), utils::bf16(2.6f));+ int constexpr runs = 100;+ q->wait();+ for (size_t i = 0; i < runs; i++) {+ Launcher<xmx::HGemmBf16Cfg, xmx::GemmCore>::run(q, {A_d, B_d, C_d, m, n, k, k, k, n});+ }+ q->wait();+ bestla::utils::timer<bestla::utils::microseconds> tm;+ tm.start();+ for (size_t i = 0; i < runs; i++) {+ Launcher<xmx::HGemmBf16Cfg, xmx::GemmCore>::run(q, {A_d, B_d, C_d, m, n, k, k, k, n});+ }+ q->wait();+ auto t_ms = tm.stop() / runs;+ double ops = (double)m * n * k * 2;+ printf("Time %f us, %f GFLOPS\n", t_ms, ops / t_ms / 1e3);+ }++ void ut_xmx_fp32(int m, int n, int k) {+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case: %d %d %d Device:%s\n", m, n, k, dev->getName().c_str());+ avector<utils::bf16> matA(m * k), matB(k * n), matBT(k * n);+ avector<float> matC(m * n), ref(m * n);+ fill_buffer_randn(matA.data(), matA.size(), utils::bf16(-0.5f), utils::bf16(0.5f));+ fill_buffer_randn(matB.data(), matB.size(), utils::bf16(-0.5f), utils::bf16(0.5f));+ gemmref_bf16bf16fp32(m, n, k, matA.data(), matB.data(), ref.data(), k, n, n);+ bestla::kernel::wrapper::Transpose2D<utils::bf16>::forward_auto(matB.data(), matBT.data(), k, n, n, k);++ sycl_vector<sycl::ext::oneapi::bfloat16> dA(matA.size(), q), dB(matB.size(), q);+ sycl_vector<float> dC(matC.size(), q);+ q->memcpy(dA.data(), matA.data(), matA.size() * 2).wait();+ q->memcpy(dB.data(), matBT.data(), matBT.size() * 2).wait();+ auto A_d = dA.data();+ auto B_d = dB.data();+ auto C_d = dC.data();+ Launcher<xmx::HGemmBf16Fp32Cfg, xmx::GemmCore>::run(q, {A_d, B_d, C_d, m, n, k, k, k, n});+ q->memcpy(matC.data(), C_d, matC.size() * 4).wait();+ buffer_error(ref.data(), matC.data(), ref.size(), float(1.6f));+ int constexpr runs = 100;+ q->wait();+ for (size_t i = 0; i < runs; i++) {+ Launcher<xmx::HGemmBf16Fp32Cfg, xmx::GemmCore>::run(q, {A_d, B_d, C_d, m, n, k, k, k, n});+ }+ q->wait();+ bestla::utils::timer<bestla::utils::microseconds> tm;+ tm.start();+ for (size_t i = 0; i < runs; i++) {+ Launcher<xmx::HGemmBf16Fp32Cfg, xmx::GemmCore>::run(q, {A_d, B_d, C_d, m, n, k, k, k, n});+ }+ q->wait();+ auto t_ms = tm.stop() / runs;+ double ops = (double)m * n * k * 2;+ printf("Time %f us, %f GFLOPS\n", t_ms, ops / t_ms / 1e3);+ }+};+#ifdef BTLA_UT_SYCL+static UT_SyclHGemmBF16 sUT_SyclHGemmBF16;+#endif++class UT_SyclInt4Dequant {+ public:+ UT_SyclInt4Dequant() {+ UT_START();+ ut_fp32_T(1024, 384, 32);+ ut_fp32_T(1024, 768, 32);+ ut_fp32_T(1024, 1024, 32);+ }++ void ut_fp32_T(int n, int k, int blocksize) {+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case: %d %d %d Device:%s\n", n, k, blocksize, dev->getName().c_str());+ avector<uint8_t> rawB(k * n / 2);+ int blks = updiv(k, blocksize);+ avector<float> scale(blks * n), dequant(n * k), ref(n * k);+ fill_buffer_randn(scale.data(), scale.size(), 0.01f, 0.03f);+ fill_buffer_randn(rawB.data(), rawB.size(), uint8_t(0), uint8_t(255));+ auto srcptr = (utils::int4x2*)rawB.data();+ for (int i = 0; i < n; i++) {+ for (int j = 0; j < k; j += 2) {+ auto tmp = srcptr[i * k / 2 + j / 2];+ auto noffset = i * blks + j / blocksize;+ ref[i * k + j + 0] = static_cast<float>(static_cast<int8_t>(tmp.x) - 8) * scale[noffset];+ ref[i * k + j + 1] = static_cast<float>(static_cast<int8_t>(tmp.y) - 8) * scale[noffset];+ }+ }+ using ProB = sycl_prologue_b::WeightS4T<float>;+ sycl_vector<float> dS(scale.size(), q), dequantB(n * k, q);+ sycl_vector<uint8_t> dB(rawB.size(), q);+ q->memcpy(dS.data(), scale.data(), scale.size() * 4).wait();+ q->memcpy(dB.data(), rawB.data(), rawB.size() * 1).wait();+ auto S_d = dS.data();+ auto B_d = dB.data();+ auto DB_d = dequantB.data();+ auto ev = ProB::template dequant<ProB::CfgDequantF32>(n, k, blocksize, {B_d, S_d, blks}, DB_d, q);+ ev.wait();+ q->memcpy(dequant.data(), DB_d, dequant.size() * 4).wait();+ buffer_error(ref.data(), dequant.data(), dequant.size(), 0.001f);+ }+};+#ifdef BTLA_UT_SYCL+static UT_SyclInt4Dequant sUT_SyclInt4Dequant;+#endif++class UT_SyclInt8Dequant {+ public:+ UT_SyclInt8Dequant() {+ UT_START();+ ut_fp32_T(1024, 384, 32);+ ut_fp32_T(1024, 768, 32);+ ut_fp32_T(1024, 1024, 32);+ }++ void ut_fp32_T(int n, int k, int blocksize) {+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case: %d %d %d Device:%s\n", n, k, blocksize, dev->getName().c_str());+ avector<int8_t> rawB(k * n);+ int blks = updiv(k, blocksize);+ avector<float> scale(blks * n), dequant(n * k), ref(n * k);+ fill_buffer_randn(scale.data(), scale.size(), 0.01f, 0.03f);+ fill_buffer_randn(rawB.data(), rawB.size(), int8_t(-128), int8_t(127));+ auto srcptr = rawB.data();+ for (int i = 0; i < n; i++) {+ for (int j = 0; j < k; j += 2) {+ auto tmp = *(std::array<int8_t, 2>*)&srcptr[i * k + j];+ auto noffset = i * blks + j / blocksize;+ ref[i * k + j + 0] = tmp[0] * scale[noffset];+ ref[i * k + j + 1] = tmp[1] * scale[noffset];+ }+ }+ using ProB = sycl_prologue_b::WeightS8T<float>;+ sycl_vector<float> dS(scale.size(), q), dequantB(n * k, q);+ sycl_vector<int8_t> dB(rawB.size(), q);+ q->memcpy(dS.data(), scale.data(), scale.size() * 4).wait();+ q->memcpy(dB.data(), rawB.data(), rawB.size() * 1).wait();+ auto S_d = dS.data();+ auto B_d = dB.data();+ auto DB_d = dequantB.data();+ auto ev = ProB::template dequant<ProB::CfgDequantF32>(n, k, blocksize, {B_d, S_d, blks}, DB_d, q);+ ev.wait();+ q->memcpy(dequant.data(), DB_d, dequant.size() * 4).wait();+ buffer_error(ref.data(), dequant.data(), dequant.size(), 0.001f);+ }+};+#ifdef BTLA_UT_SYCL+static UT_SyclInt8Dequant sUT_SyclInt8Dequant;+#endif++class UT_SyclInt8Quant {+ public:+ UT_SyclInt8Quant() {+ UT_START();+ ut_T<float>(4096, 4096, 1);+ ut_T<utils::fp16, sycl::half>(4096, 4096, 1);+ ut_T<utils::bf16, sycl::ext::oneapi::bfloat16>(4096, 4096, 1);+ ut_T<float>(4096, 4096, 0);+ ut_T<utils::fp16, sycl::half>(4096, 4096, 0);+ ut_T<utils::bf16, sycl::ext::oneapi::bfloat16>(4096, 4096, 0);+ // ut_fp32_T(1024, 768, 0);+ // ut_fp32_T(1024, 1024, 0);+ }++ template <typename T, typename SYCLT = T>+ void ut_T(int m, int k, int mask) {+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case: %d %d %d Device:%s\n", m, k, mask, dev->getName().c_str());+ avector<int8_t> raw(k * m), q_dev(k * m);+ avector<T> scale(m), scale_dev(m), ref(m * k);+ fill_buffer_randn(scale.data(), scale.size(), T(0.01f), T(0.03f));+ if (mask == 1) {+ for (size_t i = 1; i < scale.size(); i++) {+ scale[i] = scale[0];+ }+ }+ fill_buffer_randn(raw.data(), raw.size(), int8_t(-128), int8_t(127));+ auto srcptr = raw.data();+ for (int i = 0; i < m; i++) {+ for (int j = 0; j < k; j += 1) {+ ref[i * k + j] = raw[i * k + j] * (float)scale[i];+ }+ }+ using Pro = sycl_prologue_a::ActivationBase<SYCLT>;+ sycl_vector<T> dS(scale.size(), q), dequantB(ref.size(), q);+ sycl_vector<int8_t> dRaw(raw.size(), q);+ q->memcpy(dequantB.data(), ref.data(), ref.size() * sizeof(T)).wait();+ auto ev = Pro::template quant_s8<typename Pro::CfgQuantF32>(m, k, mask, {(SYCLT*)dequantB.data(), k}, dRaw.data(),+ (SYCLT*)dS.data(), q);+ ev.wait();+ q->memcpy(q_dev.data(), dRaw.data(), q_dev.size()).wait();+ q->memcpy(scale_dev.data(), dS.data(), scale_dev.size() * sizeof(T)).wait();+ buffer_error(raw.data(), q_dev.data(), q_dev.size(), (int8_t)2);+ buffer_error(scale.data(), scale_dev.data(), scale_dev.size(), T(0.001));++ int constexpr runs = 1000;+ q->wait();+ for (size_t i = 0; i < runs; i++) {+ Pro::template quant_s8<typename Pro::CfgQuantF32>(m, k, mask, {(SYCLT*)dequantB.data(), k}, dRaw.data(),+ (SYCLT*)dS.data(), q);+ }+ q->wait();+ bestla::utils::timer<bestla::utils::microseconds> tm;+ tm.start();+ for (size_t i = 0; i < runs; i++) {+ Pro::template quant_s8<typename Pro::CfgQuantF32>(m, k, mask, {(SYCLT*)dequantB.data(), k}, dRaw.data(),+ (SYCLT*)dS.data(), q);+ }+ q->wait();+ auto t_ms = tm.stop() / runs;+ double memsize = (double)m * k * (sizeof(T) + 1);+ if (mask) memsize += m * k * 2;+ printf("Time %f us, %f GB/s\n", t_ms, memsize / t_ms / 1e3);+ }+};+#ifdef BTLA_UT_SYCL+static UT_SyclInt8Quant sUT_SyclInt8Quant;+#endif++#if 0+class UT_SyclF8Dequant {+ public:+ UT_SyclF8Dequant() {+ UT_START();+ ut_fp32_T<true>(1024, 384, 64);+ ut_fp32_T<false>(1024, 384, 64);+ }++ template <bool IsE4M3>+ void ut_fp32_T(int n, int k, int blocksize) {+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case FP8 %s dequant: %d %d %d Device:%s\n", IsE4M3 ? "E4M3" : "E5M2", n, k, blocksize,+ dev->getName().c_str());+ avector<uint8_t> rawB(k * n);+ int blks = k / blocksize;+ avector<float> scale(size_t(blks) * n), dequant(n * k), ref(n * k);+ fill_buffer_randn(scale.data(), scale.size(), 0.01f, 0.03f);+ fill_buffer_randn(rawB.data(), rawB.size(), uint8_t(0), uint8_t(255));+ sanitize_fp8_buffer<IsE4M3>(rawB.data(), rawB.size());+ for (int i = 0; i < n; i++) {+ for (int j = 0; j < k; j++) {+ auto noffset = i * blks + j / blocksize;+ ref[i * k + j] = decode_fp8_byte<IsE4M3>(rawB[i * k + j]) * scale[noffset];+ }+ }+ using ProB = sycl_prologue_b::WeightF8T<xve::DefaultSGemmCore, float, IsE4M3>;+ sycl_vector<float> dS(scale.size(), q), dequantB(n * k, q);+ sycl_vector<uint8_t> dB(rawB.size(), q);+ q->memcpy(dS.data(), scale.data(), scale.size() * 4).wait();+ q->memcpy(dB.data(), rawB.data(), rawB.size()).wait();+ auto S_d = dS.data();+ auto B_d = dB.data();+ auto DB_d = dequantB.data();+ auto ev = ProB::template dequant<sycl_prologue_b::KernelConfigTrans>(n, k, blocksize, {B_d, S_d, blks}, DB_d, q);+ ev.wait();+ q->memcpy(dequant.data(), DB_d, dequant.size() * 4).wait();+ buffer_error(ref.data(), dequant.data(), dequant.size(), 0.001f);++ avector<float> refNT(k * n);+ kernel::wrapper::Transpose2D<float>::forward<BTLA_ISA::NoSIMD>(ref.data(), refNT.data(), n, k, k, n);+ ev = ProB::template dequant_T<sycl_prologue_b::KernelConfigTrans>(n, k, blocksize, {B_d, S_d, blks}, DB_d, q);+ ev.wait();+ q->memcpy(dequant.data(), DB_d, dequant.size() * 4).wait();+ buffer_error(refNT.data(), dequant.data(), dequant.size(), 0.001f);+ }+};+#ifdef BTLA_UT_SYCL+static UT_SyclF8Dequant sUT_SyclF8Dequant;+#endif++class UT_SyclS8Gemv {+ public:+ UT_SyclS8Gemv() {+ UT_START();+ ut_T(1024, 11008, 32);+ ut_T(1024, 1024, 32);+ ut_half(1024, 11008, 32);+ ut_half(1024, 1024, 32);+ }++ void ut_T(int n, int k, int blocksize) {+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case %s: %d %d %d Device:%s\n", __FUNCTION__, n, k, blocksize, dev->getName().c_str());+ avector<uint8_t> rawB(k * n);+ int blks = updiv(k, blocksize);+ avector<float> scale(blks * n), C(n), dqB(n * k), A(k), refC(n);+ fill_buffer_randn(scale.data(), scale.size(), 0.01f, 0.03f);+ fill_buffer_randn(A.data(), A.size(), -0.1f, 0.3f);+ fill_buffer_randn(rawB.data(), rawB.size(), uint8_t(0), uint8_t(255));+ auto srcptr = rawB.data();+ for (int i = 0; i < n; i++) {+ for (int j = 0; j < k; j += 2) {+ auto tmp = *(std::array<int8_t, 2>*)&srcptr[i * k + j];+ auto noffset = i * blks + j / blocksize;+ dqB[i + (j + 0) * n] = tmp[0] * scale[noffset];+ dqB[i + (j + 1) * n] = tmp[1] * scale[noffset];+ }+ }+ gemmref_fp32fp32fp32(1, n, k, A.data(), dqB.data(), refC.data(), k, n, n);+ sycl_vector<float> dS(scale.size(), q), dC(n, q), dA(k, q);+ sycl_vector<int8_t> dB(rawB.size(), q);+ q->memcpy(dB.data(), rawB.data(), rawB.size() * 1).wait();+ q->memcpy(dS.data(), scale.data(), scale.size() * 4).wait();+ q->memcpy(dA.data(), A.data(), A.size() * 4).wait();+ auto S_d = dS.data();+ auto A_d = dA.data();+ auto B_d = dB.data();+ auto C_d = dC.data();+ auto ev =+ sycl_prologue_b::WeightS8T<xve::DefaultSGemmCore, float>::gemv(A_d, {B_d, S_d, blks}, C_d, n, k, blocksize, q);+ ev.wait();+ q->memcpy(C.data(), C_d, C.size() * 4).wait();+ buffer_error(refC.data(), C.data(), C.size(), 0.001f);+ }++ void ut_half(int n, int k, int blocksize) {+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case %s: %d %d %d Device:%s\n", __FUNCTION__, n, k, blocksize, dev->getName().c_str());+ avector<uint8_t> rawB(k * n);+ int blks = updiv(k, blocksize);+ avector<utils::fp16> scale(blks * n), C(n), dqB(n * k), A(k), refC(n);+ fill_buffer_randn(scale.data(), scale.size(), utils::fp16(0.01f), utils::fp16(0.03f));+ fill_buffer_randn(A.data(), A.size(), utils::fp16(-0.1f), utils::fp16(0.3f));+ fill_buffer_randn(rawB.data(), rawB.size(), uint8_t(0), uint8_t(255));+ auto srcptr = (utils::int4x2*)rawB.data();+ for (int i = 0; i < n; i++) {+ for (int j = 0; j < k; j += 2) {+ auto tmp = *(std::array<int8_t, 2>*)&srcptr[i * k + j];+ auto noffset = i * blks + j / blocksize;+ dqB[i + (j + 0) * n] = tmp[0] * float(scale[noffset]);+ dqB[i + (j + 1) * n] = tmp[1] * float(scale[noffset]);+ }+ }+ gemmref_fp16fp16fp16(1, n, k, A.data(), dqB.data(), refC.data(), k, n, n);+ sycl_vector<sycl::half> dS(scale.size(), q), dC(n, q), dA(k, q);+ sycl_vector<int8_t> dB(rawB.size(), q);+ q->memcpy(dB.data(), rawB.data(), rawB.size() * 1).wait();+ q->memcpy(dS.data(), scale.data(), scale.size() * 2).wait();+ q->memcpy(dA.data(), A.data(), A.size() * 2).wait();+ auto S_d = dS.data();+ auto A_d = dA.data();+ auto B_d = dB.data();+ auto C_d = dC.data();+ auto ev = sycl_prologue_b::WeightS8T<xve::DefaultHGemmCore, sycl::half>::gemv(A_d, {B_d, S_d, blks}, C_d, n, k,+ blocksize, q);+ ev.wait();+ q->memcpy(C.data(), C_d, C.size() * 2).wait();+ buffer_error(refC.data(), C.data(), C.size(), utils::fp16(0.5f));+ }+};+#ifdef BTLA_UT_SYCL+static UT_SyclS8Gemv sUT_SyclS8Gemv;+#endif++class UT_SyclF8Gemv {+ public:+ UT_SyclF8Gemv() {+ UT_START();+ ut_fp32_T<true>(1024, 11008, 64);+ ut_fp32_T<false>(1024, 11008, 64);+ ut_fp16<true>(1024, 4096, 64);+ ut_fp16<false>(1024, 4096, 64);+ }++ template <bool IsE4M3>+ void ut_fp32_T(int n, int k, int blocksize) {+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case FP8 %s gemv fp32: %d %d %d Device:%s\n", IsE4M3 ? "E4M3" : "E5M2", n, k, blocksize,+ dev->getName().c_str());+ avector<uint8_t> rawB(k * n);+ int blks = k / blocksize;+ avector<float> scale(size_t(blks) * n), bias(n), C(n), dqB(n * k), A(k), refC(n);+ fill_buffer_randn(scale.data(), scale.size(), 0.01f, 0.03f);+ fill_buffer_randn(bias.data(), bias.size(), -0.02f, 0.02f);+ fill_buffer_randn(A.data(), A.size(), -0.1f, 0.3f);+ fill_buffer_randn(rawB.data(), rawB.size(), uint8_t(0), uint8_t(255));+ sanitize_fp8_buffer<IsE4M3>(rawB.data(), rawB.size());+ for (int i = 0; i < n; i++) {+ for (int j = 0; j < k; j++) {+ auto noffset = i * blks + j / blocksize;+ dqB[i + j * n] = decode_fp8_byte<IsE4M3>(rawB[i * k + j]) * scale[noffset];+ }+ }+ gemmref_fp32fp32fp32(1, n, k, A.data(), dqB.data(), refC.data(), k, n, n);+ for (int i = 0; i < n; i++) {+ refC[i] += bias[i];+ }+ sycl_vector<float> dS(scale.size(), q), dC(n, q), dA(k, q);+ sycl_vector<float> dBias(bias.size(), q);+ sycl_vector<uint8_t> dB(rawB.size(), q);+ q->memcpy(dB.data(), rawB.data(), rawB.size()).wait();+ q->memcpy(dS.data(), scale.data(), scale.size() * 4).wait();+ q->memcpy(dA.data(), A.data(), A.size() * 4).wait();+ q->memcpy(dBias.data(), bias.data(), bias.size() * 4).wait();+ using ProB = sycl_prologue_b::WeightF8T<xve::DefaultSGemmCore, float, IsE4M3>;+ auto S_d = dS.data();+ auto B_d = dB.data();+ auto A_d = dA.data();+ auto C_d = dC.data();+ auto Bias_d = dBias.data();+ auto ev = ProB::gemv(A_d, {B_d, S_d, blks, Bias_d}, C_d, n, k, blocksize, q);+ ev.wait();+ q->memcpy(C.data(), C_d, C.size() * 4).wait();+ buffer_error(refC.data(), C.data(), C.size(), IsE4M3 ? 0.001f : 0.05f);+ }++ template <bool IsE4M3>+ void ut_fp16(int n, int k, int blocksize) {+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ printf("Test Case FP8 %s gemv fp16: %d %d %d Device:%s\n", IsE4M3 ? "E4M3" : "E5M2", n, k, blocksize,+ dev->getName().c_str());+ avector<uint8_t> rawB(k * n);+ int blks = k / blocksize;+ avector<utils::fp16> scale(size_t(blks) * n), bias(n), C(n), dqB(n * k), A(k), refC(n);+ fill_buffer_randn(scale.data(), scale.size(), utils::fp16(0.01f), utils::fp16(0.03f));+ fill_buffer_randn(bias.data(), bias.size(), utils::fp16(-0.02f), utils::fp16(0.02f));+ fill_buffer_randn(A.data(), A.size(), utils::fp16(-0.1f), utils::fp16(0.3f));+ fill_buffer_randn(rawB.data(), rawB.size(), uint8_t(0), uint8_t(255));+ sanitize_fp8_buffer<IsE4M3>(rawB.data(), rawB.size());+ for (int i = 0; i < n; i++) {+ for (int j = 0; j < k; j++) {+ auto noffset = i * blks + j / blocksize;+ float fscale = float(scale[noffset]);+ dqB[i + j * n] = decode_fp8_byte<IsE4M3>(rawB[i * k + j]) * fscale;+ }+ }+ gemmref_fp16fp16fp16(1, n, k, A.data(), dqB.data(), refC.data(), k, n, n);+ for (int i = 0; i < n; i++) {+ refC[i] = utils::fp16(float(refC[i]) + float(bias[i]));+ }+ sycl_vector<sycl::half> dS(scale.size(), q), dC(n, q), dA(k, q);+ sycl_vector<sycl::half> dBias(bias.size(), q);+ sycl_vector<uint8_t> dB(rawB.size(), q);+ q->memcpy(dB.data(), rawB.data(), rawB.size()).wait();+ q->memcpy(dS.data(), scale.data(), scale.size() * 2).wait();+ q->memcpy(dA.data(), A.data(), A.size() * 2).wait();+ q->memcpy(dBias.data(), bias.data(), bias.size() * 2).wait();+ using ProB = sycl_prologue_b::WeightF8T<xve::DefaultHGemmCore, sycl::half, IsE4M3>;+ auto S_d = dS.data();+ auto B_d = dB.data();+ auto A_d = dA.data();+ auto C_d = dC.data();+ auto Bias_d = dBias.data();+ auto ev = ProB::gemv(A_d, {B_d, S_d, blks, Bias_d}, C_d, n, k, blocksize, q);+ ev.wait();+ q->memcpy(C.data(), C_d, C.size() * 2).wait();+ buffer_error(refC.data(), C.data(), C.size(), IsE4M3 ? utils::fp16(0.5f) : utils::fp16(10.f));+ }+};+#ifdef BTLA_UT_SYCL+static UT_SyclF8Gemv sUT_SyclF8Gemv;+#endif++void mha_sref(float* Q, float* K, float* V, float* S, float* O, int batch, int seq, int seqA, int hnum, int hsize) {+ avector<float> tmps(seqA);+ int nf = hnum * hsize;+ const float attn_scale = 1.0f / sqrtf(static_cast<float>(hsize));+ int n_past = seqA - seq;+ for (int i = 0; i < batch; i++) {+ for (int j = 0; j < seq; j++) {+ for (int ii = 0; ii < hnum; ii++) {+ float maxs = 0.f;+ for (int jj = 0; jj < seqA; jj++) {+ float tmp = 0.f;+ if (jj <= j + n_past) {+ for (int kk = 0; kk < hsize; kk++) {+ tmp +=+ Q[i * seq * nf + j * nf + ii * hsize + kk] * K[i * nf * seqA + ii * seqA * hsize + jj * hsize + kk];+ }+ tmp *= attn_scale;+ } else {+ tmp = -INFINITY;+ }++ tmps[jj] = tmp;+ maxs = std::max(maxs, tmp);+ }+ float sums = 0.f;+ for (int jj = 0; jj < seqA; jj++) {+ tmps[jj] = std::exp(tmps[jj] - maxs);+ sums += tmps[jj];+ }+ sums = 1.f / sums;+ for (int jj = 0; jj < seqA; jj++) {+ tmps[jj] *= sums;+ S[i * seq * hnum * seqA + j * hnum * seqA + ii * seqA + jj] = tmps[jj];+ }+ for (int kk = 0; kk < hsize; kk++) {+ float tmp = 0.f;+ for (int jj = 0; jj < seqA; jj++) {+ tmp += tmps[jj] * V[i * nf * seqA + ii * hsize * seqA + kk * seqA + jj];+ }+ O[i * seq * nf + j * nf + ii * hsize + kk] = tmp;+ }+ }+ }+ }+}++class UT_MHASgemm {+ public:+ UT_MHASgemm() {+ UT_START();+ ut_T(1, 1, 1, 32, 128);+ ut_T(1, 1, 64, 32, 128);+ ut_T(4, 1, 64, 32, 128);+ ut_T(4, 64, 64, 32, 128);+ }+ template <typename T, typename T_DST>+ class MHA {+ public:+ template <bool Mask>+ static sycl::event forward(int batch, int seq, int seq_acc, int hnum, int hsize, const T* Q, const T* K, const T* V,+ T_DST* O, sycl::queue* q) {+ const float attn_scale = 1.0f / sqrtf(static_cast<float>(hsize));+ int constexpr SgSize = 16;+ assert(hsize % SgSize == 0);+ int n_past = seq_acc - seq;+ if constexpr (Mask) {+ assert(seq > 1);+ }+ int WgSize = SgSize;+ int seq_acc_pad = utils::padto_le(seq_acc, WgSize * 2);+ int nf = hnum * hsize;+ auto ev = q->submit([&](sycl::handler& cgh) {+ sycl::local_accessor<T, 1> slm(sycl::range(std::max(seq_acc, 1024)), cgh);+ cgh.parallel_for(sycl::nd_range<1>(WgSize * batch * seq * hnum, WgSize),+ [=](auto it) [[intel::reqd_sub_group_size(SgSize)]] {+ auto sg = it.get_sub_group();+ auto sg_idx = sg.get_group_id()[0];+ auto wg_idx = it.get_group(0);+ auto wg_loc_id = it.get_local_id();+ auto lane_id = sg.get_local_id()[0];++ int i = wg_idx;+ int ih = i % hnum;+ i /= hnum;+ int is = i % seq;+ i /= seq;+ int ib = i % batch;+ size_t Q_off = ib * seq * nf + is * nf + ih * hsize;+ size_t K_off = ib * seq_acc * nf + ih * hsize * seq_acc;+ size_t V_off = ib * seq_acc * nf + ih * hsize * seq_acc;+ size_t O_off = ib * seq * nf + is * nf + ih * hsize;+ typedef sycl::vec<T, 2> TC;+ T maxs = -INFINITY;+ for (int jj = 0; jj < seq_acc; jj++) {+ TC tmp = {0, 0};+ if constexpr (Mask) {+ if (jj <= is + n_past) {+ for (int ik = wg_loc_id * 2; ik < hsize; ik += WgSize * 2) {+ tmp += *(TC*)&Q[Q_off + ik] * *(TC*)&K[K_off + jj * hsize + ik];+ }+ tmp *= attn_scale;+ } else {+ tmp = {-INFINITY, -INFINITY};+ }+ } else {+ for (int ik = wg_loc_id * 2; ik < hsize; ik += WgSize * 2) {+ tmp += *(TC*)&Q[Q_off + ik] * *(TC*)&K[K_off + jj * hsize + ik];+ }+ tmp *= attn_scale;+ }+ T tmp_sum = tmp[0] + tmp[1];+ T sum = sycl::reduce_over_group(sg, tmp_sum, std::plus<>());+ slm[jj] = sum;+ maxs = std::max(maxs, sum);+ }+ float fsums = 0.f;+ float fmax = float(maxs);+ int jj = wg_loc_id * 2;+ for (; jj < seq_acc_pad; jj += WgSize * 2) {+ auto s2 = *(TC*)&slm[jj];+ s2[0] = std::exp(s2[0] - fmax);+ s2[1] = std::exp(s2[1] - fmax);+ fsums += s2[0];+ fsums += s2[1];+ *(TC*)&slm[jj] = s2;+ }+ if (jj < seq_acc) {+ slm[jj] = std::exp(float(slm[jj]) - fmax);+ fsums += slm[jj];+ if (jj + 1 < seq_acc) {+ slm[jj + 1] = std::exp(float(slm[jj + 1]) - fmax);+ fsums += slm[jj + 1];+ }+ }+ float gsum = sycl::reduce_over_group(sg, fsums, std::plus<>());+ T scale = 1.f / gsum;+ jj = wg_loc_id * 2;+ for (; jj < seq_acc_pad; jj += WgSize * 2) {+ auto s2 = *(TC*)&slm[jj];+ s2 *= scale;+ *(TC*)&slm[jj] = s2;+ }+ if (jj < seq_acc) {+ slm[jj] *= scale;+ if (jj + 1 < seq_acc) {+ slm[jj + 1] *= scale;+ }+ }++ for (int kk = 0; kk < hsize; kk++) {+ TC tmp = {0, 0};+ jj = wg_loc_id * 2;+ for (; jj < seq_acc_pad; jj += WgSize * 2) {+ auto s2 = *(TC*)&slm[jj];+ auto v2 = *(TC*)&V[V_off + kk * seq_acc + jj];+ tmp += s2 * v2;+ }+ if (jj < seq_acc) {+ tmp[0] += slm[jj] * V[V_off + kk * seq_acc + jj];+ if (jj + 1 < seq_acc) {+ tmp[1] += slm[jj + 1] * V[V_off + kk * seq_acc + jj + 1];+ }+ }+ T tmp_sum = tmp[0] + tmp[1];+ T sum = sycl::reduce_over_group(sg, tmp_sum, std::plus<>());+ O[O_off + kk] = sum;+ }+ });+ });+ return ev;+ }+ };++ void ut_T(int batch, int seq, int seqA, int hnum, int hsize) {+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ assert(seqA >= seq);+ printf("Test Case %s: %d %d %d %d %d Device:%s\n", __FUNCTION__, batch, seq, seqA, hnum, hsize,+ dev->getName().c_str());+ avector<float> Q(batch * seq * hnum * hsize), K(batch * seqA * hnum * hsize), V(batch * seqA * hnum * hsize);+ fill_buffer_randn(Q.data(), Q.size(), -0.5f, 0.5f);+ fill_buffer_randn(K.data(), K.size(), -0.5f, 0.5f);+ fill_buffer_randn(V.data(), V.size(), -0.5f, 0.5f);+ avector<float> S(batch * seq * hnum * seqA), O(batch * seq * hnum * hsize);+ mha_sref(Q.data(), K.data(), V.data(), S.data(), O.data(), batch, seq, seqA, hnum, hsize);+ sycl_vector<float> dQ(batch * seq * hnum * hsize, q), dK(batch * seqA * hnum * hsize, q),+ dV(batch * seqA * hnum * hsize, q);+ sycl_vector<float> dS(batch * seq * hnum * seqA, q), dO(batch * seq * hnum * hsize, q);+ q->memcpy(dQ.data(), Q.data(), Q.size() * sizeof(Q[0]));+ q->memcpy(dK.data(), K.data(), K.size() * sizeof(K[0]));+ q->memcpy(dV.data(), V.data(), V.size() * sizeof(V[0]));+ q->wait();+ auto Qptr = dQ.data();+ auto Kptr = dK.data();+ auto Vptr = dV.data();+ auto Sptr = dS.data();+ auto Optr = dO.data();+ int nf = hnum * hsize;+ sycl::range<1> num_items{(size_t)batch * seq * hnum};+ int n_past = seqA - seq;+ const float attn_scale = 1.0f / sqrtf(static_cast<float>(hsize));+ if (seq > 1) {+ MHA<float, float>::forward<true>(batch, seq, seqA, hnum, hsize, Qptr, Kptr, Vptr, Optr, q).wait();+ } else {+ MHA<float, float>::forward<false>(batch, seq, seqA, hnum, hsize, Qptr, Kptr, Vptr, Optr, q).wait();+ }+ // auto ev = q->submit([&](sycl::handler& cgh) {+ // cgh.parallel_for(num_items, [=](auto it) {+ // int i = it;+ // int ih = i % hnum;+ // i /= hnum;+ // int is = i % seq;+ // i /= seq;+ // int ib = i % batch;+ // float maxs = 0.f;+ // float tmps[64];+ // for (int jj = 0; jj < seqA; jj++) {+ // float tmp = 0.f;+ // if (jj <= is + n_past) {+ // for (int kk = 0; kk < hsize; kk++) {+ // tmp += Qptr[ib * seq * nf + is * nf + ih * hsize + kk] *+ // Kptr[ib * nf * seqA + kk + ih * seqA * hsize + jj * hsize];+ // }+ // tmp *= attn_scale;+ // } else {+ // tmp = -INFINITY;+ // }++ // tmps[jj] = tmp;+ // maxs = std::max(maxs, tmp);+ // }+ // float sums = 0.f;+ // for (int jj = 0; jj < seqA; jj++) {+ // tmps[jj] = std::exp(tmps[jj] - maxs);+ // sums += tmps[jj];+ // }+ // sums = 1.f / sums;+ // for (int jj = 0; jj < seqA; jj++) {+ // tmps[jj] *= sums;+ // Sptr[ib * seq * hnum * seqA + is * hnum * seqA + ih * seqA + jj] = tmps[jj];+ // }+ // for (int kk = 0; kk < hsize; kk++) {+ // float tmp = 0.f;+ // for (int jj = 0; jj < seqA; jj++) {+ // tmp += tmps[jj] * Vptr[ib * seqA * nf + jj + ih * hsize * seqA + kk * seqA];+ // }+ // Optr[ib * seq * nf + is * nf + ih * hsize + kk] = tmp;+ // }+ // });+ //});+ q->wait();+ avector<float> STar(batch * seq * hnum * seqA), OTar(batch * seq * hnum * hsize);+ q->memcpy(STar.data(), Sptr, STar.size() * sizeof(STar[0]));+ q->memcpy(OTar.data(), Optr, OTar.size() * sizeof(OTar[0]));+ q->wait();+ // buffer_error(S.data(), STar.data(), S.size(), 0.001f);+ buffer_error(O.data(), OTar.data(), O.size(), 0.001f);+ }+};+#ifdef BTLA_UT_SYCL+#endif+#endif+// static UT_MHASgemm sUT_MHASgemm;+} // namespace sycl_ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_misc.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_misc.cppnew file mode 100644index 000000000..332317a35--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_misc.cpp@@ -0,0 +1,40 @@+#include "bestla_ut.h"+#include "bestla_prologue_b.h"+#include "sycl_ut.h"+#include "sycl/sycl_device.h"+#include "sycl/sycl_utils.h"+#include "sycl/sycl_storage.h"+#include "sycl/sycl_gemm.h"+#include "sycl/sycl_prologue_b.h"+namespace bestla {+using namespace ut;+using namespace utils;+namespace sycl_ut {+class UT_SyclDevice {+ public:+ UT_SyclDevice() {+ UT_START();+ auto dev = UT_Device::get();+ dev->print();+ }+};+// static UT_SyclDevice sUT_SyclDevice;++class UT_SyclVector {+ public:+ UT_SyclVector() {+ UT_START();+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ auto svec = sycl_utils::sycl_vector<float>(1000, q);+ utils::avector<float> hsrc(1000);+ ut::fill_buffer_randn(hsrc.data(), hsrc.size(), -0.5f, 0.5f);+ q->memcpy(svec.data(), hsrc.data(), hsrc.size() * 4).wait();+ auto hdst = sycl_utils::sycl2host(svec.data(), svec.size(), q);+ ut::buffer_error(hsrc.data(), hdst.data(), hsrc.size(), 0.f);+ }+};+// static UT_SyclVector sUT_SyclVector;++} // namespace sycl_ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_tla_flash_attn_decode_bench.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_tla_flash_attn_decode_bench.cppnew file mode 100644index 000000000..676f19df1--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_tla_flash_attn_decode_bench.cpp@@ -0,0 +1,489 @@+// Flash Attention Decode Benchmarks using sycl-tla+// Separated from sycl_benchmark.cpp to avoid header conflicts with prefill kernels.+//+// Copyright (C) 2026 Intel Corporation+// SPDX-License-Identifier: MIT++#include <stdio.h>+#include <type_traits>+#include "bestla_ut.h"+#include "bestla_utils.h"+#include "sycl_ut.h"+#include "sycl/sycl_utils.h"++#if defined(ARK_SYCL_TLA)++#include "cutlass/util/device_memory.h"+#include "cutlass/util/packed_stride.hpp"++// FlashAttention-v2 (Xe) building blocks (from sycl-tla).+#include "cutlass/epilogue/collective/default_epilogue.hpp"+#include "cutlass/epilogue/fusion/xe_callbacks.hpp"+#include "flash_attention_v2/collective/fmha_fusion.hpp"+#include "flash_attention_v2/kernel/tile_scheduler.hpp"++// FlashAttention-v2 Decode kernels only+#include "flash_attention_v2/kernel/xe_flash_attn_decode.hpp"+#include "flash_attention_v2/collective/xe_flash_attn_decode_epilogue.hpp"+#include "flash_attention_v2/collective/xe_flash_attn_decode_softmax_epilogue.hpp"++// CUTLASS-SYCL/CuTe compat layer+#include "cute/util/compat.hpp"++// sycl-tla example utilities+#include "helper.h"+#include "sycl_common.hpp"++#include "cute/tensor.hpp"++namespace bestla {+using namespace ut;+using namespace utils;+using namespace sycl_utils;+namespace sycl_ut {++int constexpr TestMs = 1000;++template <typename Kernel, typename = void>+struct BestlaBenchSubgroupSizeDecode {+ static constexpr int value = Kernel::DispatchPolicy::SubgroupSize;+};++template <typename Kernel>+struct BestlaBenchSubgroupSizeDecode<Kernel, std::void_t<decltype(Kernel::SubgroupSize)>> {+ static constexpr int value = Kernel::SubgroupSize;+};++// ========================================================================+// Flash Attention Decode Benchmarks (FP16)+// ========================================================================++class Benchmark_SyclTla_FlashAttnDecode {+ public:+ Benchmark_SyclTla_FlashAttnDecode() {+ UT_START();+ // Baseline+ run_all_fp16(/*batch=*/1, /*num_heads_q=*/32, /*num_heads_kv=*/32, /*head_dim=*/128, /*cache_len=*/4096,+ /*is_causal=*/true);++ // Heavier-cache cases to expose bandwidth effects (use fewer heads to keep memory reasonable).+ // KV cache elems = cache_len * head_dim * batch * num_heads_kv.+ run_all_fp16(/*batch=*/4, /*num_heads_q=*/16, /*num_heads_kv=*/16, /*head_dim=*/128, /*cache_len=*/8192,+ /*is_causal=*/true);+ run_all_fp16(/*batch=*/8, /*num_heads_q=*/8, /*num_heads_kv=*/8, /*head_dim=*/128, /*cache_len=*/8192,+ /*is_causal=*/true);+ }++ template <typename ElementInput>+ static const char* dtype_str() {+ if constexpr (std::is_same_v<ElementInput, cutlass::bfloat16_t>) {+ return "bf16";+ } else if constexpr (std::is_same_v<ElementInput, cutlass::half_t>) {+ return "fp16";+ } else if constexpr (std::is_same_v<ElementInput, cutlass::float_e4m3_t>) {+ return "f8_e4m3";+ } else if constexpr (std::is_same_v<ElementInput, cutlass::float_e5m2_t>) {+ return "f8_e5m2";+ } else {+ return "unknown";+ }+ }++ private:+ template <typename KernelT>+ static sycl::event launch_decode(typename KernelT::Params const& params) {+ compat::dim3 const block = KernelT::get_block_shape();+ compat::dim3 const grid = KernelT::get_grid_shape(params);+ int smem_size = KernelT::SharedStorageSize;++ const auto sycl_block = compat::dim3(block.x, block.y, block.z);+ const auto sycl_grid = compat::dim3(grid.x, grid.y, grid.z);++#if !defined(SYCL_EXT_ONEAPI_WORK_GROUP_SCRATCH_MEMORY)+ using namespace compat::experimental;+ return launch<cutlass::device_kernel<KernelT>>(+ launch_policy{sycl_grid, sycl_block, local_mem_size{static_cast<std::size_t>(smem_size)},+ kernel_properties{sycl_exp::sub_group_size<BestlaBenchSubgroupSizeDecode<KernelT>::value>}},+ params);+#else+ compat::experimental::launch_properties launch_props{+ sycl::ext::oneapi::experimental::work_group_scratch_size(smem_size),+ };+ compat::experimental::kernel_properties kernel_props{+ sycl::ext::oneapi::experimental::sub_group_size<BestlaBenchSubgroupSizeDecode<KernelT>::value>};+ compat::experimental::launch_policy policy{sycl_grid, sycl_block, launch_props, kernel_props};+ return compat::experimental::launch<cutlass::device_kernel<KernelT>, KernelT>(policy, params);+#endif+ }++ template <int KVTile, int NumSGs>+ struct DecodeTileConfigH64 {+ using ShapeQK = cute::Shape<cute::_1, cute::Int<KVTile>, cute::_64>;+ using ShapePV = cute::Shape<cute::_1, cute::_32, cute::Int<KVTile>>;+ using ShapeOutput = cute::Shape<cute::_1, cute::_64, cute::Int<KVTile>>;+ // Match sycl-tla examples/06_bmg_flash_attention subgroup layout+ using SubgroupLayout =+ cute::Layout<cute::Shape<cute::Int<NumSGs>, cute::_1, cute::_1>, cute::Stride<cute::_1, cute::_1, cute::_1>>;+ };++ template <int KVTile, int NumSGs>+ struct DecodeTileConfigH128 {+ using ShapeQK = cute::Shape<cute::_1, cute::Int<KVTile>, cute::_64>;+ using ShapePV = cute::Shape<cute::_1, cute::_32, cute::Int<KVTile>>;+ using ShapeOutput = cute::Shape<cute::_1, cute::_128, cute::Int<KVTile>>;+ // Match sycl-tla examples/06_bmg_flash_attention subgroup layout+ using SubgroupLayout =+ cute::Layout<cute::Shape<cute::Int<NumSGs>, cute::_1, cute::_1>, cute::Stride<cute::_1, cute::_1, cute::_1>>;+ };++ template <int HeadDim>+ using DecodeTileConfig = std::conditional_t<HeadDim == 64, DecodeTileConfigH64</*KVTile=*/512, /*NumSGs=*/8>,+ DecodeTileConfigH128</*KVTile=*/512, /*NumSGs=*/8>>;++ template <typename ElementInput, bool Causal, int HeadDim, typename = void>+ struct DecodeKernelBuilder {+ static constexpr bool supported = false;+ };++ // FP16 decode kernel for BMG+ template <bool Causal, int HeadDim>+ struct DecodeKernelBuilder<cutlass::half_t, Causal, HeadDim> {+ static constexpr bool supported = true;++ using ElementOutput = float;+ using ElementAccumulator = float;+ using LayoutQ = cutlass::layout::RowMajor;+ using LayoutK = cutlass::layout::ColumnMajor;+ using LayoutV = cutlass::layout::RowMajor;+ using LayoutO = cutlass::layout::RowMajor;++ using GEMMDispatchPolicy = cutlass::gemm::MainloopIntelXeXMX16</*Stages=*/2>;+ using EpilogueDispatchPolicy = cutlass::epilogue::IntelXeXMX16;+ using MMAOperation = cute::XE_1x16x16_F32F16F16F32_TT;++ using Tile = DecodeTileConfig<HeadDim>;++ using GmemTiledCopyQ = cute::XE_2D_U16x1x16_LD_N;+ using GmemTiledCopyK = cute::XE_2D_U16x16x16_LD_T;+ using GmemTiledCopyV = cute::XE_2D_U16x32x32_LD_V;+ using GmemTiledCopyO = cute::XE_2D_U32x1x16_ST_N;++ using CollectiveEpilogue = cutlass::flash_attention::collective::FlashDecodeEpilogue<+ EpilogueDispatchPolicy, MMAOperation, typename Tile::ShapeOutput, typename Tile::SubgroupLayout,+ ElementAccumulator, ElementOutput, cutlass::gemm::TagToStrideC_t<LayoutO>, ElementOutput, GmemTiledCopyO>;++ using CollectiveSoftmaxEpilogue =+ cutlass::flash_attention::collective::FlashDecodeSoftmaxEpilogue<Causal, EpilogueDispatchPolicy,+ ElementAccumulator>;++ using ProblemShapeType = cute::tuple<int, int, int, int, int, int, int, int>;++ using CollectiveMainloop = cutlass::flash_attention::collective::FlashDecodeMma<+ GEMMDispatchPolicy, ProblemShapeType, cutlass::half_t, cutlass::gemm::TagToStrideA_t<LayoutQ>, cutlass::half_t,+ cutlass::gemm::TagToStrideB_t<LayoutK>, cutlass::half_t, cutlass::gemm::TagToStrideB_t<LayoutV>, MMAOperation,+ typename Tile::ShapeQK, typename Tile::ShapePV, typename Tile::SubgroupLayout, GmemTiledCopyQ, GmemTiledCopyK,+ GmemTiledCopyV, Causal,+ /*PagedKV=*/false>;++ using Kernel = cutlass::flash_attention::kernel::FMHADecode<ProblemShapeType, CollectiveMainloop,+ CollectiveSoftmaxEpilogue, CollectiveEpilogue>;+ };++ // FP8 (E5M2) decode kernel for BMG - only HeadDim=128 is supported+ // HeadDim=64 fails FP8->FP16 conversion constraint (num_elements must be multiple of fragment_size)+ template <bool Causal>+ struct DecodeKernelBuilder<cutlass::float_e5m2_t, Causal, 128> {+ static constexpr bool supported = true;+ static constexpr int HeadDim = 128;++ using ElementInput = cutlass::float_e5m2_t;+ using ElementOutput = float;+ using ElementAccumulator = float;+ using LayoutQ = cutlass::layout::RowMajor;+ using LayoutK = cutlass::layout::ColumnMajor;+ using LayoutV = cutlass::layout::RowMajor;+ using LayoutO = cutlass::layout::RowMajor;++ using GEMMDispatchPolicy = cutlass::gemm::MainloopIntelXeXMX16</*Stages=*/2>;+ using EpilogueDispatchPolicy = cutlass::epilogue::IntelXeXMX16;+ using MMAOperation = cute::XE_1x16x16_F32F16F16F32_TT;++ using Tile = DecodeTileConfig<HeadDim>;++ // FP8 uses U8 copy operations+ using GmemTiledCopyQ = cute::XE_2D_U8x1x32_LD_N;+ using GmemTiledCopyK = cute::XE_2D_U8x16x16_LD_T;+ using GmemTiledCopyV = cute::XE_2D_U8x32x32_LD_V;+ using GmemTiledCopyO = cute::XE_2D_U32x1x16_ST_N;++ using CollectiveEpilogue = cutlass::flash_attention::collective::FlashDecodeEpilogue<+ EpilogueDispatchPolicy, MMAOperation, typename Tile::ShapeOutput, typename Tile::SubgroupLayout,+ ElementAccumulator, ElementOutput, cutlass::gemm::TagToStrideC_t<LayoutO>, ElementOutput, GmemTiledCopyO>;++ using CollectiveSoftmaxEpilogue =+ cutlass::flash_attention::collective::FlashDecodeSoftmaxEpilogue<Causal, EpilogueDispatchPolicy,+ ElementAccumulator>;++ using ProblemShapeType = cute::tuple<int, int, int, int, int, int, int, int>;++ using CollectiveMainloop = cutlass::flash_attention::collective::FlashDecodeMma<+ GEMMDispatchPolicy, ProblemShapeType, ElementInput, cutlass::gemm::TagToStrideA_t<LayoutQ>, ElementInput,+ cutlass::gemm::TagToStrideB_t<LayoutK>, ElementInput, cutlass::gemm::TagToStrideB_t<LayoutV>, MMAOperation,+ typename Tile::ShapeQK, typename Tile::ShapePV, typename Tile::SubgroupLayout, GmemTiledCopyQ, GmemTiledCopyK,+ GmemTiledCopyV, Causal,+ /*PagedKV=*/false>;++ using Kernel = cutlass::flash_attention::kernel::FMHADecode<ProblemShapeType, CollectiveMainloop,+ CollectiveSoftmaxEpilogue, CollectiveEpilogue>;+ };++ // FP8 (E4M3) decode kernel for BMG - only HeadDim=128 is supported+ // HeadDim=64 fails FP8->FP16 conversion constraint (num_elements must be multiple of fragment_size)+ template <bool Causal>+ struct DecodeKernelBuilder<cutlass::float_e4m3_t, Causal, 128> {+ static constexpr bool supported = true;+ static constexpr int HeadDim = 128;++ using ElementInput = cutlass::float_e4m3_t;+ using ElementOutput = float;+ using ElementAccumulator = float;+ using LayoutQ = cutlass::layout::RowMajor;+ using LayoutK = cutlass::layout::ColumnMajor;+ using LayoutV = cutlass::layout::RowMajor;+ using LayoutO = cutlass::layout::RowMajor;++ using GEMMDispatchPolicy = cutlass::gemm::MainloopIntelXeXMX16</*Stages=*/2>;+ using EpilogueDispatchPolicy = cutlass::epilogue::IntelXeXMX16;+ using MMAOperation = cute::XE_1x16x16_F32F16F16F32_TT;++ using Tile = DecodeTileConfig<HeadDim>;++ // FP8 uses U8 copy operations+ using GmemTiledCopyQ = cute::XE_2D_U8x1x32_LD_N;+ using GmemTiledCopyK = cute::XE_2D_U8x16x16_LD_T;+ using GmemTiledCopyV = cute::XE_2D_U8x32x32_LD_V;+ using GmemTiledCopyO = cute::XE_2D_U32x1x16_ST_N;++ using CollectiveEpilogue = cutlass::flash_attention::collective::FlashDecodeEpilogue<+ EpilogueDispatchPolicy, MMAOperation, typename Tile::ShapeOutput, typename Tile::SubgroupLayout,+ ElementAccumulator, ElementOutput, cutlass::gemm::TagToStrideC_t<LayoutO>, ElementOutput, GmemTiledCopyO>;++ using CollectiveSoftmaxEpilogue =+ cutlass::flash_attention::collective::FlashDecodeSoftmaxEpilogue<Causal, EpilogueDispatchPolicy,+ ElementAccumulator>;++ using ProblemShapeType = cute::tuple<int, int, int, int, int, int, int, int>;++ using CollectiveMainloop = cutlass::flash_attention::collective::FlashDecodeMma<+ GEMMDispatchPolicy, ProblemShapeType, ElementInput, cutlass::gemm::TagToStrideA_t<LayoutQ>, ElementInput,+ cutlass::gemm::TagToStrideB_t<LayoutK>, ElementInput, cutlass::gemm::TagToStrideB_t<LayoutV>, MMAOperation,+ typename Tile::ShapeQK, typename Tile::ShapePV, typename Tile::SubgroupLayout, GmemTiledCopyQ, GmemTiledCopyK,+ GmemTiledCopyV, Causal,+ /*PagedKV=*/false>;++ using Kernel = cutlass::flash_attention::kernel::FMHADecode<ProblemShapeType, CollectiveMainloop,+ CollectiveSoftmaxEpilogue, CollectiveEpilogue>;+ };++ public:+ template <typename ElementInput, int HeadDim, bool Causal>+ static void bench_one(int batch, int num_heads_q, int num_heads_kv, int cache_len, float softmax_scale,+ float timems) {+ if constexpr (!DecodeKernelBuilder<ElementInput, Causal, HeadDim>::supported) {+ printf("[sycl-tla flash_attn decode] Skip: unsupported dtype/head (only fp16 kernels wired for BMG)\n");+ return;+ } else {+ using Kernel = typename DecodeKernelBuilder<ElementInput, Causal, HeadDim>::Kernel;++ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ compat::set_default_queue(*q);++ const int seq_qo = 1;+ const int seq_kv_new = 1;++ const size_t q_elems = size_t(seq_qo) * size_t(HeadDim) * size_t(batch) * size_t(num_heads_q);+ const size_t kv_new_elems = size_t(seq_kv_new) * size_t(HeadDim) * size_t(batch) * size_t(num_heads_kv);+ const size_t kv_cache_elems = size_t(cache_len) * size_t(HeadDim) * size_t(batch) * size_t(num_heads_kv);+ const size_t o_elems = q_elems;++ cutlass::DeviceAllocation<ElementInput> dQ(q_elems);+ cutlass::DeviceAllocation<ElementInput> dK_new(kv_new_elems);+ cutlass::DeviceAllocation<ElementInput> dV_new(kv_new_elems);+ cutlass::DeviceAllocation<ElementInput> dK_cache(kv_cache_elems);+ cutlass::DeviceAllocation<ElementInput> dV_cache(kv_cache_elems);+ cutlass::DeviceAllocation<float> dO(o_elems);++ q->memset(dQ.get(), 0, q_elems * sizeof(ElementInput)).wait();+ q->memset(dK_new.get(), 0, kv_new_elems * sizeof(ElementInput)).wait();+ q->memset(dV_new.get(), 0, kv_new_elems * sizeof(ElementInput)).wait();+ q->memset(dK_cache.get(), 0, kv_cache_elems * sizeof(ElementInput)).wait();+ q->memset(dV_cache.get(), 0, kv_cache_elems * sizeof(ElementInput)).wait();+ q->memset(dO.get(), 0, o_elems * sizeof(float)).wait();++ typename Kernel::StrideQ stride_Q = cutlass::make_cute_packed_stride(+ typename Kernel::StrideQ{}, cute::make_shape(seq_qo, HeadDim, batch * num_heads_q));+ typename Kernel::StrideK stride_K = cutlass::make_cute_packed_stride(+ typename Kernel::StrideK{}, cute::make_shape(seq_kv_new, HeadDim, batch * num_heads_kv));+ typename Kernel::StrideV stride_V = cutlass::make_cute_packed_stride(+ typename Kernel::StrideV{}, cute::make_shape(HeadDim, seq_kv_new, batch * num_heads_kv));+ typename Kernel::StrideK stride_K_cache = cutlass::make_cute_packed_stride(+ typename Kernel::StrideK{}, cute::make_shape(cache_len, HeadDim, batch * num_heads_kv));+ typename Kernel::StrideV stride_V_cache = cutlass::make_cute_packed_stride(+ typename Kernel::StrideV{}, cute::make_shape(HeadDim, cache_len, batch * num_heads_kv));+ typename Kernel::StrideO stride_O = cutlass::make_cute_packed_stride(+ typename Kernel::StrideO{}, cute::make_shape(seq_qo, HeadDim, batch * num_heads_q));++ typename Kernel::ProblemShape problem_size =+ cute::make_tuple(batch, num_heads_q, num_heads_kv, seq_qo, seq_kv_new, cache_len, HeadDim, HeadDim);++ cutlass::KernelHardwareInfo hw_info;+ typename Kernel::Arguments arguments{cutlass::gemm::GemmUniversalMode::kGemm,+ problem_size,+ {+ dQ.get(),+ stride_Q,+ dK_new.get(),+ stride_K,+ dV_new.get(),+ stride_V,+ dK_cache.get(),+ stride_K_cache,+ dV_cache.get(),+ stride_V_cache,+ /*ptr_page_table=*/nullptr,+ /*page_size=*/0,+ /*num_pages_per_seq=*/nullptr,+ },+ {softmax_scale},+ {dO.get(), stride_O},+ hw_info};++ if (!Kernel::can_implement(arguments)) {+ printf("[sycl-tla flash_attn decode] Skip: can_implement failed\n");+ return;+ }++ size_t workspace_size = Kernel::get_workspace_size(arguments);+ cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);+ auto status = Kernel::initialize_workspace(arguments, workspace.get());+ if (status != cutlass::Status::kSuccess) {+ printf("[sycl-tla flash_attn decode] Skip: initialize_workspace failed\n");+ return;+ }++ auto params = Kernel::to_underlying_arguments(arguments, workspace.get());+ (void)launch_decode<Kernel>(params).wait();++ using LOG = timer_statistics_logger<TestMs * 2>;+ LOG log;+ utils::timer<std::chrono::milliseconds> wall;+ wall.start();+ while (wall.stop() < timems) {+ auto ev = launch_decode<Kernel>(params);+ ev.wait();+ log.add(event_helper::execute_time(ev) * 1000);+ if (wall.stop() >= timems) break;+ }+ log.record();++ // Print sycl-tla runner style metrics for easy comparison.+ // For decode, Sq=1 and Skv=cache_len; FLOPs convention matches the prefill runner:+ // flops = 4 * B * Hq * Sq * Skv * D+ const double time_ms = static_cast<double>(log.avg_val);+ const double time_s = time_ms * 1e-3;+ const double flops = 4.0 * static_cast<double>(batch) * static_cast<double>(num_heads_q) * 1.0 *+ static_cast<double>(cache_len) * static_cast<double>(HeadDim);++ // Count K/V bytes against Hq (logical per-q-head work), matching the runner's intent.+ const double kv_read_elems_equiv = static_cast<double>(batch) * static_cast<double>(num_heads_q) *+ static_cast<double>(cache_len) * static_cast<double>(HeadDim);+ const double bytes = static_cast<double>(q_elems) * sizeof(ElementInput) ++ 2.0 * kv_read_elems_equiv * sizeof(ElementInput) ++ static_cast<double>(o_elems) * sizeof(float);+ const double tflops = (time_s > 0.0) ? (flops / time_s / 1.0e12) : 0.0;+ const double gbs = (time_s > 0.0) ? (bytes / time_s / 1.0e9) : 0.0;+ printf("Performance: %.3f GB/s, %.3f TFlop/s, %.4f ms (dtype=%s)\n", gbs, tflops, time_ms,+ dtype_str<ElementInput>());++ printf("[sycl-tla flash_attn decode] dtype=%s B=%d Hq=%d Hkv=%d cache=%d D=%d causal=%d %s\n",+ dtype_str<ElementInput>(), batch, num_heads_q, num_heads_kv, cache_len, HeadDim, int(Causal),+ log.get_log_str());+ }+ }++ static void run_all_fp16(int batch, int num_heads_q, int num_heads_kv, int head_dim, int cache_len, bool is_causal) {+ printf("\n[sycl-tla flash_attn decode fp16] case: B=%d Hq=%d Hkv=%d cache=%d D=%d causal=%d\n", batch, num_heads_q,+ num_heads_kv, cache_len, head_dim, int(is_causal));+ if (head_dim == 64) {+ if (is_causal)+ bench_one<cutlass::half_t, 64, true>(batch, num_heads_q, num_heads_kv, cache_len, 1.0f, float(TestMs));+ else+ bench_one<cutlass::half_t, 64, false>(batch, num_heads_q, num_heads_kv, cache_len, 1.0f, float(TestMs));+ } else if (head_dim == 128) {+ if (is_causal)+ bench_one<cutlass::half_t, 128, true>(batch, num_heads_q, num_heads_kv, cache_len, 1.0f, float(TestMs));+ else+ bench_one<cutlass::half_t, 128, false>(batch, num_heads_q, num_heads_kv, cache_len, 1.0f, float(TestMs));+ }+ }+};+#if 0+static Benchmark_SyclTla_FlashAttnDecode sBenchmark_SyclTla_FlashAttnDecode;+#endif+// ========================================================================+// Flash Attention Decode FP8+// ========================================================================++class Benchmark_SyclTla_FlashAttnDecodeFP8 {+ public:+ Benchmark_SyclTla_FlashAttnDecodeFP8() {+ UT_START();+ // Baseline+ run_all_fp8(/*batch=*/1, /*num_heads_q=*/32, /*num_heads_kv=*/32, /*head_dim=*/128, /*cache_len=*/4096,+ /*is_causal=*/true);++ // Heavier-cache cases to expose bandwidth effects (use fewer heads to keep memory reasonable).+ run_all_fp8(/*batch=*/4, /*num_heads_q=*/16, /*num_heads_kv=*/16, /*head_dim=*/128, /*cache_len=*/8192,+ /*is_causal=*/true);+ run_all_fp8(/*batch=*/8, /*num_heads_q=*/8, /*num_heads_kv=*/8, /*head_dim=*/128, /*cache_len=*/8192,+ /*is_causal=*/true);+ }++ private:+ static void run_all_fp8(int batch, int num_heads_q, int num_heads_kv, int head_dim, int cache_len, bool is_causal) {+ printf("\n[sycl-tla flash_attn decode fp8] case: B=%d Hq=%d Hkv=%d cache=%d D=%d causal=%d\n", batch, num_heads_q,+ num_heads_kv, cache_len, head_dim, int(is_causal));++ // NOTE:+ // sycl-tla BMG FP8 decode example (examples/06_bmg_flash_attention/06_bmg_decode_attention_fp8.cpp)+ // is wired for FP8 E5M2. Instantiating E4M3 currently triggers a static_assert in fp8_to_fp16.h+ // (work-item must convert a multiple of fragment_size).++ if (head_dim == 64) {+ if (is_causal) {+ Benchmark_SyclTla_FlashAttnDecode::bench_one<cutlass::float_e5m2_t, 64, true>(batch, num_heads_q, num_heads_kv,+ cache_len, 1.0f, float(TestMs));+ } else {+ Benchmark_SyclTla_FlashAttnDecode::bench_one<cutlass::float_e5m2_t, 64, false>(batch, num_heads_q, num_heads_kv,+ cache_len, 1.0f, float(TestMs));+ }+ } else if (head_dim == 128) {+ if (is_causal) {+ Benchmark_SyclTla_FlashAttnDecode::bench_one<cutlass::float_e5m2_t, 128, true>(batch, num_heads_q, num_heads_kv,+ cache_len, 1.0f, float(TestMs));+ } else {+ Benchmark_SyclTla_FlashAttnDecode::bench_one<cutlass::float_e5m2_t, 128, false>(+ batch, num_heads_q, num_heads_kv, cache_len, 1.0f, float(TestMs));+ }+ }+ }+};++#if 0+static Benchmark_SyclTla_FlashAttnDecodeFP8 sBenchmark_SyclTla_FlashAttnDecodeFP8;+#endif++#endif // ARK_SYCL_TLA+} // namespace sycl_ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_tla_flash_attn_prefill_bench.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_tla_flash_attn_prefill_bench.cppnew file mode 100644index 000000000..716512281--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_tla_flash_attn_prefill_bench.cpp@@ -0,0 +1,433 @@+// Flash Attention Prefill Benchmarks using sycl-tla+// Separated from sycl_benchmark.cpp to avoid header conflicts with decode kernels.+//+// Copyright (C) 2026 Intel Corporation+// SPDX-License-Identifier: MIT++#include <stdio.h>+#include <type_traits>+#include "bestla_ut.h"+#include "bestla_utils.h"+#include "sycl_ut.h"+#include "sycl/sycl_utils.h"++#if defined(ARK_SYCL_TLA)++#include "cutlass/util/device_memory.h"+#include "cutlass/util/packed_stride.hpp"++// FlashAttention-v2 (Xe) building blocks (from sycl-tla).+#include "cutlass/epilogue/collective/default_epilogue.hpp"+#include "cutlass/epilogue/fusion/xe_callbacks.hpp"+#include "flash_attention_v2/collective/fmha_fusion.hpp"+#include "flash_attention_v2/kernel/tile_scheduler.hpp"++// FlashAttention-v2 Prefill kernels only+#include "flash_attention_v2/kernel/xe_flash_attn_prefill.hpp"+#include "flash_attention_v2/collective/xe_flash_attn_prefill_epilogue.hpp"+#include "flash_attention_v2/collective/xe_flash_attn_prefill_softmax_epilogue.hpp"++// CUTLASS-SYCL/CuTe compat layer+#include "cute/util/compat.hpp"++// sycl-tla example utilities+#include "helper.h"+#include "sycl_common.hpp"++#include "cute/tensor.hpp"++// sycl-tla example 06 (SDPA) runner header+#if !defined(HEAD_DIM)+#define HEAD_DIM 128+#endif+#include "xe_fmha_fwd_runner.hpp"++namespace bestla {+using namespace ut;+using namespace utils;+using namespace sycl_utils;+namespace sycl_ut {++int constexpr TestMs = 1000;++template <typename Kernel, typename = void>+struct BestlaBenchSubgroupSize {+ static constexpr int value = Kernel::DispatchPolicy::SubgroupSize;+};++template <typename Kernel>+struct BestlaBenchSubgroupSize<Kernel, std::void_t<decltype(Kernel::SubgroupSize)>> {+ static constexpr int value = Kernel::SubgroupSize;+};++// ========================================================================+// Flash Attention Prefill Benchmarks (FP16)+// ========================================================================++class Benchmark_SyclTla_FlashAttnPrefill {+ public:+ Benchmark_SyclTla_FlashAttnPrefill() {+ UT_START();+ // SDPA/FlashAttention forward (fp16) using sycl-tla example-06 runner.+ // Include a couple larger shapes to better expose bandwidth effects.+ run_fp16_prefill(/*batch=*/1, /*num_heads_q=*/32, /*num_heads_kv=*/32, /*seq_qo=*/1024, /*seq_kv=*/1024,+ /*head_dim=*/128,+ /*is_causal=*/true,+ /*seq_kv_cache=*/0);+ // Non-causal baseline for fair comparison with FP8 non-causal cases.+ run_fp16_prefill(/*batch=*/1, /*num_heads_q=*/32, /*num_heads_kv=*/32, /*seq_qo=*/1024, /*seq_kv=*/1024,+ /*head_dim=*/128,+ /*is_causal=*/false,+ /*seq_kv_cache=*/0);+ run_fp16_prefill(/*batch=*/4, /*num_heads_q=*/32, /*num_heads_kv=*/32, /*seq_qo=*/1024, /*seq_kv=*/1024,+ /*head_dim=*/128,+ /*is_causal=*/false,+ /*seq_kv_cache=*/0);+ run_fp16_prefill(/*batch=*/4, /*num_heads_q=*/32, /*num_heads_kv=*/32, /*seq_qo=*/2048, /*seq_kv=*/2048,+ /*head_dim=*/128,+ /*is_causal=*/true,+ /*seq_kv_cache=*/0);+ }++ static void run_fp16_prefill(int batch, int num_heads_q, int num_heads_kv, int seq_len_qo, int seq_len_kv,+ int head_dim, bool is_causal, int seq_kv_cache) {+ if (head_dim == 64) {+ if (is_causal) {+ run_fp16_prefill_impl</*HeadDim=*/64, /*Causal=*/true>(batch, num_heads_q, num_heads_kv, seq_len_qo, seq_len_kv,+ seq_kv_cache);+ } else {+ run_fp16_prefill_impl</*HeadDim=*/64, /*Causal=*/false>(batch, num_heads_q, num_heads_kv, seq_len_qo,+ seq_len_kv, seq_kv_cache);+ }+ } else if (head_dim == 128) {+ if (is_causal) {+ run_fp16_prefill_impl</*HeadDim=*/128, /*Causal=*/true>(batch, num_heads_q, num_heads_kv, seq_len_qo,+ seq_len_kv, seq_kv_cache);+ } else {+ run_fp16_prefill_impl</*HeadDim=*/128, /*Causal=*/false>(batch, num_heads_q, num_heads_kv, seq_len_qo,+ seq_len_kv, seq_kv_cache);+ }+ } else {+ printf("[sycl-tla flash_attn prefill fp16] Skip: unsupported head_dim=%d (supported 64/128)\n", head_dim);+ }+ }++ private:+ template <int HeadDim, bool Causal>+ static void run_fp16_prefill_impl(int batch, int num_heads_q, int num_heads_kv, int seq_len_qo, int seq_len_kv,+ int seq_kv_cache) {+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ compat::set_default_queue(*q);++ Options options;+ options.is_causal = Causal;+ options.varlen = false;+ options.use_paged_kv = false;+ options.scheduler = std::string("Individual");+ options.batch = batch;+ options.num_heads_q = num_heads_q;+ options.num_heads_kv = num_heads_kv;+ options.seq_len_qo = seq_len_qo;+ options.seq_len_kv = seq_len_kv;+ options.seq_len_kv_cache = seq_kv_cache;+ options.page_size = 128;+ options.head_size_qk = HeadDim;+ options.head_size_vo = HeadDim;+ // Keep runtime reasonable for larger shapes.+ if (batch >= 4 || seq_len_qo >= 2048 || seq_len_kv >= 2048) {+ options.warmup = 5;+ options.iterations = 20;+ } else {+ options.warmup = 10;+ options.iterations = 50;+ }+ options.verify = 0;+ options.softmax_scale = 1.0f;++ using namespace cute;+ using ShapeQK = Shape<_128, _64, _32>;+ using ShapePV = Shape<_128, _32, _64>;+ using ShapeOut = Shape<_128, Int<HeadDim>>;+ using SubgroupLayoutQK = std::conditional_t<HeadDim == 64, Layout<Shape<_8, _1, _1>>, Layout<Shape<_16, _1, _1>>>;++ // BMG does not support native BF16; use FP16 instead+ using ElemQ = cutlass::half_t;+ using ElemK = cutlass::half_t;+ using ElemV = cutlass::half_t;++ int rc = FMHAConfig<Causal, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK,+ /*SubgroupLayoutPV_*/ void,+ /*PipelineStages*/ 2,+ /*persistent*/ false, ElemQ, ElemK, ElemV>::run(options);+ (void)rc;++ printf("[sycl-tla flash_attn prefill fp16] B=%d Hq=%d Hkv=%d Sq=%d Skv=%d Skv_cache=%d D=%d causal=%d\n", batch,+ num_heads_q, num_heads_kv, seq_len_qo, seq_len_kv, seq_kv_cache, HeadDim, int(Causal));+ }+};++#if 0+static Benchmark_SyclTla_FlashAttnPrefill sBenchmark_SyclTla_FlashAttnPrefill;+#endif+// ========================================================================+// Flash Attention Prefill with Cached KV+// ========================================================================++class Benchmark_SyclTla_FlashAttnPrefillCachedKV {+ public:+ Benchmark_SyclTla_FlashAttnPrefillCachedKV() {+ UT_START();+ // Prefill with KV-cache (simulate attention over cached KV + current KV).+ // Keep non-causal to avoid example constraints; this is intended as a perf microbench.+ Benchmark_SyclTla_FlashAttnPrefill::run_fp16_prefill(+ /*batch=*/1, /*num_heads_q=*/32, /*num_heads_kv=*/32, /*seq_len_qo=*/128, /*seq_len_kv=*/128,+ /*head_dim=*/128, /*is_causal=*/false, /*seq_kv_cache=*/4096);+ // Larger batch to expose cache bandwidth.+ Benchmark_SyclTla_FlashAttnPrefill::run_fp16_prefill(+ /*batch=*/4, /*num_heads_q=*/32, /*num_heads_kv=*/32, /*seq_len_qo=*/128, /*seq_len_kv=*/128,+ /*head_dim=*/128, /*is_causal=*/false, /*seq_kv_cache=*/4096);+ }+};++#if 0+static Benchmark_SyclTla_FlashAttnPrefillCachedKV sBenchmark_SyclTla_FlashAttnPrefillCachedKV;+#endif++// ========================================================================+// Flash Attention Prefill FP8+// ========================================================================++class Benchmark_SyclTla_FlashAttnPrefillFP8 {+ public:+ Benchmark_SyclTla_FlashAttnPrefillFP8() {+ UT_START();+ // Kernel-only microbench for the FP8 FMHA prefill path.+ run_all(1, 32, 32, 1024, 1024, 64, false);+ run_all(1, 32, 32, 1024, 1024, 128, false);+ run_all(1, 32, 32, 2048, 2048, 128, true);+ // Larger batch/seq cases to better expose bandwidth effects.+ run_all(4, 32, 32, 1024, 1024, 128, false);+ run_all(4, 32, 32, 2048, 2048, 128, true);+ }++ private:+ template <typename T>+ static constexpr const char* dtype_str() {+ if constexpr (std::is_same_v<T, cutlass::float_e4m3_t> || std::is_same_v<T, cute::float_e4m3_t>) return "f8_e4m3";+ if constexpr (std::is_same_v<T, cutlass::float_e5m2_t> || std::is_same_v<T, cute::float_e5m2_t>) return "f8_e5m2";+ return "unknown";+ }++ template <typename KernelT>+ static sycl::event launch_prefill(typename KernelT::Params const& kernel_params) {+ compat::dim3 const block = KernelT::get_block_shape();+ compat::dim3 const grid = KernelT::get_grid_shape(kernel_params);+ int smem_size = KernelT::SharedStorageSize;++#if !defined(SYCL_EXT_ONEAPI_WORK_GROUP_SCRATCH_MEMORY)+ using namespace compat::experimental;+ return launch<cutlass::device_kernel<KernelT>>(+ launch_policy{grid, block, local_mem_size{static_cast<std::size_t>(smem_size)},+ kernel_properties{sycl_exp::sub_group_size<BestlaBenchSubgroupSize<KernelT>::value>}},+ kernel_params);+#else+ compat::experimental::launch_properties launch_props{+ sycl::ext::oneapi::experimental::work_group_scratch_size(smem_size),+ };+ compat::experimental::kernel_properties kernel_props{+ sycl::ext::oneapi::experimental::sub_group_size<BestlaBenchSubgroupSize<KernelT>::value>,+ };+ compat::experimental::launch_policy policy{grid, block, launch_props, kernel_props};+ return compat::experimental::launch<cutlass::device_kernel<KernelT>, KernelT>(policy, kernel_params);+#endif+ }++ template <typename ElementInput, int HeadDim, bool Causal>+ static void bench_one(int batch, int num_heads_q, int num_heads_kv, int seq_len_qo, int seq_len_kv,+ float softmax_scale, float timems) {+ using namespace cute;++ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ compat::set_default_queue(*q);++ // Match sycl-tla example-06 fp8 prefill defaults.+ using MMAOperation = XE_8x16x16_F32F16F16F32_TT;+ using GmemTiledCopyQ = XE_2D_U8x8x32_LD_N;+ using GmemTiledCopyK = XE_2D_U8x16x16_LD_T;+ using GmemTiledCopyV = XE_2D_U8x32x32_LD_V;+ using GmemTiledCopyO = XE_2D_U32x8x16_ST_N;++ using ElementAccumulator = float;+ using ElementComputeEpilogue = float;+ // Keep output as float for this FP8 kernel microbench; changing element size+ // affects the sycl-tla epilogue tiled store configuration.+ using ElementOutput = float;+ constexpr int PipelineStages = 2;++ using ShapeQK = Shape<_128, _64, _64>;+ using ShapePV = Shape<_128, _32, _64>;+ using ShapeOutput = std::conditional_t<HeadDim == 64, Shape<_128, _64, _64>, Shape<_128, _128, _64>>;+ using SubgroupLayout = std::conditional_t<HeadDim == 64, Layout<Shape<_8, _1, _1>, Stride<_1, _1, _1>>,+ Layout<Shape<_16, _1, _1>, Stride<_1, _1, _1>>>;++ using LayoutQ = cutlass::layout::RowMajor;+ using LayoutK = cutlass::layout::ColumnMajor;+ using LayoutV = cutlass::layout::RowMajor;+ using LayoutO = cutlass::layout::RowMajor;++ using EpilogueDispatchPolicy = cutlass::epilogue::IntelXeXMX16;+ using GEMMDispatchPolicy = cutlass::gemm::MainloopIntelXeXMX16<PipelineStages>;++ using CollectiveEpilogue = cutlass::flash_attention::collective::FlashPrefillEpilogue<+ EpilogueDispatchPolicy, MMAOperation, ShapeOutput, SubgroupLayout, ElementComputeEpilogue, ElementOutput,+ cutlass::gemm::TagToStrideC_t<LayoutO>, ElementOutput, GmemTiledCopyO>;++ using CollectiveSoftmaxEpilogue =+ cutlass::flash_attention::collective::FlashPrefillSoftmaxEpilogue<Causal, EpilogueDispatchPolicy,+ ElementAccumulator>;++ using ProblemShape = cute::tuple<int, int, int, int, int, int, int>;++ using CollectiveMainloop = cutlass::flash_attention::collective::FlashPrefillMma<+ GEMMDispatchPolicy, ProblemShape, ElementInput, cutlass::gemm::TagToStrideA_t<LayoutQ>, ElementInput,+ cutlass::gemm::TagToStrideB_t<LayoutK>, ElementInput, cutlass::gemm::TagToStrideB_t<LayoutV>, MMAOperation,+ ShapeQK, ShapePV, SubgroupLayout, GmemTiledCopyQ, GmemTiledCopyK, GmemTiledCopyV, Causal>;++ using Kernel = cutlass::flash_attention::kernel::FMHAPrefill<ProblemShape, CollectiveMainloop,+ CollectiveSoftmaxEpilogue, CollectiveEpilogue,+ cutlass::flash_attention::IndividualScheduler>;++ const size_t q_elems = size_t(seq_len_qo) * size_t(HeadDim) * size_t(batch) * size_t(num_heads_q);+ const size_t kv_elems = size_t(seq_len_kv) * size_t(HeadDim) * size_t(batch) * size_t(num_heads_kv);+ const size_t o_elems = q_elems;++ cutlass::DeviceAllocation<ElementInput> dQ(q_elems);+ cutlass::DeviceAllocation<ElementInput> dK(kv_elems);+ cutlass::DeviceAllocation<ElementInput> dV(kv_elems);+ cutlass::DeviceAllocation<ElementOutput> dO(o_elems);++ q->memset(dQ.get(), 0, q_elems * sizeof(ElementInput)).wait();+ q->memset(dK.get(), 0, kv_elems * sizeof(ElementInput)).wait();+ q->memset(dV.get(), 0, kv_elems * sizeof(ElementInput)).wait();+ q->memset(dO.get(), 0, o_elems * sizeof(ElementOutput)).wait();++ typename Kernel::StrideQ stride_Q = cutlass::make_cute_packed_stride(+ typename Kernel::StrideQ{}, make_shape(seq_len_qo, HeadDim, batch * num_heads_q));+ typename Kernel::StrideK stride_K = cutlass::make_cute_packed_stride(+ typename Kernel::StrideK{}, make_shape(seq_len_kv, HeadDim, batch * num_heads_kv));+ typename Kernel::StrideV stride_V = cutlass::make_cute_packed_stride(+ typename Kernel::StrideV{}, make_shape(HeadDim, seq_len_kv, batch * num_heads_kv));+ typename Kernel::StrideO stride_O = cutlass::make_cute_packed_stride(+ typename Kernel::StrideO{}, make_shape(seq_len_qo, HeadDim, batch * num_heads_q));++ typename Kernel::ProblemShape problem_size =+ make_tuple(batch, num_heads_q, num_heads_kv, seq_len_qo, seq_len_kv, HeadDim, HeadDim);+ cutlass::KernelHardwareInfo hw_info;+ typename Kernel::Arguments arguments{cutlass::gemm::GemmUniversalMode::kGemm,+ problem_size,+ {dQ.get(), stride_Q, dK.get(), stride_K, dV.get(), stride_V},+ {softmax_scale},+ {dO.get(), stride_O},+ hw_info};++ if (!Kernel::can_implement(arguments)) {+ printf("[sycl-tla flash_attn prefill fp8] Skip: can_implement failed\n");+ return;+ }++ size_t workspace_size = Kernel::get_workspace_size(arguments);+ cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);+ auto status = Kernel::initialize_workspace(arguments, workspace.get());+ if (status != cutlass::Status::kSuccess) {+ printf("[sycl-tla flash_attn prefill fp8] Skip: initialize_workspace failed\n");+ return;+ }++ auto params = Kernel::to_underlying_arguments(arguments, workspace.get());+ (void)launch_prefill<Kernel>(params).wait();++ using LOG = timer_statistics_logger<TestMs * 2>;+ LOG log;+ utils::timer<std::chrono::milliseconds> wall;+ wall.start();+ while (wall.stop() < timems) {+ auto ev = launch_prefill<Kernel>(params);+ ev.wait();+ log.add(event_helper::execute_time(ev) * 1000);+ if (wall.stop() >= timems) break;+ }+ log.record();++ // Print sycl-tla runner style metrics for easy comparison.+ // Runner FLOPs convention (counts both QK and PV GEMM-like work):+ // flops_full = 4 * B * Hq * Sq * Skv * D+ // For causal attention, effective work is reduced by a triangular mask.+ const double time_ms = static_cast<double>(log.avg_val);+ const double time_s = time_ms * 1e-3;+ const double sq = static_cast<double>(seq_len_qo);+ const double skv = static_cast<double>(seq_len_kv);++ double causal_ratio = 1.0;+ if constexpr (Causal) {+ // Total attended keys = sum_{i=1..Sq} min(i, Skv)+ const double denom = sq * skv;+ const double attended = (sq <= skv) ? (sq * (sq + 1.0) * 0.5) : (skv * (skv + 1.0) * 0.5 + (sq - skv) * skv);+ causal_ratio = (denom > 0.0) ? (attended / denom) : 1.0;+ }++ const double flops = 4.0 * static_cast<double>(batch) * static_cast<double>(num_heads_q) * sq * skv *+ static_cast<double>(HeadDim) * causal_ratio;++ // Bytes: count Q and O fully; count K/V proportionally to attended keys.+ const double kv_read_elems_equiv =+ static_cast<double>(batch) * static_cast<double>(num_heads_q) * skv * static_cast<double>(HeadDim);+ const double bytes = static_cast<double>(q_elems) * sizeof(ElementInput) ++ 2.0 * kv_read_elems_equiv * sizeof(ElementInput) * causal_ratio ++ static_cast<double>(o_elems) * sizeof(ElementOutput);+ const double tflops = (time_s > 0.0) ? (flops / time_s / 1.0e12) : 0.0;+ const double gbs = (time_s > 0.0) ? (bytes / time_s / 1.0e9) : 0.0;+ printf("Performance: %.3f GB/s, %.3f TFlop/s, %.4f ms (dtype=%s)\n", gbs, tflops, time_ms,+ dtype_str<ElementInput>());++ printf("[sycl-tla flash_attn prefill fp8] B=%d Hq=%d Hkv=%d Sq=%d Skv=%d D=%d causal=%d %s\n", batch, num_heads_q,+ num_heads_kv, seq_len_qo, seq_len_kv, HeadDim, int(Causal), log.get_log_str());+ }++ static void run_all(int batch, int num_heads_q, int num_heads_kv, int seq_len_qo, int seq_len_kv, int head_dim,+ bool is_causal) {+ const float softmax_scale = 1.0f;+ printf("\n[sycl-tla flash_attn prefill fp8] case: B=%d Hq=%d Hkv=%d Sq=%d Skv=%d D=%d causal=%d\n", batch,+ num_heads_q, num_heads_kv, seq_len_qo, seq_len_kv, head_dim, int(is_causal));++ // Note: Only E5M2 is supported. E4M3 is not optimized in the original sycl-tla FP8 prefill kernel+ // and causes severe register spill, resulting in ~10x slower performance.+ if (head_dim == 64) {+ if (is_causal) {+ bench_one<cute::float_e5m2_t, 64, true>(batch, num_heads_q, num_heads_kv, seq_len_qo, seq_len_kv, softmax_scale,+ float(TestMs));+ } else {+ bench_one<cute::float_e5m2_t, 64, false>(batch, num_heads_q, num_heads_kv, seq_len_qo, seq_len_kv,+ softmax_scale, float(TestMs));+ }+ } else if (head_dim == 128) {+ if (is_causal) {+ bench_one<cute::float_e5m2_t, 128, true>(batch, num_heads_q, num_heads_kv, seq_len_qo, seq_len_kv,+ softmax_scale, float(TestMs));+ } else {+ bench_one<cute::float_e5m2_t, 128, false>(batch, num_heads_q, num_heads_kv, seq_len_qo, seq_len_kv,+ softmax_scale, float(TestMs));+ }+ } else {+ printf("[sycl-tla flash_attn prefill fp8] Skip: unsupported head_dim=%d (supported 64/128)\n", head_dim);+ }+ }+};++#if 0+static Benchmark_SyclTla_FlashAttnPrefillFP8 sBenchmark_SyclTla_FlashAttnPrefillFP8;+#endif+#endif // ARK_SYCL_TLA+} // namespace sycl_ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_tla_moe_bench.cpp b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_tla_moe_bench.cppnew file mode 100644index 000000000..6ba0ff3fc--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_tla_moe_bench.cpp@@ -0,0 +1,338 @@+// MOE GEMM Benchmarks using sycl-tla+// Based on sycl-tla/examples/12_xe20_moe_gemm_cute_interface+//+// Copyright (C) 2026 Intel Corporation+// SPDX-License-Identifier: MIT++#include <stdio.h>+#include <random>+#include <type_traits>+#include "bestla_ut.h"+#include "bestla_utils.h"+#include "sycl_ut.h"+#include "sycl/sycl_utils.h"++#if defined(ARK_SYCL_TLA)++#include "cutlass/util/device_memory.h"+#include "cutlass/util/packed_stride.hpp"+#include "cutlass/util/GPU_Clock.hpp"+#include "cutlass/util/sycl_event_manager.hpp"+#include "cutlass/util/initialize_block.hpp"+#include "cutlass/util/reference/device/gemm_complex.h"+#include "cutlass/util/reference/device/tensor_compare.h"++#include "cute/util/compat.hpp"+#include <sycl/ext/intel/experimental/grf_size_properties.hpp>+#include <sycl/sycl.hpp>++#include "cute/tensor.hpp"++#include "cutlass/kernel_hardware_info.h"+#include "cutlass/platform/platform.h"+#include "cutlass/tensor_ref.h"++// MOE kernel headers from sycl-tla+#include "moe_grouped_gemm.hpp"+#include "moe_tile_scheduler.hpp"++#pragma clang diagnostic ignored "-Wpass-failed"+#pragma clang diagnostic ignored "-Wdeprecated-declarations"++namespace bestla {+using namespace ut;+using namespace utils;+using namespace sycl_utils;+namespace sycl_ut {++using namespace cute;+using namespace MoE;++int constexpr TestMs = 1000;++using ElementAccumulator = float;++// ========================================================================+// Helper to choose TiledMMA based on element types+// ========================================================================+template <class TA, class TB>+auto choose_tiled_mma(TA* A, TB* B) {+ using TA_non_CV = cutlass::platform::remove_cv_t<TA>;+ using TB_non_CV = cutlass::platform::remove_cv_t<TB>;+ auto op = XE_DPAS_TT<8, float, TA_non_CV, TB_non_CV>{};++ using WGTile = Shape<_256, _128, _32>; // 256x128 WG tile size+ using SGLayout = Layout<Shape<_8, _2, _1>, Stride<_2, _1, _0>>; // 8x2 SG tiling, n-major++ using MMA = typename TiledMMAHelper<MMA_Atom<decltype(op)>, Layout<WGTile>, SGLayout>::TiledMMA;++ return MMA{};+}++// Unique kernel name tags+template <typename, typename, typename, char, char>+class MoEGemmCuteName;++template <typename, typename, typename, char, char>+class MoEGemmCuteNameWarmup;++// ========================================================================+// MOE GEMM Launcher+// ========================================================================+template <char layoutA, char layoutB, typename ElementA, typename ElementB, typename ElementS, typename ElementD>+void MoEGEMMLauncher(const ElementA* activations, const ElementB* weights, const ElementS* scales, ElementD* outputs,+ const int gemm_n, const int gemm_k, const int* num_rows_per_expert_device,+ const int* num_tokens_per_expert_host, const int num_experts, int iterations,+ double& avg_time_ms) {+ int sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(0);+ cutlass::KernelHardwareInfo hw_info{0, sm_count};++ auto dummy_problem_shape = cute::Shape<int, int, int>{1, gemm_k, gemm_n};+ auto dummy_group_problem_shape =+ cutlass::gemm::GroupProblemShape<Shape<int, int, int>>{1, &dummy_problem_shape, nullptr};++ using TileShape = Shape<_256, _128, _32>;+ using ClusterShape = Shape<_1, _1, _1>;++ auto scheduler_params = PersistentTileSchedulerXeMoE<ProblemShape>::to_underlying_arguments(+ dummy_group_problem_shape, TileShape{}, ClusterShape{}, hw_info,+ PersistentTileSchedulerXeMoE<ProblemShape>::Arguments{1, RasterOrderOptions::AlongN});++ auto group_distribution = PersistentTileSchedulerXeMoE<ProblemShape>::get_grid_shape(+ scheduler_params, dummy_group_problem_shape, TileShape{}, ClusterShape{}, hw_info,+ PersistentTileSchedulerXeMoE<ProblemShape>::Arguments{1, RasterOrderOptions::AlongN});++ auto mma = choose_tiled_mma(activations, weights);+ auto MaxThreadsPerWorkgroup = size(mma);+ dim3 local_range{static_cast<unsigned int>(MaxThreadsPerWorkgroup), 1, 1};++ sycl::range<3> local = {local_range.z, local_range.y, local_range.x};+ sycl::range<3> groups = {group_distribution.z, group_distribution.y, group_distribution.x};+ sycl::range<3> global = {local[0] * groups[0], local[1] * groups[1], local[2] * groups[2]};++ namespace syclex = sycl::ext::oneapi::experimental;+ namespace intelex = sycl::ext::intel::experimental;++ syclex::properties kernel_props{syclex::sub_group_size<16>, intelex::grf_size<256>};+ sycl::queue Q = compat::get_default_queue();++ // Warmup+ for (int i = 0; i < 5; i++) {+ Q.parallel_for<MoEGemmCuteNameWarmup<ElementA, ElementB, ElementD, layoutA,+ layoutB>>(sycl::nd_range<3>(global, local), kernel_props, [=](auto) {+ MoE::MoEGEMM<XE_LOAD_2D<16, 32, 32, 16>, XE_LOAD_2D_VNNI<16, 32, 16, 16>, XE_STORE_2D<16, 8, 32>, 'R', 'R', 'R'>(+ activations, weights, scales, outputs, mma, num_rows_per_expert_device, num_experts, gemm_n, gemm_k,+ scheduler_params);+ }).wait();+ }++ // Timing+ GPU_Clock timer;+ timer.start();+ for (int i = 0; i < iterations; ++i) {+ Q.parallel_for<MoEGemmCuteName<ElementA, ElementB, ElementD, layoutA,+ layoutB>>(sycl::nd_range<3>(global, local), kernel_props, [=](auto) {+ MoE::MoEGEMM<XE_LOAD_2D<16, 32, 32, 16>, XE_LOAD_2D_VNNI<16, 32, 16, 16>, XE_STORE_2D<16, 8, 32>, 'R', 'R', 'R'>(+ activations, weights, scales, outputs, mma, num_rows_per_expert_device, num_experts, gemm_n, gemm_k,+ scheduler_params);+ }).wait();+ }+ avg_time_ms = timer.seconds() * 1000.0 / iterations;+}++// ========================================================================+// Verification Helper+// ========================================================================+struct VerificationHelper {+ int m = 0, n = 0, k = 0, groups;+ std::vector<typename MoE::ProblemShape::UnderlyingProblemShape> problem_sizes_host;++ void parse(const int num_experts, const int* num_tokens_per_expert_host, int moe_n, int moe_k) {+ n = moe_n;+ k = moe_k;+ groups = num_experts;+ problem_sizes_host.clear();+ problem_sizes_host.reserve(groups);+ for (int i = 0; i < groups; i++) {+ problem_sizes_host.push_back({num_tokens_per_expert_host[i], n, k});+ m += num_tokens_per_expert_host[i];+ }+ }++ std::tuple<double, double> gflops(double runtime_s) const {+ uint64_t fmas = 0;+ uint64_t bytes_loaded = 0;++ for (auto const& problem : problem_sizes_host) {+ auto M = static_cast<uint64_t>(get<0>(problem));+ auto N = static_cast<uint64_t>(get<1>(problem));+ auto K = static_cast<uint64_t>(get<2>(problem));+ fmas += M * N * K;+ bytes_loaded += 2 * (2 * M * N + N * K + M * K); // sizeof(bfloat16_t) = 2+ }++ uint64_t flop = uint64_t(2) * fmas;+ double gflop = double(flop) / double(1.0e9);+ double gbytes = double(bytes_loaded) / double(1.0e9);++ return std::make_tuple(gflop / runtime_s, gbytes / runtime_s);+ }++ template <class ElementA, class ElementB, class ElementD>+ bool verify(const ElementA* activations, const ElementB* weights, ElementD* outputs) {+ cutlass::DeviceAllocation<ElementD> output_ref;+ cutlass::DeviceAllocation<ElementD> unused_c_matrix;+ output_ref.reset(m * n);+ unused_c_matrix.reset(m * n);++ using LayoutA = cutlass::layout::RowMajor;+ using LayoutB = cutlass::layout::RowMajor;+ using LayoutC = cutlass::layout::RowMajor;+ using LayoutD = cutlass::layout::RowMajor;++ bool passed = true;+ int cumulative_sum = 0;+ for (int32_t i = 0; i < groups; ++i) {+ auto problem = problem_sizes_host.at(i);+ auto M = get<0>(problem);+ cutlass::TensorRef ref_A(const_cast<ElementA*>(activations) + cumulative_sum * k, LayoutA::packed({M, k}));+ cutlass::TensorRef ref_B(const_cast<ElementB*>(weights) + i * n * k, LayoutB::packed({k, n}));+ cutlass::TensorRef ref_C(unused_c_matrix.get() + cumulative_sum * n, LayoutC::packed({M, n}));+ cutlass::TensorRef ref_D(output_ref.get() + cumulative_sum * n, LayoutD::packed({M, n}));++ cutlass::reference::device::GemmComplex({M, n, k}, 1.0, ref_A, cutlass::ComplexTransform::kNone, ref_B,+ cutlass::ComplexTransform::kNone, 0.0, ref_C, ref_D,+ ElementAccumulator(0), 1, M * k, k * n, M * n, M * n);+ compat::wait();++ passed &= cutlass::reference::device::BlockCompareEqual(output_ref.get() + cumulative_sum * n,+ outputs + cumulative_sum * n, M * n);+ if (!passed) {+ break;+ }+ cumulative_sum += M;+ }+ return passed;+ }+};++// ========================================================================+// MOE GEMM Benchmark Class+// ========================================================================+class Benchmark_SyclTla_MoEGemm {+ public:+ Benchmark_SyclTla_MoEGemm() {+ UT_START();++ // Test cases similar to sycl-tla example+ // Typical MOE configurations: num_experts=32, N=5760/2880, K=2880+ printf("\n[sycl-tla MOE GEMM] Starting benchmarks...\n");++ // Case 1: Small experts, typical shape+ {+ std::vector<int> tokens_per_expert = {256, 256, 256, 256, 256, 256, 256, 256};+ run_moe_gemm(tokens_per_expert, 2880, 2880);+ }++ // Case 2: Medium experts+ {+ std::vector<int> tokens_per_expert = {512, 384, 640, 256, 448, 320, 576, 416};+ run_moe_gemm(tokens_per_expert, 2880, 2880);+ }++ // Case 3: Larger scale (32 experts)+ {+ std::vector<int> tokens_per_expert = {148, 231, 404, 180, 127, 244, 224, 244, 110, 617, 289,+ 845, 191, 424, 30, 97, 57, 324, 62, 77, 75, 144,+ 250, 287, 629, 370, 161, 101, 215, 113, 224, 35};+ run_moe_gemm(tokens_per_expert, 5760, 2880);+ }++ // Case 4: Same experts with different N/K+ {+ std::vector<int> tokens_per_expert = {148, 231, 404, 180, 127, 244, 224, 244, 110, 617, 289,+ 845, 191, 424, 30, 97, 57, 324, 62, 77, 75, 144,+ 250, 287, 629, 370, 161, 101, 215, 113, 224, 35};+ run_moe_gemm(tokens_per_expert, 2880, 2880);+ }+ }++ private:+ void run_moe_gemm(const std::vector<int>& tokens_per_expert, int N, int K, bool verify = false) {+ auto dev = UT_Device::get();+ auto q = dev->getQueue();+ compat::set_default_queue(*q);++ int num_experts = static_cast<int>(tokens_per_expert.size());+ int num_tokens_total = 0;+ for (int m : tokens_per_expert) {+ num_tokens_total += m;+ }++ // Calculate M-occupancy+ float actual_num_units = 0.0f;+ int total_num_M_tiles = 0;+ for (int m : tokens_per_expert) {+ total_num_M_tiles += (m + 255) / 256;+ actual_num_units += m / 256.0f;+ }+ float M_occupancy = actual_num_units / total_num_M_tiles;++ printf("\n[sycl-tla MOE GEMM] experts=%d, total_tokens=%d, N=%d, K=%d, M_occupancy=%.2f\n", num_experts,+ num_tokens_total, N, K, M_occupancy);++ // Allocate memory+ using ElementA = bfloat16_t;+ using ElementB = bfloat16_t;+ using ElementD = bfloat16_t;++ size_t A_size = static_cast<size_t>(num_tokens_total) * K;+ size_t B_size = static_cast<size_t>(num_experts) * N * K;+ size_t D_size = static_cast<size_t>(num_tokens_total) * N;++ cutlass::DeviceAllocation<int32_t> num_rows_per_expert_device;+ cutlass::DeviceAllocation<ElementA> activations_data;+ cutlass::DeviceAllocation<ElementB> weights_data;+ cutlass::DeviceAllocation<ElementD> output_data;++ num_rows_per_expert_device.reset(num_experts);+ num_rows_per_expert_device.copy_from_host(tokens_per_expert.data());+ activations_data.reset(A_size);+ weights_data.reset(B_size);+ output_data.reset(D_size);++ uint64_t seed = 2023;+ initialize_block(activations_data, seed + 2023);+ initialize_block(weights_data, seed + 2022);+ initialize_block(output_data, seed + 2021);++ int iterations = 50;+ double avg_time_ms = 0.0;++ MoEGEMMLauncher<'R', 'R'>(activations_data.get(), weights_data.get(), static_cast<void*>(nullptr),+ output_data.get(), N, K, num_rows_per_expert_device.get(), tokens_per_expert.data(),+ num_experts, iterations, avg_time_ms);++ // Verify if requested+ if (verify) {+ VerificationHelper helper;+ helper.parse(num_experts, tokens_per_expert.data(), N, K);+ bool passed = helper.verify(activations_data.get(), weights_data.get(), output_data.get());+ printf(" Verification: %s\n", passed ? "PASSED" : "FAILED");+ }++ // Calculate performance+ VerificationHelper perf_helper;+ perf_helper.parse(num_experts, tokens_per_expert.data(), N, K);+ auto [gflops, gbps] = perf_helper.gflops(avg_time_ms / 1000.0);++ printf(" Performance: %.3f GFlop/s, %.3f GB/s, %.4f ms\n", gflops, gbps, avg_time_ms);+ }+};++#if 0+static Benchmark_SyclTla_MoEGemm sBenchmark_SyclTla_MoEGemm;+#endif+#endif // ARK_SYCL_TLA+} // namespace sycl_ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_ut.h b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_ut.hnew file mode 100644index 000000000..b14b4d449--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/bestla/ut/sycl_ut.h@@ -0,0 +1,16 @@+#pragma once++#include "sycl/sycl_device.h"++namespace bestla {+namespace sycl_ut {++class UT_Device {+ public:+ static bestla::sycl_device::SyclDevice* get() {+ static bestla::sycl_device::SyclDevice Instance(true);+ return &Instance;+ }+};+}; // namespace sycl_ut+} // namespace bestladiff --git a/auto_round_extension/ark/auto_round_kernel/bestla/cmake/FindSIMD.cmake b/auto_round_extension/ark/auto_round_kernel/bestla/cmake/FindSIMD.cmakenew file mode 100644index 000000000..15cd22cfc--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/cmake/FindSIMD.cmake@@ -0,0 +1,146 @@+include(CheckCXXSourceCompiles)++# "avx2", "fma", "f16c"+set(AVX2_CODE "+ #include <immintrin.h>+ int main()+ {+ __m128i raw_data = _mm_set1_epi32(1); // sse2+ __m256i ymm0 = _mm256_cvtepu8_epi16(raw_data); // avx2+ __m256 ymm1 = _mm256_castsi256_ps(ymm0);+ ymm1 = _mm256_fmadd_ps(ymm1, ymm1, ymm1); // fma+ __m128i xmm = _mm256_cvtps_ph(ymm1, _MM_FROUND_TO_NEAREST_INT); // f16c+ return 0;+ }+")++set(AVX2_FLAGS "-mavx2 -mfma -mf16c")++# "avx512f", "avx512bw", "avx512vl", "avx512dq"+set(AVX512_CODE "+ #include <immintrin.h>+ int main()+ {+ __m512i a = _mm512_set1_epi32(1); // avx512f+ a = _mm512_abs_epi16(a); // avx512bw+ __m512 b = _mm512_castsi512_ps(a);+ b = _mm512_and_ps(b, b); // avx512dq+ __m256 c = _mm512_castps512_ps256(b);+ __m256i d = _mm256_castps_si256(c);+ d = _mm256_abs_epi64(d);+ return 0;+ }+")++set(AVX512_FLAGS "-mavx512f -mavx512bw -mavx512vl -mavx512dq")++# "avx512_vnni"+set(AVX512_VNNI_CODE "+ #include <immintrin.h>+ int main()+ {+ __m512i a = _mm512_set1_epi32(1); // avx512f+ a = _mm512_dpbusds_epi32(a, a, a); // avx512_vnni+ return 0;+ }+")+set(AVX512_VNNI_FLAGS "-mavx512f -mavx512vnni")++# "avx512_vnni"+set(AVX_VNNI_CODE "+ #include <immintrin.h>+ int main()+ {+ __m256i a = _mm256_set1_epi32(1); // avx+ a = _mm256_dpbusd_avx_epi32(a, a, a); // avx_vnni+ return 0;+ }+")+set(AVX_VNNI_FLAGS "-mavx -mavxvnni")++# "amx_tile", "amx_bf16"+set(AMX_BF16_CODE "+ #include <immintrin.h>+ int main()+ {+ _tile_dpbf16ps(0, 1, 2); // amx_bf16+ return 0;+ }+")+set(AMX_BF16_FLAGS " ")++# "amx_tile", "amx_int8"+set(AMX_INT8_CODE "+ #include <immintrin.h>+ int main()+ {+ _tile_dpbusd(0, 1, 2); // amx_bf16+ return 0;+ }+")+set(AMX_INT8_FLAGS " ")++# "amx_tile", "amx_fp16"+set(AMX_FP16_CODE "+ #include <immintrin.h>+ int main()+ {+ _tile_dpfp16ps(0, 1, 2); // amx_bf16+ return 0;+ }+")+set(AMX_FP16_FLAGS " ")++# "avx512_fp16"+set(AVX512_FP16_CODE "+ #include <immintrin.h>+ int main()+ {+ char tmp[512];+ __m256h a = _mm256_loadu_ph(tmp); // avx512vl + avx512fp16+ __m512 b = _mm512_cvtxph_ps(a); // avx512fp16+ a = _mm512_cvtxps_ph(b); // avx512fp16+ _mm256_storeu_ph(tmp,a);+ return 0;+ }+")+set(AVX512_FP16_FLAGS "-mavx512f -mavx512vl -mavx512fp16")++# "avx512_bf16"+set(AVX512_BF16_CODE "+ #include <immintrin.h>+ int main()+ {+ __m512 a = _mm512_set1_ps(1.f); // avx512f+ __m256bh b = _mm512_cvtneps_pbh(a); // avx512_bf16 + AVX512F+ return 0;+ }+")+set(AVX512_BF16_FLAGS "-mavx512f -mavx512bf16")++macro(check_isa type)+ set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS})+ if (NOT ${type}_FOUND)+ set(CMAKE_REQUIRED_FLAGS ${${type}_FLAGS})+ check_cxx_source_compiles("${${type}_CODE}" HAS_${type})+ if (HAS_${type})+ set(${type}_FOUND TRUE CACHE BOOL "${type} support")+ set(${type}_FLAGS "${${type}_FLAGS}" CACHE STRING "${type} flags")+ endif()+ endif()+ set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE})++ if (NOT ${type}_FOUND)+ set(${type}_FOUND FALSE CACHE BOOL "${type} support")+ set(${type}_FLAGS "" CACHE STRING "${type} flags")+ endif()++ mark_as_advanced(${type}_FOUND ${type}_FLAGS)+endmacro()++set(ISA_SET AVX2 AVX512 AVX512_VNNI AVX_VNNI AVX512_BF16 AVX512_FP16 AMX_BF16 AMX_INT8 AMX_FP16)+foreach (ISA ${ISA_SET})+ check_isa(${ISA})+endforeach()++diff --git a/auto_round_extension/ark/auto_round_kernel/bestla/cmake/sycl.cmake b/auto_round_extension/ark/auto_round_kernel/bestla/cmake/sycl.cmakenew file mode 100644index 000000000..ac041982a--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/bestla/cmake/sycl.cmake@@ -0,0 +1,5 @@+cmake_minimum_required(VERSION 3.23)++find_package(IntelSYCL REQUIRED)++# set(CMAKE_CXX_COMPILER ${SYCL_COMPILER})diff --git a/auto_round_extension/ark/auto_round_kernel/qlinear.py b/auto_round_extension/ark/auto_round_kernel/qlinear.pynew file mode 100644index 000000000..125e77e90--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/qlinear.py@@ -0,0 +1,554 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import math++import torch+import torch.nn as nn++try:+ import auto_round_kernel++ ARK_INSTALLED = True+ ark = auto_round_kernel.ARK()+except:+ ARK_INSTALLED = False++BITS_DTYPE_MAPPING = {+ 2: "int2",+ 4: "int4",+ 8: "int8",+}++FP8_DTYPE_MAPPING = {+ "fp8_e4m3": "fp8_e4m3",+ "fp8_e5m2": "fp8_e5m2",+}++AWQ_REVERSE_ORDER = [0, 4, 1, 5, 2, 6, 3, 7]+++def unpack_awq(qweight: torch.Tensor, qzeros: torch.Tensor, bits: int):+ shifts = torch.arange(0, 32, bits, device=qzeros.device)++ # unpacking columnwise+ iweights = torch.bitwise_right_shift(qweight[:, :, None], shifts[None, None, :]).to(+ torch.int8 # smallest dtype available+ )+ iweights = iweights.view(iweights.shape[0], -1)++ # unpacking columnwise+ if qzeros is not None:+ izeros = torch.bitwise_right_shift(qzeros[:, :, None], shifts[None, None, :]).to(+ torch.int8 # smallest dtype available+ )+ izeros = izeros.view(izeros.shape[0], -1)+ else:+ izeros = qzeros++ return iweights, izeros+++def reverse_awq_order(iweights: torch.Tensor, izeros: torch.Tensor, bits: int):+ reverse_order_tensor = torch.arange(+ iweights.shape[-1],+ dtype=torch.int32,+ device=izeros.device,+ )+ reverse_order_tensor = reverse_order_tensor.view(-1, 32 // bits)+ reverse_order_tensor = reverse_order_tensor[:, AWQ_REVERSE_ORDER]+ reverse_order_tensor = reverse_order_tensor.view(-1)++ if izeros is not None:+ izeros = izeros[:, reverse_order_tensor]+ iweights = iweights[:, reverse_order_tensor]++ return iweights, izeros+++def convert_dtype_torch2str(dtype):+ if dtype == torch.int8:+ return "int8"+ elif dtype == torch.float:+ return "fp32"+ elif dtype == torch.float16:+ return "fp16"+ elif dtype == torch.bfloat16:+ return "bf16"+ elif isinstance(dtype, str) and dtype in ["int8", "fp32", "fp16", "bf16"]:+ return dtype+ else:+ assert False, "Unsupported pytorch dtype {} to str dtype".format(dtype)+++class QuantLinear(nn.Module):+ QUANT_TYPE = "ark_gptq_nozp"+ ZP_BIAS = 0++ def __init__(+ self,+ bits,+ group_size,+ sym,+ in_features,+ out_features,+ bias,+ weight_dtype=torch.bfloat16,+ **kwargs,+ ):+ super().__init__()+ if bits not in [2, 4, 8]:+ raise NotImplementedError("Only 2, 4, 8 bits are supported for ARK.")+ if not ARK_INSTALLED:+ raise ModuleNotFoundError(+ "The 'auto_round_kernel' module is required but not installed. "+ "Please install the 'auto-round-lib' package, for example:\n"+ " pip install auto-round-lib"+ )++ self.infeatures = in_features+ self.outfeatures = out_features+ self.bits = bits+ self.group_size = group_size if group_size != -1 else self.infeatures+ self.maxq = 2**self.bits - 1+ self.qbias = 2 ** (self.bits - 1)+ self.pack_num = 32 // self.bits+ self.weight_dtype = weight_dtype+ self.asym = not sym+ if not self.infeatures % self.group_size == 0:+ raise NotImplementedError("in_features must be divisible by group_size")+ if "awq" in self.QUANT_TYPE:+ self.register_buffer(+ "qweight", torch.zeros((in_features, out_features // self.pack_num), dtype=torch.int32)+ )+ else:+ self.register_buffer(+ "qweight",+ torch.zeros((self.infeatures // 32 * self.bits, self.outfeatures), dtype=torch.int32),+ )+ self.register_buffer(+ "qzeros",+ torch.zeros(+ (+ math.ceil(self.infeatures / self.group_size),+ self.outfeatures // 32 * self.bits,+ ),+ dtype=torch.int32,+ ),+ )+ self.register_buffer(+ "scales",+ torch.zeros(+ (math.ceil(self.infeatures / self.group_size), self.outfeatures),+ dtype=weight_dtype,+ ),+ )+ if bias:+ self.register_buffer("bias", torch.zeros((self.outfeatures), dtype=torch.float))+ else:+ self.bias = None++ def extra_repr(self) -> str:+ return "in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(+ self.infeatures,+ self.outfeatures,+ self.bias is not None,+ self.bits,+ self.group_size,+ )++ def post_init(self):+ if self.qweight.device.type not in ["cpu", "xpu"]:+ raise NotImplementedError(+ f"Device type {self.qweight.device.type} is not supported. Only CPU and XPU devices are supported."+ )+ if self.qweight.device.type != "cpu" and self.asym:+ raise NotImplementedError("Asymmetric quantization is only supported on CPU devices")+ if "awq" in self.QUANT_TYPE:+ intweight, zeros = unpack_awq(+ self.qweight, self.qzeros, self.bits+ ) # weight: k x n zeros: k / group_size x n+ intweight, zeros = reverse_awq_order(intweight, zeros, self.bits) # weight: k x n zeros: k / group_size x n+ intweight = torch.bitwise_and(intweight, self.maxq)+ zeros = torch.bitwise_and(zeros, self.maxq)++ intweight = intweight - self.qbias # from uint8_t to int8_t+ zeros = zeros - self.qbias # from uint8_t to int8_t+ else:+ # intweight: k x n, zeros: k / group_size x n+ intweight, zeros = unpack_to_8bit_signed(self.qweight, self.qzeros, self.bits, self.ZP_BIAS, self.asym)+ if zeros is None:+ zeros = torch.empty(0, dtype=torch.int8)+ else:+ # change it to int8 with offset 128+ zeros = (zeros.to(torch.int32) - self.qbias).to(torch.int8)+ if self.asym:+ intweight = (intweight.to(torch.int32) - self.qbias).to(torch.int8)+ else:+ intweight -= self.qbias+ intweight = intweight.to(torch.int8)++ if self.qweight.device.type == "xpu":+ self.sdt = "fp16"+ self.cdt = "int8"+ self.torch_dt = torch.float16+ else:+ self.sdt = "fp32"+ self.cdt = "auto"+ self.torch_dt = torch.float32++ self.wdt = BITS_DTYPE_MAPPING[self.bits]+ scales = self.scales.to(self.torch_dt).contiguous()++ self.qweight = ark.repack_quantized_weight(+ intweight.contiguous(),+ scales,+ zeros.contiguous(),+ self.group_size,+ # compute_dtype+ self.cdt,+ # weight_dtype+ self.wdt,+ # scale_dtype+ self.sdt,+ self.asym,+ )++ # free mem+ self.qzeros = torch.empty(0)+ self.scales = torch.empty(0)+ if self.bias is not None:+ self.bias = self.bias.to(self.torch_dt)+ else:+ self.bias = torch.empty(0)++ def forward(self, x: torch.Tensor):+ raw_input_dtype = x.dtype+ x = x.to(self.torch_dt)+ out_shape = x.shape[:-1] + (self.outfeatures,)+ x = x.view(-1, x.shape[-1])+ self.bias = self.bias.to(self.torch_dt)+ outputs = ark.woqgemm(+ x, # convert xd to 2d,+ self.qweight,+ self.bias,+ self.outfeatures,+ self.infeatures,+ self.group_size,+ self.cdt, # compute_dtype+ self.wdt, # weight_dtype+ self.sdt, # scale_dtype+ self.asym,+ )+ return outputs.to(raw_input_dtype).view(out_shape)+++class QuantLinearGPTQ(QuantLinear):+ QUANT_TYPE = "ark_gptq"+ ZP_BIAS = 1+++class QuantLinearAWQ(QuantLinear):+ QUANT_TYPE = "ark_awq"++[email protected]_grad()+def unpack_to_8bit_signed(qweight, qzeros, bits, gptq_bias, asym):+ wf = torch.tensor(list(range(0, 32, bits)), dtype=torch.int32, device=qweight.device).unsqueeze(0)+ zeros = None+ if asym:+ zp_shape = list(qzeros.shape)+ zp_shape[1] = zp_shape[1] * (32 // bits)++ zeros = torch.bitwise_right_shift(torch.unsqueeze(qzeros, 2).expand(-1, -1, 32 // bits), wf.unsqueeze(0)).to(+ torch.int16 if bits == 8 else torch.int8+ )+ torch.bitwise_and(zeros, (2**bits) - 1, out=zeros)+ if bits == 8:+ zeros = zeros.to(torch.uint8)+ zeros += gptq_bias+ try:+ zeros = zeros.reshape(zp_shape)+ except:+ # zeros and scales have different iteam numbers.+ # remove 1 (due to 0 + 1 in line 252)+ zeros = zeros[zeros != 1]+ zeros = zeros.reshape(zp_shape)+ weight = torch.bitwise_right_shift(torch.unsqueeze(qweight, 1).expand(-1, 32 // bits, -1), wf.unsqueeze(-1)).to(+ torch.int16 if bits == 8 else torch.int8+ )+ weight.bitwise_and_((2**bits) - 1)+ weight = weight.view(-1, weight.shape[-1])++ return weight, zeros+++# Copied from qlinear_marlin.py[email protected]_grad()+def dequantize_weight(qweight, qzeros, scales, bits):+ unpacked_qweight, unpacked_qzeros = unpack_to_8bit_signed(qweight, qzeros, bits)+ group_size = unpacked_qweight.shape[0] // scales.shape[0]+ scales = scales.repeat_interleave(group_size, dim=0)+ if unpacked_qzeros is not None:+ unpacked_qzeros = unpacked_qzeros.repeat_interleave(group_size, dim=0)+ else:+ unpacked_qzeros = torch.full_like(scales, 8 if bits == 4 else 128, dtype=torch.int32, device=qweight.device)+ unpacked_qweight = (unpacked_qweight - unpacked_qzeros) * scales++ return unpacked_qweight, unpacked_qzeros+++def ark_post_init(model):+ for _, submodule in model.named_modules():+ if isinstance(submodule, (QuantLinear, QuantLinearFP8)):+ submodule.post_init()++ return model+++class QuantLinearFP8(nn.Module):+ """Quantized Linear layer using FP8 format with ARK kernel."""++ QUANT_TYPE = "ark_fp8"++ def __init__(+ self,+ bits,+ group_size,+ infeatures,+ outfeatures,+ bias,+ kernel_switch_threshold=128,+ trainable=False,+ weight_dtype=torch.float16,+ data_type="fp8_e4m3",+ **kwargs,+ ):+ super().__init__()++ data_type = str(data_type)+ self.original_data_type = data_type+ self.is_mx = "mx_fp" in data_type.lower()+ # Normalize to the underlying FP8 weight format. MXFP8 stores weights in+ # FP8 (E4M3/E5M2) but uses E8M0 exponent-only scales.+ if self.is_mx:+ if "e5m2" in data_type.lower():+ data_type = "fp8_e5m2"+ else:+ data_type = "fp8_e4m3"+ else:+ # Support both fp8_e4m3 and fp8_e5m2 format names+ if data_type not in ["fp8_e4m3", "fp8_e5m2", "fp8"]:+ data_type = "fp8_e4m3" # Default to E4M3+ if data_type == "fp8":+ data_type = "fp8_e4m3"++ assert ARK_INSTALLED, "Please install auto_round_kernel."++ self.infeatures = infeatures+ self.outfeatures = outfeatures+ self.bits = 8 # FP8 is always 8-bit+ self.group_size = group_size if group_size != -1 else infeatures+ self.weight_dtype = weight_dtype+ self.data_type = data_type+ if hasattr(ark, "set_threads"):+ ark.set_threads(torch.get_num_threads())++ # FP8 weights - load from safetensors with correct FP8 dtype+ # Shape is [outfeatures, infeatures] to match standard Linear layer+ # IMPORTANT: Must use float8 dtype to match safetensors, not uint8+ fp8_dtype = torch.float8_e5m2 if data_type == "fp8_e5m2" else torch.float8_e4m3fn+ self.register_buffer(+ "weight",+ torch.zeros((outfeatures, infeatures), dtype=fp8_dtype),+ )++ # Scales for dequantization - load from safetensors as "weight_scale"+ # Shape is [outfeatures, num_groups] to match per-channel quantization+ # where num_groups = ceil(infeatures / group_size)+ scale_storage_dtype = torch.uint8 if self.is_mx else weight_dtype+ self.register_buffer(+ "weight_scale",+ torch.zeros(+ (outfeatures, math.ceil(infeatures / self.group_size)),+ dtype=scale_storage_dtype,+ ),+ )+ if bias:+ self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float))+ else:+ self.bias = None++ self.kernel_switch_threshold = kernel_switch_threshold+ self.trainable = trainable++ def _load_from_state_dict(+ self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs+ ):+ """Override to handle weight_scale shape mismatch between checkpoint and model."""+ weight_scale_key = prefix + "weight_scale"+ if weight_scale_key in state_dict:+ loaded_scale = state_dict[weight_scale_key]+ expected_shape = self.weight_scale.shape # [outfeatures, num_groups]+ out_features, num_groups = expected_shape+ expected_numel = out_features * num_groups++ if loaded_scale.ndim == 2 and loaded_scale.shape == (num_groups, out_features):+ state_dict[weight_scale_key] = loaded_scale.t().contiguous()+ loaded_scale = state_dict[weight_scale_key]++ # If loaded scale is 1D but we expect 2D, reshape or broadcast it.+ if loaded_scale.ndim == 1 and len(expected_shape) == 2:+ numel = loaded_scale.numel()+ if numel == expected_numel:+ state_dict[weight_scale_key] = loaded_scale.view(expected_shape)+ elif numel == out_features:+ expanded = loaded_scale.view(out_features, 1).expand(-1, num_groups)+ state_dict[weight_scale_key] = expanded.to(self.weight_scale.dtype)+ elif numel == num_groups:+ expanded = loaded_scale.view(1, num_groups).expand(out_features, -1)+ state_dict[weight_scale_key] = expanded.to(self.weight_scale.dtype)+ elif numel % out_features == 0:+ inferred_num_groups = numel // out_features+ inferred_shape = (out_features, inferred_num_groups)+ state_dict[weight_scale_key] = loaded_scale.view(inferred_shape)+ self.weight_scale = torch.zeros(+ inferred_shape, dtype=self.weight_scale.dtype, device=self.weight_scale.device+ )++ else:+ print(+ f"Cannot reshape scale! loaded_scale.numel()={numel}, expected={expected_numel}, out_features={out_features}"+ )+ super()._load_from_state_dict(+ state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs+ )++ def post_init(self):+ target_fp8_dtype = torch.float8_e5m2 if self.data_type == "fp8_e5m2" else torch.float8_e4m3fn+ weight_tensor = self.weight+ converted_via_cpu = False+ if weight_tensor.dtype != target_fp8_dtype:+ weight_cpu = weight_tensor.detach().to("cpu")+ fp8_cpu = weight_cpu.to(target_fp8_dtype)+ converted_via_cpu = True+ else:+ fp8_cpu = weight_tensor.detach()+ if weight_tensor.device.type != "cpu":+ fp8_cpu = fp8_cpu.to("cpu")+ converted_via_cpu = True+ fp8_cpu = fp8_cpu.contiguous().view(self.outfeatures, self.infeatures)++ fp8_uint8_cpu = fp8_cpu.view(torch.uint8)+ if converted_via_cpu and self.weight.device.type != "cpu":+ fp8_uint8 = fp8_uint8_cpu.to(self.weight.device)+ else:+ fp8_uint8 = fp8_uint8_cpu+ fp8_weight = fp8_uint8.view(self.outfeatures, self.infeatures).t().contiguous()+ zeros = torch.empty(0, dtype=torch.uint8, device=fp8_weight.device) # No zero points for FP8++ # Compute type selection (independent of how scales are stored).+ if self.weight.device.type == "xpu":+ self.cdt = "fp16"+ else:+ self.cdt = "fp32"++ if self.is_mx:+ # MXFP8 stores per-group scales as E8M0 exponents in uint8 with bias 127.+ # ARK expects fp8_e8m0 scales as float32 exponents (int-valued), which will+ # be packed to bestla::utils::f8 (int8 exponent) internally.+ self.sdt = "fp8_e8m0"+ if self.weight_scale.dtype == torch.uint8:+ exponents = self.weight_scale.to(torch.int16) - 127+ else:+ # Fallback: allow feeding exponents directly.+ exponents = self.weight_scale.to(torch.int16)+ exponents = exponents.clamp(-127, 127)+ scales = exponents.t().to(torch.float32).contiguous() # [num_groups,out]+ else:+ if self.weight.device.type == "xpu":+ self.sdt = "fp16"+ scales = self.weight_scale.t().to(torch.float16).contiguous() # [num_groups,out]+ else:+ self.sdt = "fp32"+ scales = self.weight_scale.t().float().contiguous() # [num_groups,out]++ self.wdt = FP8_DTYPE_MAPPING[self.data_type]+ # Repack weights using ARK kernel+ packed_weight = ark.repack_quantized_weight(+ fp8_weight,+ scales,+ zeros.contiguous(),+ torch.empty(0, dtype=torch.int32, device=fp8_weight.device),+ self.cdt, # compute_dtype+ self.wdt, # weight_dtype (fp8_e4m3 or fp8_e5m2)+ self.sdt, # scale_dtype+ False, # asym = False for FP8+ self.group_size,+ )+ # Replace weight buffer with packed weight+ self.weight = packed_weight+ # Free original weight_scale+ self.weight_scale = torch.empty(0)+ if self.bias is not None:+ if self.bias.device.type == "cpu":+ self.bias = self.bias.to(torch.float32)+ else:+ self.bias = self.bias.to(torch.float16)++ def forward(self, x: torch.Tensor):+ raw_input_dtype = x.dtype+ if x.device.type == "cpu":+ odt = torch.float32+ if raw_input_dtype != torch.float32:+ x = x.to(torch.float32)+ else:+ odt = x.dtype+ out_shape = x.shape[:-1] + (self.outfeatures,)+ x = x.view(-1, x.shape[-1]) # convert xd to 2d+ out_2d_shape = x.shape[:-1] + (self.outfeatures,)+ outputs = torch.empty(out_2d_shape, device=x.device, dtype=odt)+ if self.bias is None:+ bias = torch.empty(0, device=x.device, dtype=odt)+ else:+ bias = self.bias+ if bias.device != x.device:+ bias = bias.to(x.device)+ if bias.dtype != odt:+ bias = bias.to(odt)+ ark.woq_linear(+ x,+ self.weight,+ bias,+ outputs,+ self.cdt, # compute_dtype+ self.wdt, # weight_dtype+ self.sdt, # scale_dtype+ False, # asym+ self.group_size,+ )+ return outputs.to(raw_input_dtype).view(out_shape)++ def extra_repr(self) -> str:+ return "in_features={}, out_features={}, bias={}, bits={}, " "group_size={}, data_type={}".format(+ self.infeatures,+ self.outfeatures,+ self.bias is not None,+ self.bits,+ self.group_size,+ self.data_type,+ )+++__all__ = ["QuantLinear", "QuantLinearGPTQ", "QuantLinearAWQ", "QuantLinearFP8"]diff --git a/auto_round_extension/ark/auto_round_kernel/sdpa.cpp b/auto_round_extension/ark/auto_round_kernel/sdpa.cppnew file mode 100644index 000000000..990c1c9d4--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/sdpa.cpp@@ -0,0 +1,247 @@+// Copyright (c) 2026 Intel Corporation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++// Flash Attention Prefill implementation+// Separated from decode to avoid convert_type ODR violation in sycl-tla++#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)++#include <cute/numeric/int.hpp>+#include <cute/numeric/numeric_types.hpp>+#include <sycl/aliases.hpp>+#include "bestla/bestla.h"++#include <sycl/sycl.hpp>+#include "sycl_tla_sdpa.hpp"+#include "sdpa_kernel_declarations.hpp" // generated from CMake, contains declarations for all kernel launchers++namespace ark {++namespace {++using KernelLauncher = int (*)(detail::Options const& options);++int launch_prefill_kernel_f16_128_sage(detail::Options const& options) {+ return launch_sage_prefill_kernel_128<cute::int8_t, cute::int8_t, cute::half_t>(options);+}++int launch_prefill_kernel_f16_64_sage(detail::Options const& options) {+ return launch_sage_prefill_kernel_64<cute::int8_t, cute::int8_t, cute::half_t>(options);+}++KernelLauncher select_sage_prefill_launcher(BTLA_DTYPE dtype, int head_dim) {+ switch (dtype) {+ case BTLA_DTYPE::S8:+ switch (head_dim) {+ case 128:+ return launch_prefill_kernel_f16_128_sage;+ case 64:+ return launch_prefill_kernel_f16_64_sage;+ default:+ return nullptr;+ }++ default:+ return nullptr;+ }+}+++KernelLauncher select_prefill_launcher(BTLA_DTYPE dtype, int head_dim) {+ switch (dtype) {+ case BTLA_DTYPE::F16:+ switch (head_dim) {+ case 128:+ return detail::launch_prefill_kernel_f16_128;+ case 64:+ return detail::launch_prefill_kernel_f16_64;+ case 96:+ return detail::launch_prefill_kernel_f16_96;+ case 192:+ return detail::launch_prefill_kernel_f16_192;+ default:+ return nullptr;+ }+ case BTLA_DTYPE::BF16:+ switch (head_dim) {+ case 64:+ return detail::launch_prefill_kernel_bf16_64;+ case 96:+ return detail::launch_prefill_kernel_bf16_96;+ case 128:+ return detail::launch_prefill_kernel_bf16_128;+ case 192:+ return detail::launch_prefill_kernel_bf16_192;+ default:+ return nullptr;+ }+ default:+ return nullptr;+ }+}++KernelLauncher select_decode_launcher(BTLA_DTYPE dtype, int head_dim) {+ switch (dtype) {+ case BTLA_DTYPE::F16:+ switch (head_dim) {+ case 128:+ return detail::launch_decode_kernel_f16_128;+ case 64:+ return detail::launch_decode_kernel_f16_64;+ case 96:+ return detail::launch_decode_kernel_f16_96;+ case 192:+ return detail::launch_decode_kernel_f16_192;+ default:+ return nullptr;+ }+ case BTLA_DTYPE::BF16:+ switch (head_dim) {+ case 64:+ return detail::launch_decode_kernel_bf16_64;+ case 96:+ return detail::launch_decode_kernel_bf16_96;+ case 128:+ return detail::launch_decode_kernel_bf16_128;+ case 192:+ return detail::launch_decode_kernel_bf16_192;+ default:+ return nullptr;+ }+ default:+ return nullptr;+ }+}++detail::Options make_common_options(void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int batch,+ int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,+ float softmax_scale, bool is_causal) {+ detail::Options options;+ options.q = Q_ptr;+ options.k = K_ptr;+ options.v = V_ptr;+ options.mask = mask;+ options.o = O_ptr;+ options.batch = batch;+ options.num_heads_q = num_heads_q;+ options.num_heads_kv = num_heads_kv;+ options.seq_len_qo = seq_len_q;+ options.seq_len_kv = seq_len_kv;+ options.head_size_qk = head_dim;+ options.head_size_vo = head_dim;+ options.softmax_scale = softmax_scale;+ options.is_causal = is_causal;+ return options;+}++} // namespace++void sage_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int scale_block_size, void* qscale,+ void* kscale, BTLA_DTYPE q_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {+ detail::Options options = make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, batch, num_heads_q, num_heads_kv,+ seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+ options.scale_block_size = scale_block_size;+ options.qscale = qscale;+ options.kscale = kscale;+ compat::set_default_queue(*q);++ KernelLauncher launcher = select_sage_prefill_launcher(q_dtype, head_dim);+ if (launcher == nullptr) {+ throw std::runtime_error(+ "Unsupported dtype or head dimension for sage_prefill / SAGE (only F16/BF16 and 64/96/128/192 are supported)");+ }++ launcher(options);+}++void flash_attn_prefill(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,+ BTLA_DTYPE q_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,+ int head_dim, float softmax_scale, bool is_causal) {+ detail::Options options = make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, batch, num_heads_q, num_heads_kv,+ seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+ compat::set_default_queue(*q);++ KernelLauncher launcher = select_prefill_launcher(q_dtype, head_dim);+ if (launcher == nullptr) {+ throw std::runtime_error(+ "Unsupported dtype or head dimension for flash_attn_prefill (only F16/BF16 and 64/96/128/192 are supported)");+ }++ launcher(options);+}++void flash_attn_decode(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,+ BTLA_DTYPE q_dtype, int batch, int num_heads_q, int num_heads_kv, int seq_len_kv, int head_dim,+ float softmax_scale, bool is_causal) {+ detail::Options options = make_common_options(Q_ptr, K_ptr, V_ptr, O_ptr, mask, batch, num_heads_q,+ num_heads_kv, 1, seq_len_kv, head_dim, softmax_scale, is_causal);+ compat::set_default_queue(*q);++ KernelLauncher launcher = select_decode_launcher(q_dtype, head_dim);+ if (launcher == nullptr) {+ throw std::runtime_error(+ "Unsupported dtype or head dimension for flash_attn_decode (only F16/BF16 and 64/96/128/192 are supported)");+ }++ launcher(options);+}++void sdpa_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, BTLA_DTYPE q_dtype,+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,+ float softmax_scale, bool is_causal) {+ // if (q_dtype != BTLA_DTYPE::F16 && q_dtype != BTLA_DTYPE::BF16) {+ // throw std::invalid_argument("sdpa_impl: only FP16 and BF16 are supported");+ // }+ if (mask && is_causal) {+ throw std::invalid_argument("sdpa_impl: mask and is_causal cannot both be set");+ }++ if (seq_len_q <= 0 || seq_len_kv <= 0) {+ throw std::invalid_argument("sdpa_impl: seq_len_q and seq_len_kv must be greater than 0");+ }+ if (seq_len_q == 1) {+ flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_dtype, batch, num_heads_q, num_heads_kv, seq_len_kv,+ head_dim, softmax_scale, is_causal);+ return;+ }++ flash_attn_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, q_dtype, batch, num_heads_q, num_heads_kv, seq_len_q,+ seq_len_kv, head_dim, softmax_scale, is_causal);+}++void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int scale_block_size, void* qscale,+ void* kscale, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal) {+ if (mask && is_causal) {+ throw std::invalid_argument("sdpa_impl: mask and is_causal cannot both be set");+ }++ if (seq_len_q <= 0 || seq_len_kv <= 0) {+ throw std::invalid_argument("sdpa_impl: seq_len_q and seq_len_kv must be greater than 0");+ }+ if (seq_len_q == 1) {+ flash_attn_decode(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, BTLA_DTYPE::F16, batch, num_heads_q, num_heads_kv, seq_len_kv,+ head_dim, softmax_scale, is_causal);+ return;+ }+++ sage_prefill(q, Q_ptr, K_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale, BTLA_DTYPE::S8, batch, num_heads_q, num_heads_kv, seq_len_q,+ seq_len_kv, head_dim, softmax_scale, is_causal);+}++} // namespace ark++#endif // ARK_XPU && ARK_SYCL_TLAdiff --git a/auto_round_extension/ark/auto_round_kernel/sdpa_generation.cmake b/auto_round_extension/ark/auto_round_kernel/sdpa_generation.cmakenew file mode 100644index 000000000..051de0c68--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/sdpa_generation.cmake@@ -0,0 +1,33 @@+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/generated/sdpa)+set(SDPA_DECLARATIONS_FILE ${CMAKE_CURRENT_BINARY_DIR}/generated/sdpa/sdpa_kernel_declarations.hpp)++function(generate_sdpa_instantiation mode dtype_name dtype_type head_dim template_name function_name)+ set(SDPA_FUNCTION_NAME ${function_name})+ set(SDPA_TEMPLATE_NAME ${template_name})+ set(SDPA_ELEMENT_TYPE ${dtype_type})+ string(APPEND SDPA_KERNEL_DECLARATIONS "int ${function_name}(Options const& options);\n")+ set(SDPA_KERNEL_DECLARATIONS ${SDPA_KERNEL_DECLARATIONS} PARENT_SCOPE)+ set(_generated_src+ ${CMAKE_CURRENT_BINARY_DIR}/generated/sdpa/${mode}_${dtype_name}_${head_dim}.cpp)+ configure_file(${CMAKE_CURRENT_LIST_DIR}/sdpa_kernel_instantiation.cpp.in ${_generated_src} @ONLY)+ list(APPEND SDPA_GENERATED_SRCS ${_generated_src})+ set(SDPA_GENERATED_SRCS ${SDPA_GENERATED_SRCS} PARENT_SCOPE)+endfunction()++foreach(mode IN ITEMS prefill decode)+ foreach(dtype_name IN ITEMS f16 bf16)+ if(dtype_name STREQUAL "f16")+ set(dtype_type "cute::half_t")+ else()+ set(dtype_type "cute::bfloat16_t")+ endif()++ foreach(head_dim IN ITEMS 128 64 96 192)+ set(template_name launch_${mode}_kernel_${head_dim})+ set(function_name launch_${mode}_kernel_${dtype_name}_${head_dim})+ generate_sdpa_instantiation(${mode} ${dtype_name} "${dtype_type}" ${head_dim} ${template_name} ${function_name})+ endforeach()+ endforeach()+endforeach()++configure_file(${CMAKE_CURRENT_LIST_DIR}/sdpa_kernel_declarations.hpp.in ${SDPA_DECLARATIONS_FILE} @ONLY)\ No newline at end of filediff --git a/auto_round_extension/ark/auto_round_kernel/sdpa_kernel_declarations.hpp.in b/auto_round_extension/ark/auto_round_kernel/sdpa_kernel_declarations.hpp.innew file mode 100644index 000000000..e2c45bc13--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/sdpa_kernel_declarations.hpp.in@@ -0,0 +1,15 @@+// Generated by CMake. Do not edit directly.++#pragma once++#if defined(ARK_SYCL_TLA)++namespace ark {+namespace detail {++@SDPA_KERNEL_DECLARATIONS@++} // namespace detail+} // namespace ark++#endif // ARK_SYCL_TLA\ No newline at end of filediff --git a/auto_round_extension/ark/auto_round_kernel/sdpa_kernel_instantiation.cpp.in b/auto_round_extension/ark/auto_round_kernel/sdpa_kernel_instantiation.cpp.innew file mode 100644index 000000000..b9f7f004e--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/sdpa_kernel_instantiation.cpp.in@@ -0,0 +1,19 @@+// Generated by CMake. Do not edit directly.++#if defined(ARK_XPU) && defined(ARK_SYCL_TLA)++#include <sycl/sycl.hpp>+#include "sycl_tla_sdpa.hpp"++namespace ark {+namespace detail {++int @SDPA_FUNCTION_NAME@(Options const& options) {+ return @SDPA_TEMPLATE_NAME@<@SDPA_ELEMENT_TYPE@, @SDPA_ELEMENT_TYPE@, @SDPA_ELEMENT_TYPE@>(options);+}++} // namespace detail+} // namespace ark++#endif // ARK_XPU && ARK_SYCL_TLA+diff --git a/auto_round_extension/ark/__init__.py b/auto_round_extension/ark/auto_round_kernel/version.pysimilarity index 72%rename from auto_round_extension/ark/__init__.pyrename to auto_round_extension/ark/auto_round_kernel/version.pyindex 76e0da18c..85a13f188 100644--- a/auto_round_extension/ark/__init__.py+++ b/auto_round_extension/ark/auto_round_kernel/version.py@@ -1,4 +1,4 @@-# Copyright (c) 2025 Intel Corporation+# Copyright (c) 2023 Intel Corporation## Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.@@ -11,9 +11,7 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+"""Intel® auto-round: An open-source Python library+supporting popular model weight only compression based on signround."""-from auto_round_extension.ark.qlinear import QuantLinear, QuantLinearGPTQ, QuantLinearAWQ--qlinear_classes = (QuantLinear, QuantLinearGPTQ)--awq_classes = (QuantLinearAWQ,)+__version__ = "0.13.0"diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/cpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/cpu_wrapper.hppnew file mode 100644index 000000000..b39e6a216--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/cpu_wrapper.hpp@@ -0,0 +1,569 @@+//+// MIT license+// Copyright (C) 2024 Intel Corporation+// SPDX-License-Identifier: MIT+//++//+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.+// See https://llvm.org/LICENSE.txt for license information.+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception+//++#pragma once+#include "utils.hpp"+#include "bestla/bestla.h"+#include "bestla/bestla_prologue_a.h"+#include "bestla/bestla_epilogue.h"+#include "bestla/bestla_gemm.h"+#include "bestla/bestla_gemm_kblock.h"+#include "bestla/bestla_device.h"+#include "bestla/bestla_wrapper.h"++namespace bestla {+template <class GemmCore_T, template <class> class Wei_T>+using tLauncher_Fp_F32F32 = wrapper::gemm::LauncherBase<GemmCore_T, prologue_a::gemm::ShuffleActivationKBlockBaseF32,+ Wei_T, epilogue::gemm::AccumulatorWriteBackFp32>;++template <class GemmCore_T, template <class> class Wei_T>+using tLauncher_Int8_F32F32 =+ wrapper::gemm::LauncherIntKBlock<GemmCore_T, prologue_a::gemm::ShuffleActivationKBlockQuantizeF32, Wei_T,+ epilogue::gemm::AccumulatorWriteBackFp32>;++using PcWriteBackF32 = epilogue::gemm::PcKBlockCompInt8Epilogue<epilogue::gemm::AccumulatorWriteBackFp32>;++template <class GemmCore_T, template <class> class Wei_T>+using tLauncher_Int8Pc_F32F32 =+ wrapper::gemm::LauncherBase<GemmCore_T, prologue_a::gemm::ShuffleActivationKBlockQuantizeF32, Wei_T,+ PcWriteBackF32>;++using tAVX2 = gemm::SCoreRowNAvx2<24, 4>;+using tAVX2_VNNI = gemm::ICoreRowNAvx2vnni<24, 4>;+using tAVX_VNNI = gemm::ICoreRowNAvxvnni<24, 4>;+using tAVX512F = gemm::SCoreRowNAvx512f<48, 8>;+using tAVX512BW = gemm::ICoreRowNAvx512bw<48, 8>;+using tAVX512_VNNI = gemm::ICoreRowNAvx512vnni<48, 8>;+using tAMX_BF16 = gemm::HCoreRowNAmxbf16<64, 16>;+using tAMX_FP16 = gemm::HCoreRowNAmxfp16<64, 16>;+using tAVX512_BF16 = gemm::HCoreRowNAvx512bf16<48, 8>;+using tAVX512_FP16 = gemm::HCoreRowNAvx512fp16<96, 8>;+using tAMX_INT8_US = gemm::ICoreRowNAmxint8<64, 16>;+using tAMX_INT8_SS = gemm::ICoreRowNAmxint8SS<64, 16>;++using tAVX2_VNNI_KBlock = gemm::KblockGemmAvx2vnni<24, 2>;+using tAVX_VNNI_KBlock = gemm::KblockGemmAvxvnni<24, 2>;+using tAVX512BW_KBlock = gemm::KblockGemmAvx512bw<48, 4>;+using tAVX512_VNNI_KBlock = gemm::KblockGemmAvx512vnni<48, 4>;+using tAMX_INT8_US_KBlock = gemm::KblockGemmAmxint8<64, 16>;+using tAMX_INT8_SS_KBlock = gemm::KblockGemmAmxint8SS<64, 16>;++template <class GC_T>+using tWeiNInt = prologue_b::gemm::WeightKBlockNInteger<GC_T>;+template <class GC_T>+using tWeiNFloat = prologue_b::gemm::WeightKBlockNFloat<GC_T>;++template <class GC_T>+using tActKBaseF32 = prologue_a::gemm::ShuffleActivationKBlockBaseF32<GC_T>;++constexpr uint64_t Fp32Cores[] = {tAVX2::ID, tAVX512F::ID};+constexpr uint64_t Bf16Cores[] = {tAMX_BF16::ID, tAVX512_BF16::ID};+constexpr uint64_t Fp16Cores[] = {tAVX512_FP16::ID, tAMX_FP16::ID};+constexpr uint64_t Int8Cores[] = {tAVX2::ID, tAVX_VNNI::ID, tAVX2_VNNI_KBlock::ID, tAVX512BW_KBlock::ID,+ tAVX512_VNNI::ID, tAMX_INT8_US::ID, tAMX_INT8_SS::ID};+constexpr uint64_t FloatCores[] = {tAVX2::ID, tAVX512F::ID, tAMX_BF16::ID, tAVX512_FP16::ID, tAMX_FP16::ID};+constexpr uint64_t AllKBlockCores[] = {tAVX2::ID,+ tAVX512F::ID,+ tAMX_BF16::ID,+ tAMX_FP16::ID,+ tAVX512_FP16::ID,+ tAVX2_VNNI_KBlock::ID,+ tAVX_VNNI_KBlock::ID,+ tAVX512BW_KBlock::ID,+ tAVX512_VNNI_KBlock::ID,+ tAMX_INT8_US_KBlock::ID,+ tAMX_INT8_SS_KBlock::ID};++static inline bool can_comps8(ark::QuantParam* p) {+ if (p->asym && p->weight_type == BTLA_DTYPE::S8) return false;+ if (p->blocksize == -1 || p->blocksize == p->k) return true;+ if (p->blocksize % 4 == 0) return true;+ return false;+}++static inline uint64_t BTLAGemmCoreId(ark::QuantParam* p) {+ GetCPUDevice();+ using namespace bestla;+ if (!_cd->INTEL()) {+ return tAVX2::ID;+ }+ switch (p->compute_type) {+ case BTLA_DTYPE::EleBitsUndef: // from low precision to high precision+ [[fallthrough]];+ case BTLA_DTYPE::S8:+ if (can_comps8(p)) {+ if (_cd->AMX_INT8()) {+ if (p->blocksize == -1) {+ return tAMX_INT8_SS::ID;+ } else if (p->blocksize % tAMX_INT8_SS_KBlock::KTILE == 0) {+ return tAMX_INT8_SS_KBlock::ID;+ }+ }+ if (_cd->AVX512_VNNI()) {+ if (p->blocksize == -1) {+ return tAVX512_VNNI::ID;+ } else if (p->blocksize % tAVX512_VNNI_KBlock::KTILE == 0) {+ return tAVX512_VNNI_KBlock::ID;+ }+ }+ if (_cd->AVX_VNNI()) {+ if (p->blocksize == -1) {+ return tAVX_VNNI::ID;+ } else if (p->blocksize % tAVX_VNNI_KBlock::KTILE == 0) {+ return tAVX_VNNI_KBlock::ID;+ }+ }+ }+ [[fallthrough]];+ case BTLA_DTYPE::F16:+ if (_cd->AMX_FP16() && (p->blocksize % tAMX_FP16::KTILE == 0)) {+ return tAMX_FP16::ID;+ }+ [[fallthrough]];+ case BTLA_DTYPE::BF16:+ if (_cd->AMX_BF16() && (p->blocksize % tAMX_BF16::KTILE == 0)) {+ return tAMX_BF16::ID;+ }+ [[fallthrough]];+ case BTLA_DTYPE::F32:+ if (_cd->AVX512F()) {+ return tAVX512F::ID;+ }+ if (_cd->AVX2()) {+ return tAVX2::ID;+ }+ [[fallthrough]];+ default:+ return 0;+ }+}++static inline uint64_t BTLAGemmCoreId(BTLA_DTYPE dt) {+ GetCPUDevice();+ using namespace bestla;+ if (!_cd->INTEL()) {+ return tAVX2::ID;+ }+ switch (dt) {+ case BTLA_DTYPE::EleBitsUndef: // from low precision to high precision+ [[fallthrough]];+ case BTLA_DTYPE::S8:+ if (_cd->AMX_INT8()) {+ return tAMX_INT8_SS::ID;+ }+ if (_cd->AVX512_VNNI()) {+ return tAVX512_VNNI::ID;+ }+ if (_cd->AVX_VNNI()) {+ return tAVX_VNNI::ID;+ }+ [[fallthrough]];+ case BTLA_DTYPE::F16:+ if (_cd->AMX_FP16()) {+ return tAMX_FP16::ID;+ }+ [[fallthrough]];+ case BTLA_DTYPE::BF16:+ if (_cd->AMX_BF16()) {+ return tAMX_BF16::ID;+ }+ [[fallthrough]];+ case BTLA_DTYPE::F32:+ if (_cd->AVX512F()) {+ return tAVX512F::ID;+ }+ if (_cd->AVX2()) {+ return tAVX2::ID;+ }+ [[fallthrough]];+ default:+ return 0;+ }+}++template <class GC_T>+static inline size_t BTLAGemmPackBSizeT(const ark::QuantParam* p) {+ using ProB = bestla::prologue_b::gemm::WeightKBlockNInt<GC_T>;+ auto stor = ProB::create_storage(p->n, p->k, p->blocksize, p->weight_type, p->scale_type, p->asym);+ return stor.total_size();+}++static inline size_t BTLAGemmPackBSize(uint64_t coreID, const ark::QuantParam* p) {+ using namespace bestla;+ switch (coreID) {+ case tAMX_INT8_SS::ID:+ return BTLAGemmPackBSizeT<tAMX_INT8_SS>(p);+ case tAMX_INT8_SS_KBlock::ID:+ return BTLAGemmPackBSizeT<tAMX_INT8_SS_KBlock>(p);+ case tAVX512_VNNI::ID:+ return BTLAGemmPackBSizeT<tAVX512_VNNI>(p);+ case tAVX512_VNNI_KBlock::ID:+ return BTLAGemmPackBSizeT<tAVX512_VNNI_KBlock>(p);+ case tAVX_VNNI::ID:+ return BTLAGemmPackBSizeT<tAVX_VNNI>(p);+ case tAVX_VNNI_KBlock::ID:+ return BTLAGemmPackBSizeT<tAVX_VNNI_KBlock>(p);+ case tAMX_FP16::ID:+ return BTLAGemmPackBSizeT<tAMX_FP16>(p);+ case tAMX_BF16::ID:+ return BTLAGemmPackBSizeT<tAMX_BF16>(p);+ case tAVX512F::ID:+ return BTLAGemmPackBSizeT<tAVX512F>(p);+ case tAVX2::ID:+ return BTLAGemmPackBSizeT<tAVX2>(p);+ default:+ return 0;+ }+ return 0;+}++static inline BTLA_DTYPE BTLAGemmBaseDType(size_t N, size_t K, uint64_t coreID) {+ using namespace bestla;+ switch (coreID) {+ case tAMX_FP16::ID:+ return BTLA_DTYPE::F16;+ case tAMX_BF16::ID:+ return BTLA_DTYPE::BF16;+ case tAVX512F::ID:+ case tAVX2::ID:+ return BTLA_DTYPE::F32;+ default:+ break;+ }+ return BTLA_DTYPE::EleBitsUndef;+}++template <class GC_T>+static inline void BTLAGemmPackQBT(const int8_t* QB, const float* scales, const int8_t* zero_points, void* PackedBuf,+ const ark::QuantParam* p, void* ThreadPool) {+ using ProB = bestla::prologue_b::gemm::WeightKBlockNInt<GC_T>;+ auto stor = ProB::create_storage(p->n, p->k, p->blocksize, p->weight_type, p->scale_type, p->asym);+ auto wptr = (int8_t*)PackedBuf;+ auto data_ptr = stor.serialize(wptr);+ stor.set_buffers(data_ptr);+ auto thd = (bestla::parallel::IThreading*)ThreadPool;+ ProB::pack_qweight(p->n, p->k, QB, p->n, scales, zero_points, stor, thd);+}++template <class GC_T>+static inline void BTLAGemmUnpackQBT(storage::gemm::StorageWeightNInt* B, float* DQB, void* ThreadPool) {+ using ProB = bestla::prologue_b::gemm::WeightKBlockNInt<GC_T>;+ auto thd = (bestla::parallel::IThreading*)ThreadPool;++ ProB::unpack_weight(B->info_.n_, B->info_.k_, *B, DQB, B->info_.n_, thd);+}++static inline void BTLAGemmPackQB(void* PackedBuf, const int8_t* QB, const float* scales, const int8_t* zero_points,+ uint64_t coreID, const ark::QuantParam* p, void* ThreadPool) {+ using namespace bestla;+ switch (coreID) {+ case tAMX_INT8_SS::ID:+ return BTLAGemmPackQBT<tAMX_INT8_SS>(QB, scales, zero_points, PackedBuf, p, ThreadPool);+ case tAMX_INT8_SS_KBlock::ID:+ return BTLAGemmPackQBT<tAMX_INT8_SS_KBlock>(QB, scales, zero_points, PackedBuf, p, ThreadPool);+ case tAVX512_VNNI::ID:+ return BTLAGemmPackQBT<tAVX512_VNNI>(QB, scales, zero_points, PackedBuf, p, ThreadPool);+ case tAVX512_VNNI_KBlock::ID:+ return BTLAGemmPackQBT<tAVX512_VNNI_KBlock>(QB, scales, zero_points, PackedBuf, p, ThreadPool);+ case tAVX_VNNI::ID:+ return BTLAGemmPackQBT<tAVX_VNNI>(QB, scales, zero_points, PackedBuf, p, ThreadPool);+ case tAVX_VNNI_KBlock::ID:+ return BTLAGemmPackQBT<tAVX_VNNI_KBlock>(QB, scales, zero_points, PackedBuf, p, ThreadPool);+ case tAMX_BF16::ID:+ return BTLAGemmPackQBT<tAMX_BF16>(QB, scales, zero_points, PackedBuf, p, ThreadPool);+ case tAMX_FP16::ID:+ return BTLAGemmPackQBT<tAMX_FP16>(QB, scales, zero_points, PackedBuf, p, ThreadPool);+ case tAVX512F::ID:+ return BTLAGemmPackQBT<tAVX512F>(QB, scales, zero_points, PackedBuf, p, ThreadPool);+ case tAVX2::ID:+ return BTLAGemmPackQBT<tAVX2>(QB, scales, zero_points, PackedBuf, p, ThreadPool);+ default:+ break;+ }+}++static inline void BTLAGemmUnpack(void* B, float* DQB, void* ThreadPool) {+ GetCPUDevice();+ using namespace bestla;+ auto prologue = storage::CollectionParser::parse_prologue_id(B);+ if (storage::CollectionParser::valid_prologue(prologue)) {+ if (prologue == BTLA_PROLOGUEB_IDS::WeightKBlockNInteger) {+ auto stor_ptr = (int8_t*)(B);+ storage::gemm::StorageWeightNInt stor;+ auto data_ptr = stor.deserialize(stor_ptr);+ stor.set_buffers(data_ptr);+ auto& id = stor.info_.core_id_;+ if (id == tAVX512_VNNI::ID) {+ BTLAGemmUnpackQBT<tAVX512_VNNI>(&stor, DQB, ThreadPool);+ } else if (id == tAVX512_VNNI_KBlock::ID) {+ BTLAGemmUnpackQBT<tAVX512_VNNI_KBlock>(&stor, DQB, ThreadPool);+ } else if (id == tAMX_INT8_SS::ID) {+ BTLAGemmUnpackQBT<tAMX_INT8_SS>(&stor, DQB, ThreadPool);+ } else if (id == tAMX_INT8_SS_KBlock::ID) {+ BTLAGemmUnpackQBT<tAMX_INT8_SS_KBlock>(&stor, DQB, ThreadPool);+ } else if (id == tAVX_VNNI::ID) {+ BTLAGemmUnpackQBT<tAVX_VNNI>(&stor, DQB, ThreadPool);+ } else if (id == tAVX_VNNI_KBlock::ID) {+ BTLAGemmUnpackQBT<tAVX_VNNI_KBlock>(&stor, DQB, ThreadPool);+ } else if (id == tAMX_BF16::ID) {+ BTLAGemmUnpackQBT<tAMX_BF16>(&stor, DQB, ThreadPool);+ } else if (id == tAMX_FP16::ID) {+ BTLAGemmUnpackQBT<tAMX_FP16>(&stor, DQB, ThreadPool);+ } else if (id == tAVX512F::ID) {+ BTLAGemmUnpackQBT<tAVX512F>(&stor, DQB, ThreadPool);+ } else if (id == tAVX2::ID) {+ BTLAGemmUnpackQBT<tAVX2>(&stor, DQB, ThreadPool);+ }+ }+ }+}++template <class GC_T, bool PC = false>+static inline void BTLAWOQGemmFp32S8ForwardT(storage::gemm::StorageWeightNInt* B, const float* A, float* C,+ const float* bias, int M, int N, int K, int lda, int ldc, void* ws_ptr,+ parallel::IThreading* ThreadPool) {+ if constexpr (PC) {+ using Parallel = parallel::gemm::SchedulerBase<GC_T>;+ using Launcher =+ wrapper::gemm::LauncherBaseV2<GC_T, prologue_a::gemm::ActivationF32KBlockQuantize,+ prologue_b::gemm::WeightKBlockNInt,+ epilogue::gemm::PcKBlockCompInt8Epilogue<epilogue::gemm::AlphaBetaProcessFp32>>;+ utils::GemmProblem gp(1, M, N, K, B->block_);+ float alpha = 1.f, beta = bias == nullptr ? 0.f : 1.f;+ auto storA = Launcher::PrologueA::createStorage(M, K, B->block_, B->corr_.zptr());+ auto wptr = (int8_t*)ws_ptr;+ storA.assign(wptr);+ typename Launcher::Param args{gp,+ {A, lda, &storA},+ {B},+ {{B->corr_.sptr(), B->corr_.sdtype_, storA.template SPtr<float>(),+ storA.template ZPtr<uint8_t>(), (int*)B->corr_.rptr(), nullptr, nullptr, K},+ {C, (float*)bias, ldc, 0, alpha, beta}}};+ parallel::GemmRunWithA<Parallel, Launcher>(args, ThreadPool);+ } else {+ using Parallel = parallel::gemm::SchedulerIntKBlock<GC_T>;+ using Launcher =+ wrapper::gemm::LauncherIntKBlockV2<GC_T, prologue_a::gemm::ActivationF32KBlockQuantize,+ prologue_b::gemm::WeightKBlockNInt, epilogue::gemm::AlphaBetaProcessFp32>;+ utils::GemmProblem gp(1, M, N, K, B->block_);+ float alpha = 1.f, beta = bias == nullptr ? 0.f : 1.f;+ auto storA = Launcher::PrologueA::createStorage(M, K, B->block_, B->corr_.zptr());+ auto wptr = (int8_t*)ws_ptr;+ storA.assign(wptr);+ typename Launcher::Param args{gp, {A, lda, &storA}, {B}, {C, (float*)bias, ldc, 0, alpha, beta}};+ parallel::GemmRunWithA<Parallel, Launcher>(args, ThreadPool);+ }+}++template <class GC_T>+static inline void BTLAWOQGemmFp32ForwardT(storage::gemm::StorageWeightNInt* B, const float* A, float* C,+ const float* bias, int M, int N, int K, int lda, int ldc, void* ws_ptr,+ parallel::IThreading* ThreadPool) {+ using Parallel = parallel::gemm::SchedulerBase<GC_T>;+ using Launcher =+ wrapper::gemm::LauncherBaseV2<GC_T, prologue_a::gemm::ActivationConverterFp32, prologue_b::gemm::WeightKBlockNInt,+ epilogue::gemm::AlphaBetaProcessFp32>;+ utils::GemmProblem gp(1, M, N, K, B->block_);+ float alpha = 1.f, beta = bias == nullptr ? 0.f : 1.f;+ typename Launcher::Param args{gp, {A, lda}, {B}, {C, (float*)bias, ldc, 0, alpha, beta}};+ parallel::GemmRun<Parallel, Launcher>(args, ThreadPool);+}++template <class GC_T>+static inline void BTLAGemmNPackBForwardT(void* wsptr, const void* A, const void* B, float* C, const float* bias, int M,+ int N, int K, parallel::IThreading* ThreadPool) {+ using Parallel = parallel::gemm::SchedulerBase<GC_T>;+ using Launcher = wrapper::gemm::LauncherBaseV2<GC_T, prologue_a::gemm::ActivationBase, prologue_b::gemm::WeightNPack,+ epilogue::gemm::AlphaBetaProcessFp32>;+ using ProB = bestla::prologue_b::gemm::WeightNPack<GC_T>;+ auto stor = ProB::create_storage(N, K);+ auto wptr = (int8_t*)wsptr;+ auto data_ptr = stor.serialize(wptr);+ stor.set_buffers(data_ptr);+ auto thd = (bestla::parallel::IThreading*)ThreadPool;+ storage::gemm::StorageWeight raw;+ raw.set_nt_data(N, K, N, utils::bestla_dtype<typename GC_T::BType>, (void*)B);+ ProB::convert(raw, stor, thd);+ utils::GemmProblem gp(1, M, N, K);+ float alpha = 1.f, beta = bias == nullptr ? 0.f : 1.f;+ typename Launcher::Param args{gp, {(typename GC_T::AType*)A, K}, {0, 0, &stor}, {C, (float*)bias, N, 0, alpha, beta}};+ parallel::GemmRun<Parallel, Launcher>(args, ThreadPool);+}++static inline void BTLAGemmNPackBForward(BTLA_DTYPE dt, const void* A, const void* B, float* C, const float* bias,+ int M, int N, int K, int lda, int ldc, void* wsptr,+ parallel::IThreading* ThreadPool) {+ auto id = BTLAGemmCoreId(dt);+ if (id == tAMX_BF16::ID) {+ BTLAGemmNPackBForwardT<tAMX_BF16>(wsptr, A, B, C, bias, M, N, K, ThreadPool);+ } else if (id == tAMX_FP16::ID) {+ BTLAGemmNPackBForwardT<tAMX_FP16>(wsptr, A, B, C, bias, M, N, K, ThreadPool);+ } else if (id == tAVX512F::ID) {+ BTLAGemmNPackBForwardT<tAVX512F>(wsptr, A, B, C, bias, M, N, K, ThreadPool);+ } else if (id == tAVX2::ID) {+ BTLAGemmNPackBForwardT<tAVX2>(wsptr, A, B, C, bias, M, N, K, ThreadPool);+ }+}++static inline size_t BTLAGemmNPackBSize(int N, int K, BTLA_DTYPE dt) {+ auto id = BTLAGemmCoreId(dt);+ if (id == tAMX_BF16::ID) {+ using ProB = prologue_b::gemm::WeightNPack<tAMX_BF16>;+ auto stor = ProB::create_storage(N, K);+ return stor.total_size();+ } else if (id == tAMX_FP16::ID) {+ using ProB = prologue_b::gemm::WeightNPack<tAMX_FP16>;+ auto stor = ProB::create_storage(N, K);+ return stor.total_size();+ } else if (id == tAVX512F::ID) {+ using ProB = prologue_b::gemm::WeightNPack<tAVX512F>;+ auto stor = ProB::create_storage(N, K);+ return stor.total_size();+ } else if (id == tAVX2::ID) {+ using ProB = prologue_b::gemm::WeightNPack<tAVX2>;+ auto stor = ProB::create_storage(N, K);+ return stor.total_size();+ }+ return 0;+}++static inline size_t BTLAWOQGemmForwardWorkspace(void* B, int M, int N, int K, int lda, int ldc) {+ GetCPUDevice();+ using namespace bestla;+ auto prologue = storage::CollectionParser::parse_prologue_id(B);+ if (storage::CollectionParser::valid_prologue(prologue)) {+ if (prologue == BTLA_PROLOGUEB_IDS::WeightKBlockNInteger) {+ storage::gemm::StorageWeightNInt stor;+ auto stor_ptr = (int8_t*)(B);+ auto data_ptr = stor.deserialize(stor_ptr);+ stor.set_buffers(data_ptr);+ auto& id = stor.info_.core_id_;+ if (id == tAVX512_VNNI_KBlock::ID) {+ auto storA = prologue_a::gemm::ActivationF32KBlockQuantize<tAVX512_VNNI_KBlock>::createStorage(+ M, K, stor.block_, stor.corr_.zptr());+ return storA.mSize;+ } else if (id == tAVX512_VNNI::ID) {+ auto storA = prologue_a::gemm::ActivationF32KBlockQuantize<tAVX512_VNNI>::createStorage(M, K, stor.block_,+ stor.corr_.zptr());+ return storA.mSize;+ } else if (id == tAMX_INT8_SS::ID) {+ auto storA = prologue_a::gemm::ActivationF32KBlockQuantize<tAMX_INT8_SS>::createStorage(M, K, stor.block_,+ stor.corr_.zptr());+ return storA.mSize;+ } else if (id == tAMX_INT8_SS_KBlock::ID) {+ auto storA = prologue_a::gemm::ActivationF32KBlockQuantize<tAMX_INT8_SS_KBlock>::createStorage(+ M, K, stor.block_, stor.corr_.zptr());+ return storA.mSize;+ } else if (id == tAVX_VNNI::ID) {+ auto storA = prologue_a::gemm::ActivationF32KBlockQuantize<tAVX_VNNI>::createStorage(M, K, stor.block_,+ stor.corr_.zptr());+ return storA.mSize;+ } else if (id == tAVX_VNNI_KBlock::ID) {+ auto storA = prologue_a::gemm::ActivationF32KBlockQuantize<tAVX_VNNI_KBlock>::createStorage(M, K, stor.block_,+ stor.corr_.zptr());+ return storA.mSize;+ }+ }+ }+ return 0;+}++static inline void BTLAWOQGemmFp32Forward(void* B, const float* A, float* C, const float* bias, int M, int N, int K,+ int lda, int ldc, void* ws_ptr, parallel::IThreading* ThreadPool) {+ GetCPUDevice();+ using namespace bestla;+ auto prologue = storage::CollectionParser::parse_prologue_id(B);+ if (storage::CollectionParser::valid_prologue(prologue)) {+ if (prologue == BTLA_PROLOGUEB_IDS::WeightKBlockNInteger) {+ auto stor_ptr = (int8_t*)(B);+ storage::gemm::StorageWeightNInt stor;+ auto data_ptr = stor.deserialize(stor_ptr);+ stor.set_buffers(data_ptr);+ auto& id = stor.info_.core_id_;+ if (id == tAVX512_VNNI::ID) {+ BTLAWOQGemmFp32S8ForwardT<tAVX512_VNNI, true>(&stor, A, C, bias, M, N, K, lda, ldc, ws_ptr, ThreadPool);+ } else if (id == tAVX512_VNNI_KBlock::ID) {+ BTLAWOQGemmFp32S8ForwardT<tAVX512_VNNI_KBlock>(&stor, A, C, bias, M, N, K, lda, ldc, ws_ptr, ThreadPool);+ } else if (id == tAMX_INT8_SS::ID) {+ BTLAWOQGemmFp32S8ForwardT<tAMX_INT8_SS, true>(&stor, A, C, bias, M, N, K, lda, ldc, ws_ptr, ThreadPool);+ } else if (id == tAMX_INT8_SS_KBlock::ID) {+ BTLAWOQGemmFp32S8ForwardT<tAMX_INT8_SS_KBlock>(&stor, A, C, bias, M, N, K, lda, ldc, ws_ptr, ThreadPool);+ } else if (id == tAVX_VNNI::ID) {+ BTLAWOQGemmFp32S8ForwardT<tAVX_VNNI, true>(&stor, A, C, bias, M, N, K, lda, ldc, ws_ptr, ThreadPool);+ } else if (id == tAVX_VNNI_KBlock::ID) {+ BTLAWOQGemmFp32S8ForwardT<tAVX_VNNI_KBlock>(&stor, A, C, bias, M, N, K, lda, ldc, ws_ptr, ThreadPool);+ } else if (id == tAMX_BF16::ID) {+ BTLAWOQGemmFp32ForwardT<tAMX_BF16>(&stor, A, C, bias, M, N, K, lda, ldc, ws_ptr, ThreadPool);+ } else if (id == tAMX_FP16::ID) {+ BTLAWOQGemmFp32ForwardT<tAMX_FP16>(&stor, A, C, bias, M, N, K, lda, ldc, ws_ptr, ThreadPool);+ } else if (id == tAVX512F::ID) {+ BTLAWOQGemmFp32ForwardT<tAVX512F>(&stor, A, C, bias, M, N, K, lda, ldc, ws_ptr, ThreadPool);+ } else if (id == tAVX2::ID) {+ BTLAWOQGemmFp32ForwardT<tAVX2>(&stor, A, C, bias, M, N, K, lda, ldc, ws_ptr, ThreadPool);+ }+ }+ }+}++} // namespace bestla++namespace ark {++class CpuWrapper {+ public:+ static bestla::parallel::IThreading* get_threading() {+ static bestla::parallel::OMPThreading DefaultThreading;+ return &DefaultThreading;+ }++ static inline size_t get_packw_size(QuantParam* p) {+ auto coreid = bestla::BTLAGemmCoreId(p);+ auto size = bestla::BTLAGemmPackBSize(coreid, p);+ return size;+ }++ static void packq(const int8_t* raws8, const float* scaleptr, const int8_t* zp, int8_t* blob, QuantParam* p) {+ auto coreid = bestla::BTLAGemmCoreId(p);+ bestla::BTLAGemmPackQB(blob, raws8, scaleptr, p->asym ? zp : nullptr, coreid, p, get_threading());+ }++ static void unpackq(BTLA_DTYPE outt, int8_t* blob, void* optr, QuantParam* p) {+ assert(outt == BTLA_DTYPE::F32);+ bestla::BTLAGemmUnpack(blob, (float*)optr, get_threading());+ }++ static void gemm(int m, int n, int k, const void* A, BTLA_DTYPE dt, const void* B, bool BT, float* C,+ const float* bias) {+ auto bsize = bestla::BTLAGemmNPackBSize(n, k, dt);+ auto wssize = bsize;+ wssize += BT ? n * k * bestla::utils::bestla_dtype_bytes(dt) : 0;+ auto ptr = DnnlContext::Instance()->get_scratch_mem(wssize, 0, nullptr);+ auto packwptr = ptr;+ auto bntptr = (int8_t*)packwptr + bsize;+ auto bptr = B;+ auto thd = get_threading();+ if (BT) {+ bestla::prologue_b::gemm::transpose(n, k, (const int8_t*)B, dt, k, bntptr, n, thd);+ bptr = bntptr;+ }+ bestla::BTLAGemmNPackBForward(dt, A, bptr, C, bias, m, n, k, k, n, packwptr, thd);+ }++ static void woq_gemm(int m, const void* a, const void* b, void* c, const void* bias, BTLA_DTYPE acdt, QuantParam* p) {+ auto wssize = bestla::BTLAWOQGemmForwardWorkspace((void*)b, m, p->n, p->k, p->k, p->n);+ auto ptr = DnnlContext::Instance()->get_scratch_mem(wssize, 0, nullptr);+ bestla::BTLAWOQGemmFp32Forward((void*)b, (const float*)a, (float*)c, (const float*)bias, m, p->n, p->k, p->k, p->n,+ ptr, get_threading());+ }+};++} // namespace ark\ No newline at end of filediff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/dnnl_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/dnnl_wrapper.hppnew file mode 100644index 000000000..7529eab33--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/dnnl_wrapper.hpp@@ -0,0 +1,622 @@+//+// MIT license+// Copyright (C) 2024 Intel Corporation+// SPDX-License-Identifier: MIT+//++//+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.+// See https://llvm.org/LICENSE.txt for license information.+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception+//++#pragma once++#include "utils.hpp"++#define GETCTX() \+ auto& eng = *DnnlContext::Instance()->get_eng(q); \+ auto& stream = *DnnlContext::Instance()->get_stream(q);++namespace ark {++class DnnlWrapper {+ public:+ using dt = dnnl::memory::data_type;+ using tag = dnnl::memory::format_tag;++ static void gemm(sycl::queue* q, int m, int n, int k, const void* a, dt at, const void* b, dt bt, bool BT, void* c,+ dt ct, const void* bias) {+ GETCTX();+ dnnl::memory::dims a_dims = {m, k};+ const auto a_in_md = dnnl::memory::desc(a_dims, at, dnnl::memory::format_tag::ab);+ auto a_mem = dnnl::memory(a_in_md, eng, const_cast<void*>(a));++ dnnl::memory::dims b_dims = {k, n};++ const auto b_in_md =+ dnnl::memory::desc(b_dims, bt, BT ? dnnl::memory::format_tag::ba : dnnl::memory::format_tag::ab);+ auto b_mem = dnnl::memory(b_in_md, eng, const_cast<void*>(b));++ auto bias_md = dnnl::memory::desc({1, n}, ct, dnnl::memory::format_tag::ab);+ auto bias_mem = dnnl::memory(bias_md, eng, const_cast<void*>(bias));++ dnnl::memory::dims c_dims = {m, n};+ const auto c_md = dnnl::memory::desc(c_dims, ct, dnnl::memory::format_tag::ab);+ dnnl::primitive_attr primitive_attr;+ primitive_attr.set_scratchpad_mode(dnnl::scratchpad_mode::user);++ // primitive_attr.set_fpmath_mode(dnnl::fpmath_mode::strict);++ auto matmul_pd = bias ? dnnl::matmul::primitive_desc(eng, a_in_md, b_in_md, bias_md, c_md, primitive_attr)+ : dnnl::matmul::primitive_desc(eng, a_in_md, b_in_md, c_md, primitive_attr);+ auto c_mem = dnnl::memory(matmul_pd.dst_desc(), eng, c);++ auto scratchpad_md = matmul_pd.scratchpad_desc();++ auto scratchpad_mem = DnnlContext::Instance()->get_scratch_mem(scratchpad_md, q);++ auto matmul_prim = dnnl::matmul(matmul_pd);++ std::unordered_map<int, dnnl::memory> matmul_args;+ matmul_args.insert({DNNL_ARG_SRC, a_mem});+ matmul_args.insert({DNNL_ARG_WEIGHTS, b_mem});+ if (bias) matmul_args.insert({DNNL_ARG_BIAS, bias_mem});+ matmul_args.insert({DNNL_ARG_DST, c_mem});+ matmul_args.insert({DNNL_ARG_SCRATCHPAD, scratchpad_mem});+ matmul_prim.execute(stream, matmul_args);+ if (q == nullptr) stream.wait();+ }++ static void dyn_quant_s8(sycl::queue* q, int m, int k, const void* a, dt adt, void* a_abs, int8_t* qa, void* maxa,+ void* scalea) {+ GETCTX();+ using namespace dnnl;+ auto src_f32_md = memory::desc({m, k}, adt, memory::format_tag::ab);+ auto src_f32_mem = memory(src_f32_md, eng, const_cast<void*>(a));+ auto src_absf32_mem = memory(src_f32_md, eng, const_cast<void*>(a_abs));++ auto abs_attr = dnnl::eltwise_forward::primitive_desc(eng, prop_kind::forward_inference, algorithm::eltwise_abs,+ src_f32_mem.get_desc(), src_f32_mem.get_desc(), 0.0f, 0.0f);++ auto abs_prim = dnnl::eltwise_forward(abs_attr);+ abs_prim.execute(stream, {{DNNL_ARG_SRC, src_f32_mem}, {DNNL_ARG_DST, src_absf32_mem}});+ // 1. 定义输出 memory (只有一个标量值)+ // --- 第二步:求最大值 (Reduction Max) ---+ auto max_abs_md = dnnl::memory::desc({1, 1}, adt, dnnl::memory::format_tag::ab);++ auto reduct_pd = dnnl::reduction::primitive_desc(eng, dnnl::algorithm::reduction_max, src_absf32_mem.get_desc(),+ max_abs_md, 0.0f, 0.0f);++ auto max_abs_mem = dnnl::memory(max_abs_md, eng, maxa);+ dnnl::reduction(reduct_pd).execute(stream, {{DNNL_ARG_SRC, src_absf32_mem}, {DNNL_ARG_DST, max_abs_mem}});++ // 4. (可选) 将 max 转换为 127/max+ // 这里展示如何用 eltwise 算子在 Device 端直接完成转换:scale = 127 * (1/x)+ auto final_scale_mem = memory(max_abs_md, eng, scalea);+ auto eltwise_pd = eltwise_forward::primitive_desc(eng, prop_kind::forward_inference, algorithm::eltwise_linear,+ max_abs_md, max_abs_md, 1 / 127.0f, .0f);++ eltwise_forward(eltwise_pd).execute(stream, {{DNNL_ARG_SRC, max_abs_mem}, {DNNL_ARG_DST, final_scale_mem}});++ auto src_s8_md = memory::desc({m, k}, memory::data_type::s8, memory::format_tag::ab);+ auto src_s8_mem = memory(src_s8_md, eng, qa);+ auto scale1_md = dnnl::memory::desc({1}, adt, dnnl::memory::format_tag::a);+ auto scale1_mem = memory(scale1_md, eng, scalea);++ // 使用 Reorder 进行量化+ primitive_attr q_attr;+ q_attr.set_scales_mask(DNNL_ARG_DST, 0); // Per-tensor++ auto q_reorder_pd = reorder::primitive_desc(eng, src_f32_md, eng, src_s8_md, q_attr);+ reorder(q_reorder_pd)+ .execute(stream, {{DNNL_ARG_SRC, src_f32_mem},+ {DNNL_ARG_DST, src_s8_mem},+ {DNNL_ARG_ATTR_SCALES | DNNL_ARG_DST, scale1_mem}});+ }++ static void sycl_dyn_quant_s8(sycl::queue* q, int m, int k, const void* a, dt adt, int8_t* qa, void* scalea,+ int mask) {+#if ARK_XPU+ if (adt == dt::f32) {+ using T = float;+ using Pro = bestla::sycl_prologue_a::ActivationBase<T>;+ Pro::template quant_s8<typename Pro::CfgQuantF32>(m, k, mask, {(T*)a, k}, qa, (T*)scalea, q);+ } else if (adt == dt::f16) {+ using T = sycl::half;+ using Pro = bestla::sycl_prologue_a::ActivationBase<T>;+ Pro::template quant_s8<typename Pro::CfgQuantF32>(m, k, mask, {(T*)a, k}, qa, (T*)scalea, q);+ } else if (adt == dt::bf16) {+ using T = sycl::ext::oneapi::bfloat16;+ using Pro = bestla::sycl_prologue_a::ActivationBase<T>;+ Pro::template quant_s8<typename Pro::CfgQuantF32>(m, k, mask, {(T*)a, k}, qa, (T*)scalea, q);+ }+#endif+ }++ // a:mxk b:nxk c:mxn+ // scale_a:m scale_b:n+ static void sycl_igemm_s8s8(sycl::queue* q, int m, int n, int k, const void* a, const void* b, bool BT, void* c,+ dt ct, void* scale_a, void* scale_b, void* bias, int blocksize) {+#if ARK_XPU+ using namespace bestla::sycl_gemm;+ if (blocksize == k || blocksize == -1) {+ if (ct == dt::f32) {+ using T = float;+ Launcher<xmx::IGemmDQCfg<T>, xmx::IGemmDQCore>::run(+ q, {(void*)a, (void*)b, (void*)c, m, n, k, k, k, n, bias, scale_a, scale_b});+ } else if (ct == dt::f16) {+ using T = sycl::half;+ Launcher<xmx::IGemmDQCfg<T>, xmx::IGemmDQCore>::run(+ q, {(void*)a, (void*)b, (void*)c, m, n, k, k, k, n, bias, scale_a, scale_b});+ } else if (ct == dt::bf16) {+ using T = sycl::ext::oneapi::bfloat16;+ Launcher<xmx::IGemmDQCfg<T>, xmx::IGemmDQCore>::run(+ q, {(void*)a, (void*)b, (void*)c, m, n, k, k, k, n, bias, scale_a, scale_b});+ }+ } else {+ if (ct == dt::f32) {+ using T = float;+ Launcher<xmx::IKblockGemmDQCfg<T>, xmx::IKblockGemmDQCore>::run(+ q, {(void*)a, (void*)b, (void*)c, m, n, k, k, k, n, bias, scale_a, scale_b, blocksize});+ } else if (ct == dt::f16) {+ using T = sycl::half;+ Launcher<xmx::IKblockGemmDQCfg<T>, xmx::IKblockGemmDQCore>::run(+ q, {(void*)a, (void*)b, (void*)c, m, n, k, k, k, n, bias, scale_a, scale_b, blocksize});+ }+ }+#endif+ }++ // a:mxk b:nxk c:mxn+ // scale_a:m scale_b:n+ static void igemm_s8s8(sycl::queue* q, int m, int n, int k, const void* a, const void* b, bool BT, void* c, dt ct,+ void* scale_a, void* scale_b, void* bias) {+ GETCTX();++ dnnl::memory::dims a_dims = {m, k};+ const auto a_in_md = dnnl::memory::desc(a_dims, dt::s8, tag::ab);++ dnnl::memory::dims b_dims = {k, n};+ const auto b_in_md = dnnl::memory::desc(b_dims, dt::s8, BT ? tag::ba : tag::ab);++ auto bias_md = dnnl::memory::desc({1, n}, ct, dnnl::memory::format_tag::ab);+ auto bias_mem = dnnl::memory(bias_md, eng, bias);++ dnnl::memory::dims c_dims = {m, n};+ const auto c_md = dnnl::memory::desc(c_dims, ct, tag::ab);+ dnnl::primitive_attr primitive_attr;+ primitive_attr.set_scratchpad_mode(dnnl::scratchpad_mode::user);++ const int src_mask = 0; // m, k per m+ auto scalea_md = dnnl::memory::desc(dnnl::memory::dims{1}, ct, dnnl::memory::format_tag::a);+ const int wei_mask = (1 << 1); // k,n per n+ auto scaleb_md = dnnl::memory::desc(dnnl::memory::dims{n}, ct, dnnl::memory::format_tag::a);++ // primitive_attr.set_fpmath_mode(dnnl::fpmath_mode::strict);+ primitive_attr.set_scales_mask(DNNL_ARG_SRC, src_mask);+ primitive_attr.set_scales_mask(DNNL_ARG_WEIGHTS, wei_mask);++ auto a_mem = dnnl::memory(a_in_md, eng, const_cast<void*>(a));+ auto b_mem = dnnl::memory(b_in_md, eng, const_cast<void*>(b));+ auto matmul_pd = bias ? dnnl::matmul::primitive_desc(eng, a_in_md, b_in_md, bias_md, c_md, primitive_attr)+ : dnnl::matmul::primitive_desc(eng, a_in_md, b_in_md, c_md, primitive_attr);++ auto scalea_mem = dnnl::memory(scalea_md, eng, const_cast<void*>(scale_a));+ auto scaleb_mem = dnnl::memory(scaleb_md, eng, const_cast<void*>(scale_b));+ auto c_mem = dnnl::memory(matmul_pd.dst_desc(), eng, c);++ auto scratchpad_md = matmul_pd.scratchpad_desc();++ auto scratchpad_mem = DnnlContext::Instance()->get_scratch_mem(scratchpad_md, q);++ auto matmul_prim = dnnl::matmul(matmul_pd);++ std::unordered_map<int, dnnl::memory> matmul_args;+ matmul_args.insert({DNNL_ARG_SRC, a_mem});+ matmul_args.insert({DNNL_ARG_WEIGHTS, b_mem});+ if (bias) matmul_args.insert({DNNL_ARG_BIAS, bias_mem});+ matmul_args.insert({DNNL_ARG_ATTR_SCALES | DNNL_ARG_SRC, scalea_mem});+ matmul_args.insert({DNNL_ARG_ATTR_SCALES | DNNL_ARG_WEIGHTS, scaleb_mem});+ matmul_args.insert({DNNL_ARG_DST, c_mem});+ matmul_args.insert({DNNL_ARG_SCRATCHPAD, scratchpad_mem});+ matmul_prim.execute(stream, matmul_args);+ if (q == nullptr) stream.wait();+ }++ // a:mxk:act b:nxk:s8 c:mxn:act+ // scale_b:n:act+ static void woq_s8(sycl::queue* q, int m, int n, int k, const void* a, const void* b, bool BT, void* c, dt act,+ void* scale_b, void* bias, int blocksize) {+#if ARK_XPU+ size_t qa_size = (size_t)m * k;+ size_t tmp_size = qa_size + m * sizeof(float);+ auto tmp_ptr = DnnlContext::Instance()->get_scratch_mem(tmp_size, 1, q);+ auto qa_ptr = (int8_t*)tmp_ptr;+ auto scalea_ptr = qa_ptr + qa_size;+ sycl_dyn_quant_s8(q, m, k, a, act, qa_ptr, scalea_ptr, 0);+ sycl_igemm_s8s8(q, m, n, k, qa_ptr, b, BT, c, act, scalea_ptr, scale_b, bias, blocksize);+#else+ size_t abs_size = (size_t)m * k * dnnl::memory::data_type_size(act);+ size_t qa_size = (size_t)m * k;+ size_t tmp_size = qa_size + 64 * sizeof(float) + abs_size;+ auto tmp_ptr = DnnlContext::Instance()->get_scratch_mem(tmp_size, 1, q);+ auto abs_ptr = (int8_t*)tmp_ptr;+ auto qa_ptr = abs_ptr + abs_size;+ auto maxa_ptr = qa_ptr + qa_size;+ auto scalea_ptr = maxa_ptr + sizeof(float) * 32;+ dyn_quant_s8(q, m, k, a, act, abs_ptr, qa_ptr, maxa_ptr, scalea_ptr);+ igemm_s8s8(q, m, n, k, qa_ptr, b, BT, c, act, scalea_ptr, scale_b, bias);+#endif+ }+ struct WeightQuantAttr {+ bool enabled{false};+ int mask{0};+ std::vector<dnnl::memory::dim> groups;+ dnnl::memory::dims scale_dims;+ dnnl::memory::dims scale_strides;+ dt scale_dt{dt::undef};+ const void* scales{nullptr};+ };++#if 0+ template <typename T>+ static void reorder_scales_to_group_major(sycl::queue* q, int n, int groups_k, const T* src, T* dst) {+ if (groups_k == 1) {+ q->memcpy(dst, src, size_t(n) * sizeof(T)).wait();+ return;+ }+ q->parallel_for(sycl::range<2>(groups_k, n), [=](sycl::id<2> idx) {+ int g = idx[0];+ int col = idx[1];+ dst[g * n + col] = src[col * groups_k + g];+ }).wait();+ }++ static const void* prepare_scale_buffer(sycl::queue* q, const void* scales, dt st, int n, int groups_k,+ bool& used_scratch) {+ used_scratch = false;+ if (!scales) return nullptr;+ if (groups_k <= 1) return scales;+ size_t elem_cnt = size_t(groups_k) * size_t(n);+ size_t bytes = elem_cnt * dnnl::memory::data_type_size(st);+ if (!bytes) return nullptr;+ auto dst = DnnlContext::Instance()->get_scratch_mem(bytes, 5, q);+ used_scratch = true;+ switch (st) {+ case dt::f32:+ reorder_scales_to_group_major(q, n, groups_k, static_cast<const float*>(scales), static_cast<float*>(dst));+ break;+ case dt::f16:+ reorder_scales_to_group_major(q, n, groups_k, static_cast<const sycl::half*>(scales),+ static_cast<sycl::half*>(dst));+ break;+ case dt::bf16:+ reorder_scales_to_group_major(q, n, groups_k, static_cast<const sycl::ext::oneapi::bfloat16*>(scales),+ static_cast<sycl::ext::oneapi::bfloat16*>(dst));+ break;+ default:+ used_scratch = false;+ return nullptr;+ }+ return dst;+ }++ static void gemm(sycl::queue* q, int m, int n, int k, const void* a, dt at, dnnl_dim_t stra0, dnnl_dim_t stra1,+ dnnl_dim_t stra2, const void* b, dt bt, dnnl_dim_t strb0, dnnl_dim_t strb1, dnnl_dim_t strb2,+ void* c, dt ct, dnnl_dim_t batches_a, dnnl_dim_t batches_b, const WeightQuantAttr* wq_attr = nullptr,+ bool sync = true) {+ auto& eng = *DnnlContext::Instance()->get_eng(q);+ auto& stream = *DnnlContext::Instance()->get_stream(q);+ dnnl::memory::dims a_dims = {batches_a, m, k};+ dnnl::memory::dims a_strides = {stra2, stra1, stra0};+ const auto a_in_md = dnnl::memory::desc(a_dims, at, a_strides);++ dnnl::memory::dims b_dims = {batches_b, k, n};+ dnnl::memory::dims b_strides = {strb2, strb1, strb0};+ const auto b_in_md = dnnl::memory::desc(b_dims, bt, b_strides);++ dnnl::memory::dims c_dims = {std::max(batches_a, batches_b), m, n};+ dnnl::memory::dims c_strides = {m * n, n, 1};+ const auto c_md = dnnl::memory::desc(c_dims, ct, c_strides);+ dnnl::primitive_attr primitive_attr;+ primitive_attr.set_scratchpad_mode(dnnl::scratchpad_mode::user);+ if (wq_attr && wq_attr->enabled) {+ primitive_attr.set_scales(DNNL_ARG_WEIGHTS, wq_attr->mask, wq_attr->groups, wq_attr->scale_dt);+ primitive_attr.set_fpmath_mode(dnnl::fpmath_mode::f16, true);+ }++ // primitive_attr.set_fpmath_mode(dnnl::fpmath_mode::strict);++ try {+ auto matmul_pd = dnnl::matmul::primitive_desc(eng, a_in_md, b_in_md, c_md, primitive_attr);+ auto a_mem = dnnl::memory(a_in_md, eng, const_cast<void*>(a));+ auto b_mem = dnnl::memory(b_in_md, eng, const_cast<void*>(b));+ auto c_mem = dnnl::memory(matmul_pd.dst_desc(), eng, c);++ auto scratchpad_md = matmul_pd.scratchpad_desc();++ auto scratchpad_mem = DnnlContext::Instance()->get_scratch_mem(scratchpad_md, q);++ auto matmul_prim = dnnl::matmul(matmul_pd);++ std::unordered_map<int, dnnl::memory> matmul_args;+ matmul_args.insert({DNNL_ARG_SRC, a_mem});+ matmul_args.insert({DNNL_ARG_WEIGHTS, b_mem});++ matmul_args.insert({DNNL_ARG_DST, c_mem});+ matmul_args.insert({DNNL_ARG_SCRATCHPAD, scratchpad_mem});+ dnnl::memory scale_mem;+ if (wq_attr && wq_attr->enabled) {+ auto scale_md = dnnl::memory::desc(wq_attr->scale_dims, wq_attr->scale_dt, wq_attr->scale_strides);+ scale_mem = dnnl::memory(scale_md, eng, const_cast<void*>(wq_attr->scales));+ matmul_args.insert({DNNL_ARG_ATTR_SCALES | DNNL_ARG_WEIGHTS, scale_mem});+ }+ matmul_prim.execute(stream, matmul_args);+ // Avoid unconditional host-side synchronization: letting the caller decide+ // enables better overlap/pipelining across ops.+ if (sync) {+ stream.wait();+ }+ } catch (const dnnl::error& e) {+ std::cerr << "DNNL Error: " << e.what() << std::endl;+ throw;+ } catch (const std::exception& e) {+ std::cerr << "Error: " << e.what() << std::endl;+ throw;+ }+ }++ static void row_gemm(sycl::queue& q, int8_t dev_idx, int m, int n, int k, const void* a, dt at, const void* b, dt bt,+ bool b_T, void* c, dt ct, const WeightQuantAttr* wq_attr = nullptr, bool sync = true) {+ gemm(q, dev_idx, m, n, k, a, at, 1, k, k * m, b, bt, b_T ? k : 1, b_T ? 1 : n, n * k, c, ct, 1, 1, wq_attr, sync);+ }++ template <typename QueueT>+ static void gemv(QueueT& q, int8_t dev_idx, int m, int n, int k, const void* a, dt at, const void* x, dt xt, void* y,+ dt yt, const WeightQuantAttr* wq_attr = nullptr, bool weights_transposed = true, bool sync = true) {+ // GEMV degenerates to GEMM with a single-row activation; default to transposed weights+ // to match the row-major NT layout used by weight-only quantization.+ row_gemm(q, dev_idx, m, n, k, a, at, x, xt, weights_transposed, y, yt, wq_attr, sync);+ }++ static float pow2_int(int power) {+ float result = 1.0f;+ if (power >= 0) {+ for (int i = 0; i < power; ++i) result *= 2.0f;+ } else {+ for (int i = 0; i < -power; ++i) result *= 0.5f;+ }+ return result;+ }++ static float decode_fp8_e4m3(uint8_t value) {+ constexpr int bias = 7;+ constexpr float max_val = 448.0f;+ int sign = (value & 0x80) ? -1 : 1;+ int exp = (value >> 3) & 0x0F;+ int mant = value & 0x07;+ if (exp == 0) {+ if (mant == 0) return 0.0f;+ float frac = mant / 8.0f;+ float val = frac * pow2_int(1 - bias);+ return sign * val;+ }+ if (exp == 0x0F) {+ return sign * max_val;+ }+ float frac = 1.0f + mant / 8.0f;+ float val = frac * pow2_int(exp - bias);+ return sign * val;+ }++ static float decode_fp8_e5m2(uint8_t value) {+ constexpr int bias = 15;+ constexpr float max_val = 57344.0f;+ int sign = (value & 0x80) ? -1 : 1;+ int exp = (value >> 2) & 0x1F;+ int mant = value & 0x03;+ if (exp == 0) {+ if (mant == 0) return 0.0f;+ float frac = mant / 4.0f;+ float val = frac * pow2_int(1 - bias);+ return sign * val;+ }+ if (exp == 0x1F) {+ return sign * max_val;+ }+ float frac = 1.0f + mant / 4.0f;+ float val = frac * pow2_int(exp - bias);+ return sign * val;+ }++ static float decode_fp8(uint8_t value, bool is_e4m3) {+ return is_e4m3 ? decode_fp8_e4m3(value) : decode_fp8_e5m2(value);+ }++ static void woq_gemm(sycl::queue* q, int m, int n, int k, const void* a, dt at, const void* b, dt bt, void* c, dt ct,+ const void* scales, dt st, int blocksize) {+ bool weights_are_fp8 = (bt == dt::f8_e4m3 || bt == dt::f8_e5m2);+ if (weights_are_fp8 && blocksize > 0 && k > 0 && (k % blocksize) == 0) {+ WeightQuantAttr attr;+ attr.enabled = true;+ int groups_k = k / blocksize;+ bool single_group = groups_k == 1;+ // oneDNN matmul weights use dims {batch, k, n}, so per-column scales map to bit 2 and+ // additional k-group scales require bit 1 as well.+ attr.mask = single_group ? (1 << 2) : ((1 << 1) | (1 << 2));+ if (!single_group) {+ attr.groups = {static_cast<dnnl::memory::dim>(blocksize), 1};+ }+ bool used_scratch = false;+ const void* scale_ptr = prepare_scale_buffer(q, scales, st, n, groups_k, used_scratch);+ if (scale_ptr) {+ if (single_group) {+ attr.scale_dims = {static_cast<dnnl::memory::dim>(n)};+ attr.scale_strides = {1};+ } else {+ attr.scale_dims = {static_cast<dnnl::memory::dim>(groups_k), static_cast<dnnl::memory::dim>(n)};+ attr.scale_strides = {static_cast<dnnl::memory::dim>(n), 1};+ }+ attr.scale_dt = st;+ attr.scales = scale_ptr;++ const void* a_exec = a;+ dt a_exec_dt = at;+ if (at == dt::f32) {+ size_t elems = size_t(m) * k;+ size_t bytes = elems * sizeof(sycl::half);+ auto a_buf = DnnlContext::Instance()->get_scratch_mem(bytes, 6, q);+ auto a_src = static_cast<const float*>(a);+ auto a_dst = static_cast<sycl::half*>(a_buf);+ q->parallel_for(sycl::range<1>(elems), [=](sycl::id<1> idx) {+ size_t i = idx[0];+ a_dst[i] = static_cast<sycl::half>(a_src[i]);+ }).wait();+ a_exec = a_buf;+ a_exec_dt = dt::f16;+ }++ void* c_exec = c;+ dt c_exec_dt = ct;+ bool need_c_cast = false;+ if (ct == dt::f32) {+ size_t elems = size_t(m) * n;+ size_t bytes = elems * sizeof(sycl::half);+ c_exec = DnnlContext::Instance()->get_scratch_ptr(bytes, 7, q, dev_idx);+ c_exec_dt = dt::f16;+ need_c_cast = true;+ }++ row_gemm(q, dev_idx, m, n, k, a_exec, a_exec_dt, b, bt, true, c_exec, c_exec_dt, &attr);++ if (need_c_cast) {+ size_t elems = size_t(m) * n;+ auto src_half = static_cast<sycl::half*>(c_exec);+ auto dst_f32 = static_cast<float*>(c);+ q.parallel_for(sycl::range<1>(elems), [=](sycl::id<1> idx) {+ size_t i = idx[0];+ dst_f32[i] = static_cast<float>(src_half[i]);+ }).wait();+ }+ return;+ }+ }++ bool need_fp32_compute = weights_are_fp8 || ct == dt::f16 || at != dt::f32;+ dt compute_dt = need_fp32_compute ? dt::f32 : ct;++ const void* a_compute = a;+ if (at != compute_dt) {+ size_t a_bytes = size_t(m) * k * sizeof(float);+ auto a_buf = DnnlContext::Instance()->get_scratch_ptr(a_bytes, 2, q, dev_idx);+ auto a_src = static_cast<const sycl::half*>(a);+ auto a_dst = static_cast<float*>(a_buf);+ q.parallel_for(sycl::range<1>(size_t(m) * k), [=](sycl::id<1> idx) {+ size_t i = idx[0];+ a_dst[i] = static_cast<float>(a_src[i]);+ }).wait();+ a_compute = a_buf;+ }++ const void* b_compute = b;+ if (weights_are_fp8) {+ size_t b_bytes = size_t(n) * k * sizeof(float);+ auto b_buf = DnnlContext::Instance()->get_scratch_ptr(b_bytes, 3, q, dev_idx);+ auto b_src = static_cast<const uint8_t*>(b);+ auto b_dst = static_cast<float*>(b_buf);+ bool is_e4m3 = (bt == dt::f8_e4m3);+ bool scale_is_f32 = (st == dt::f32);+ auto scale_f32 = static_cast<const float*>(scales);+ auto scale_f16 = static_cast<const sycl::half*>(scales);+ q.parallel_for(sycl::range<1>(size_t(n) * k), [=](sycl::id<1> idx) {+ size_t linear = idx[0];+ int col = linear % n;+ float scale = 1.0f;+ if (scale_is_f32) {+ scale = scale_f32[col];+ } else {+ scale = static_cast<float>(scale_f16[col]);+ }+ float dq = decode_fp8(b_src[linear], is_e4m3) * scale;+ b_dst[linear] = dq;+ }).wait();+ b_compute = b_buf;+ compute_dt = dt::f32;+ }++ void* c_compute = c;+ if (ct != compute_dt) {+ size_t c_bytes = size_t(m) * n * sizeof(float);+ c_compute = DnnlContext::Instance()->get_scratch_ptr(c_bytes, 4, q, dev_idx);+ }++ row_gemm(q, dev_idx, m, n, k, a_compute, compute_dt, b_compute, compute_dt, true, c_compute, compute_dt);++ if (ct != compute_dt) {+ auto src = static_cast<float*>(c_compute);+ auto dst = static_cast<sycl::half*>(c);+ q.parallel_for(sycl::range<1>(size_t(m) * n), [=](sycl::id<1> idx) {+ size_t i = idx[0];+ dst[i] = static_cast<sycl::half>(src[i]);+ }).wait();+ }+ }++ static void dequant_fp8(sycl::queue* q, int k, int n, const void* src, dt src_dt, void* dst, dt dst_dt,+ const void* scales, dt st, int blocksize, bool src_col_major = true, bool sync = true) {+ bool weights_are_fp8 = (src_dt == dt::f8_e4m3 || src_dt == dt::f8_e5m2);+ if (!weights_are_fp8 || !src || !dst || !scales) return;++ int groups_k = (blocksize > 0 && (k % blocksize) == 0) ? (k / blocksize) : 1;+ bool single_group = groups_k == 1;+ dnnl::memory::dims mat_dims = {static_cast<dnnl::memory::dim>(k), static_cast<dnnl::memory::dim>(n)};+ dnnl::memory::dims src_strides = src_col_major ? dnnl::memory::dims{1, static_cast<dnnl::memory::dim>(k)}+ : dnnl::memory::dims{static_cast<dnnl::memory::dim>(n), 1};+ dnnl::memory::dims dst_strides = {static_cast<dnnl::memory::dim>(n), 1};+ auto src_md = dnnl::memory::desc(mat_dims, src_dt, src_strides);+ auto dst_md = dnnl::memory::desc(mat_dims, dst_dt, dst_strides);++ dnnl::memory::dims scale_dims;+ dnnl::memory::dims scale_strides;+ if (single_group) {+ scale_dims = {static_cast<dnnl::memory::dim>(n)};+ scale_strides = {1};+ } else {+ scale_dims = {static_cast<dnnl::memory::dim>(groups_k), static_cast<dnnl::memory::dim>(n)};+ scale_strides = {static_cast<dnnl::memory::dim>(n), 1};+ }++ dnnl::primitive_attr attr;+ attr.set_scratchpad_mode(dnnl::scratchpad_mode::user);+ int scale_mask = single_group ? (1 << 1) : ((1 << 0) | (1 << 1));+ attr.set_scales(DNNL_ARG_SRC, scale_mask, scale_dims, st);++ bool used_scratch = false;+ const void* scale_ptr = prepare_scale_buffer(q, scales, st, n, groups_k, used_scratch);+ if (!scale_ptr) return;++ GETCTX();+ auto reorder_pd = dnnl::reorder::primitive_desc(eng, src_md, eng, dst_md, attr);+ auto reorder_prim = dnnl::reorder(reorder_pd);++ auto src_mem = dnnl::memory(src_md, eng, const_cast<void*>(src));+ auto dst_mem = dnnl::memory(dst_md, eng, dst);+ auto scratchpad_mem = DnnlContext::Instance()->get_scratch_mem(reorder_pd.scratchpad_desc(), q);+ auto scale_md = dnnl::memory::desc(scale_dims, st, scale_strides);+ auto scale_mem = dnnl::memory(scale_md, eng, const_cast<void*>(scale_ptr));++ std::unordered_map<int, dnnl::memory> reorder_args;+ reorder_args.insert({DNNL_ARG_FROM, src_mem});+ reorder_args.insert({DNNL_ARG_TO, dst_mem});+ reorder_args.insert({DNNL_ARG_SCRATCHPAD, scratchpad_mem});+ reorder_args.insert({DNNL_ARG_ATTR_SCALES | DNNL_ARG_SRC, scale_mem});+ reorder_prim.execute(stream, reorder_args);+ if (sync) {+ stream.wait();+ }+ }+#endif+};++} // namespace arkdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hppnew file mode 100644index 000000000..ff0d4bee3--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sage_fwd_kernel.hpp@@ -0,0 +1,310 @@+/***************************************************************************************************+ * Copyright (C) 2025 - 2026 Intel Corporation, All rights reserved.+ * SPDX-License-Identifier: BSD-3-Clause+ *+ * Redistribution and use in source and binary forms, with or without+ * modification, are permitted provided that the following conditions are met:+ *+ * 1. Redistributions of source code must retain the above copyright notice,+ *this list of conditions and the following disclaimer.+ *+ * 2. Redistributions in binary form must reproduce the above copyright notice,+ * this list of conditions and the following disclaimer in the documentation+ * and/or other materials provided with the distribution.+ *+ * 3. Neither the name of the copyright holder nor the names of its+ * contributors may be used to endorse or promote products derived from+ * this software without specific prior written permission.+ *+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE+ *ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE+ *LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR+ *CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF+ *SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS+ *INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN+ *CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)+ *ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE+ *POSSIBILITY OF SUCH DAMAGE.+ *+ **************************************************************************************************/++#pragma once++#include <cmath>+#include "cutlass/cutlass.h"+#include "cutlass/gemm/dispatch_policy.hpp"+#include "cutlass/gemm/gemm.h"+#include "cutlass/kernel_hardware_info.hpp"++#include "cute/util/type_traits.hpp"+#include "flash_attention_v2/collective/fmha_fusion.hpp"+#include "flash_attention_v2/collective/xe_fmha_fwd_epilogue.hpp"+#include "flash_attention_v2/collective/xe_fmha_fwd_mainloop.hpp"+#include "xe_sagev1_fwd_mainloop.hpp"+#include "flash_attention_v2/kernel/xe_tile_scheduler.hpp"++namespace cutlass::fmha::kernel {++using namespace cute;++///////////////////////////////////////////////////////////////////////////////+template <bool IsVarLen_ = false>+struct SageProblemShape {+ using SeqLenType = cute::conditional_t<IsVarLen_, cutlass::fmha::collective::VariableLength, int>;+ int batch;+ int num_heads_q, num_heads_kv;+ SeqLenType seq_len_qo, seq_len_kv, seq_len_kv_cache;+ int head_size_qk, head_size_vo;+};++///////////////////////////////////////////////////////////////////////////////++template <class ProblemShape_, class CollectiveMainloop_, class CollectiveEpilogue_, class TileScheduler_>+class XeSageFwdKernel {+ public:+ //+ // Type Aliases+ //+ using ProblemShape = ProblemShape_;+ using VariableLength = cutlass::fmha::collective::VariableLength;+ static constexpr bool is_var_len = cutlass::fmha::collective::is_variable_length_v<typename ProblemShape::SeqLenType>;+ // Mainloop derived types+ using CollectiveMainloop = CollectiveMainloop_;+ using MainloopArguments = typename CollectiveMainloop::Arguments;+ using MainloopParams = typename CollectiveMainloop::Params;++ using TiledMMAQK = typename CollectiveMainloop::TiledMMAQK;+ using TiledMMAPV = typename CollectiveMainloop::TiledMMAPV;+ using TileShapeQK = typename CollectiveMainloop::TileShapeQK;+ using TileShapePV = typename CollectiveMainloop::TileShapePV;+ using SubgroupLayoutQK = typename CollectiveMainloop::SubgroupLayoutQK;+ using ElementQ = typename CollectiveMainloop::TensorQ::element_type;+ using ElementK = typename CollectiveMainloop::TensorK::element_type;+ using ElementV = typename CollectiveMainloop::TensorV::element_type;++ using StrideQ = decltype(stride(typename CollectiveMainloop::TensorQ{}));+ using StrideK = decltype(stride(typename CollectiveMainloop::TensorK{}));+ using StrideV = decltype(stride(typename CollectiveMainloop::TensorV{}));++ using SGPerWG = typename CollectiveMainloop::SGPerWG;++ using FragA = typename CollectiveMainloop::FragA;+ using FragARow = typename CollectiveMainloop::FragARow;++ // Tile scheduler derived types+ using TileScheduler = TileScheduler_;+ using TileSchedulerParams = typename TileScheduler::Params;++ // Epilogue derived types+ using CollectiveEpilogue = CollectiveEpilogue_;+ using EpilogueArguments = typename CollectiveEpilogue::Arguments;+ using EpilogueParams = typename CollectiveEpilogue::Params;++ using TileShapeO = typename CollectiveEpilogue::TileShapeO;+ using ElementO = typename CollectiveEpilogue::TensorO::element_type;+ using StrideO = decltype(stride(typename CollectiveEpilogue::TensorO{}));++ // Kernel level shared memory storage+ using MainloopSharedStorage = typename CollectiveMainloop::SharedStorage;+ using EpilogueSharedStorage = typename CollectiveEpilogue::SharedStorage;+ union SharedStorage {+ MainloopSharedStorage mainloop;+ EpilogueSharedStorage epilogue;+ };++ static constexpr int SharedStorageSize = is_empty_v<SharedStorage> ? size_t(0) : sizeof(SharedStorage);++ // Device side arguments+ struct KernelArguments {+ ProblemShape shape;+ const ElementQ* Q;+ StrideQ dQ;+ const ElementK* K;+ StrideK dK;+ const ElementV* V;+ StrideV dV;+ ElementO* O;+ StrideO dO;+ const ElementK* K_cache;+ StrideK dK_cache{};+ const ElementV* V_cache;+ StrideV dV_cache{};+ };+ using KernelParams = KernelArguments;++ struct Arguments {+ KernelArguments kernel{};+ MainloopArguments mainloop{};+ EpilogueArguments epilogue{};+ KernelHardwareInfo hw_info{};+ };++ // Kernel entry point API+ struct Params {+ KernelParams kernel;+ MainloopParams mainloop;+ EpilogueParams epilogue;+ TileSchedulerParams scheduler;+ };++ //+ // Methods+ //++ static Params to_underlying_arguments(Arguments const& args, void* workspace) {+ return {args.kernel, CollectiveMainloop::to_underlying_arguments(args.mainloop, workspace),+ CollectiveEpilogue::to_underlying_arguments(args.epilogue, workspace),+ TileScheduler::to_underlying_arguments(args.kernel.shape, args.hw_info, TileShapeO{})};+ }++ static bool can_implement(Arguments const& args) {+ return CollectiveMainloop::can_implement(args.mainloop) && CollectiveEpilogue::can_implement(args.epilogue);+ }++ static int get_workspace_size(Arguments const& args) { return 0; }++ static cutlass::Status initialize_workspace(Arguments const& args, void* workspace = nullptr,+ cudaStream_t stream = nullptr, CudaHostAdapter* cuda_adapter = nullptr) {+ return Status::kSuccess;+ }++ static dim3 get_grid_shape(Params const& params) {+ return TileScheduler::template get_grid_shape<SGPerWG::value>(params.scheduler);+ }++ static dim3 get_block_shape() { return dim3(SGPerWG::value * intel::sg_size, 1, 1); }++ CUTLASS_DEVICE+ Shape<int, int, int> get_sequence_length_shape(ProblemShape const& problem_shape, int const& batch) {+ if constexpr (is_var_len) {+ return cutlass::fmha::collective::apply_variable_length(+ Shape<VariableLength, VariableLength, VariableLength>{problem_shape.seq_len_qo, problem_shape.seq_len_kv,+ problem_shape.seq_len_kv_cache},+ batch);+ } else {+ return Shape<int, int, int>{problem_shape.seq_len_qo, problem_shape.seq_len_kv, problem_shape.seq_len_kv_cache};+ }+ }++ CUTLASS_DEVICE+ void operator()(Params const& params, char* smem_buf) {+ using namespace sycl::ext::oneapi::this_work_item;++ SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(smem_buf);++ auto& p = params.kernel;+ ProblemShape const& s = p.shape;+ int head_group_q = s.num_heads_q / s.num_heads_kv;++ int thr_id = int(ThreadIdxX());+ int sub_group_id = thr_id / intel::sg_size;+ int q_sg_tile = get<0>(shape_div(TileShapeQK{}, shape(SubgroupLayoutQK{})));++ auto cS = make_identity_tensor(take<0, 2>(TiledMMAQK{}.tile_mnk()));+ auto tScS = TiledMMAQK{}.get_slice(thr_id).partition_C(cS);+ auto q_offset_wi = get<0>(tScS(0));+ auto q_offset_sg = group_broadcast(sycl::ext::oneapi::this_work_item::get_sub_group(), q_offset_wi, 0);++ TileScheduler tile_scheduler{params.scheduler};++ CUTLASS_PRAGMA_NO_UNROLL+ for (; tile_scheduler.is_valid(); ++tile_scheduler) {+ auto [blk_q, blk_v, head_q, idx_b] = tile_scheduler.get_block_coord(); // (Q,V,h,b)+ auto blk_qv = make_coord(blk_q, blk_v);+ int head = head_q / head_group_q;++ auto sequence_length_shape = get_sequence_length_shape(s, idx_b);+ auto [seq_len_qo, seq_len_kv, seq_len_kv_cache] = sequence_length_shape;+ if (blk_q * get<0>(TileShapeQK{}) >= seq_len_qo) continue;++ auto offset = cute::min(seq_len_qo, seq_len_kv);+ auto discard_seq_coord = seq_len_qo - offset;+ auto full_tile_offset = seq_len_kv - offset;+ int seq_coord = cute::min(seq_len_qo, (blk_q * get<0>(TileShapeQK{}) + q_offset_sg));++ if (CollectiveMainloop::CausalMask && seq_coord < discard_seq_coord) continue;+ const int seq_len_new = CollectiveMainloop::CausalMask+ ? full_tile_offset + cute::min(seq_len_kv, seq_coord - discard_seq_coord) + q_sg_tile+ : seq_len_kv;+ const int seq_len = seq_len_new + seq_len_kv_cache;+ const int k_blocks = cute::ceil_div(seq_len, get<1>(TileShapeQK{}));++ int offset_q = 0, offset_k = 0, offset_v = 0, offset_o = 0;+ int offset_k_cache = 0, offset_v_cache = 0;+ if constexpr (is_var_len) {+ auto qo_cumulative = s.seq_len_qo.cumulative_length;+ auto kv_cumulative = s.seq_len_kv.cumulative_length;+ offset_q = s.num_heads_q * s.head_size_qk * qo_cumulative[idx_b];+ offset_k = s.num_heads_kv * s.head_size_qk * kv_cumulative[idx_b];+ offset_v = s.num_heads_kv * s.head_size_vo * kv_cumulative[idx_b];+ offset_o = s.num_heads_q * s.head_size_vo * qo_cumulative[idx_b];+ if (s.seq_len_kv_cache.cumulative_length) {+ auto kv_cumulative_cache = s.seq_len_kv_cache.cumulative_length;+ offset_k_cache = s.num_heads_kv * s.head_size_qk * kv_cumulative_cache[idx_b];+ offset_v_cache = s.num_heads_kv * s.head_size_vo * kv_cumulative_cache[idx_b];+ }+ }++ auto batch_dim = is_var_len ? 1 : s.batch;+ auto shape_Q = make_shape(seq_len_qo, s.head_size_qk, s.num_heads_q, batch_dim);+ auto shape_K = make_shape(seq_len_kv, s.head_size_qk, s.num_heads_kv, batch_dim);+ auto shape_V = make_shape(s.head_size_vo, seq_len_kv, s.num_heads_kv, batch_dim);+ auto shape_O = make_shape(seq_len_qo, s.head_size_vo, s.num_heads_q, batch_dim);++ auto shape_K_cache = make_shape(seq_len_kv_cache, s.head_size_qk, s.num_heads_kv, batch_dim);+ auto shape_V_cache = make_shape(s.head_size_vo, seq_len_kv_cache, s.num_heads_kv, batch_dim);++ auto dcQ = const_cast<ElementQ*>(p.Q + offset_q);+ auto dcK = const_cast<ElementK*>(p.K + offset_k);+ auto dcV = const_cast<ElementV*>(p.V + offset_v);+ auto dcK_cache = const_cast<ElementK*>(p.K_cache + offset_k_cache);+ auto dcV_cache = const_cast<ElementV*>(p.V_cache + offset_v_cache);+ int seq_q_pad = (seq_len_qo + params.mainloop.scale_block_size - 1) / params.mainloop.scale_block_size;+ int seq_kv_pad = (seq_len_kv + params.mainloop.scale_block_size - 1) / params.mainloop.scale_block_size;+ auto scaleQ = params.mainloop.scale_block_size+ ? (float*)params.mainloop.qscale + (idx_b * s.num_heads_q * seq_q_pad + head_q * seq_q_pad)+ : nullptr; // per head qscale+ auto scaleK = params.mainloop.scale_block_size+ ? (float*)params.mainloop.kscale + (idx_b * s.num_heads_kv * seq_kv_pad + head * seq_kv_pad)+ : nullptr; // per head kscale+ auto ptrO = p.O + offset_o;++ auto stride_q = is_var_len ? cutlass::make_cute_packed_stride(StrideQ{}, shape_Q) : p.dQ;+ auto stride_k = is_var_len ? cutlass::make_cute_packed_stride(StrideK{}, shape_K) : p.dK;+ auto stride_v = is_var_len ? cutlass::make_cute_packed_stride(StrideV{}, shape_V) : p.dV;+ auto stride_o = is_var_len ? cutlass::make_cute_packed_stride(StrideO{}, shape_O) : p.dO;+ auto stride_k_cache = is_var_len ? cutlass::make_cute_packed_stride(StrideK{}, shape_K_cache) : p.dK_cache;+ auto stride_v_cache = is_var_len ? cutlass::make_cute_packed_stride(StrideV{}, shape_V_cache) : p.dV_cache;++ Tensor Q = make_tensor(make_gmem_ptr(dcQ), make_layout(shape_Q, stride_q));+ Tensor K = make_tensor(make_gmem_ptr(dcK), make_layout(shape_K, stride_k));+ Tensor V = make_tensor(make_gmem_ptr(dcV), make_layout(shape_V, stride_v));+ Tensor K_cache = make_tensor(make_gmem_ptr(dcK_cache), make_layout(shape_K_cache, stride_k_cache));+ Tensor V_cache = make_tensor(make_gmem_ptr(dcV_cache), make_layout(shape_V_cache, stride_v_cache));+ Tensor O = make_tensor(make_gmem_ptr(ptrO), make_layout(shape_O, stride_o));++ // O accumulator types+ FragA tArA;+ FragARow tA_max, tA_sum;++ // Main loop+ int l_coord = is_var_len ? 0 : idx_b;+ CollectiveMainloop mainloop(params.mainloop, shared_storage.mainloop);+ mainloop(Q(_, _, head_q, l_coord), K(_, _, head, l_coord), V(_, _, head, l_coord), tArA, tA_max, tA_sum, blk_qv,+ 0, k_blocks, k_blocks, thr_id, seq_len, seq_len_kv_cache, idx_b, scaleQ, scaleK, full_tile_offset,+ discard_seq_coord, K_cache(_, _, head, l_coord), V_cache(_, _, head, l_coord));++ if constexpr (!is_empty_v<MainloopSharedStorage> && !is_empty_v<EpilogueSharedStorage>) {+ sycl::group_barrier(get_work_group<3>());+ }++ // Epilogue+ CollectiveEpilogue epilogue{params.epilogue, shared_storage.epilogue};+ epilogue(O(_, _, head_q, l_coord), tArA, tA_max, tA_sum, blk_qv, thr_id);+ }+ }+};++} // namespace cutlass::fmha::kerneldiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hppnew file mode 100644index 000000000..d4a6eaa5a--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sagev1_fwd_mainloop.hpp@@ -0,0 +1,563 @@+/***************************************************************************************************+ * Copyright (C) 2025 Intel Corporation, All rights reserved.+ * SPDX-License-Identifier: BSD-3-Clause+ *+ * Redistribution and use in source and binary forms, with or without+ * modification, are permitted provided that the following conditions are met:+ *+ * 1. Redistributions of source code must retain the above copyright notice,+ *this list of conditions and the following disclaimer.+ *+ * 2. Redistributions in binary form must reproduce the above copyright notice,+ * this list of conditions and the following disclaimer in the documentation+ * and/or other materials provided with the distribution.+ *+ * 3. Neither the name of the copyright holder nor the names of its+ * contributors may be used to endorse or promote products derived from+ * this software without specific prior written permission.+ *+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE+ *ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE+ *LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR+ *CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF+ *SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS+ *INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN+ *CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)+ *ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE+ *POSSIBILITY OF SUCH DAMAGE.+ *+ **************************************************************************************************/++#pragma once++#include "cute/util/print_tensor.hpp"+#include "cutlass/cutlass.h"+#include "cutlass/gemm/dispatch_policy.hpp"++#include "cute/algorithm/functional.hpp"+#include "cute/algorithm/gemm.hpp"+#include "cute/algorithm/subgroup_algorithms.hpp"+#include "cute/atom/mma_atom.hpp"+#include "flash_attention_v2/collective/fmha_fusion.hpp"+#include <cmath>+#include <cute/util/xe_split_barrier.hpp>+namespace cutlass::sage {++template <int Stages>+class XeDefault {}; // Default FMHA mainloop, P in registers.++}; // namespace cutlass::sage++namespace cutlass::fmha::collective {++using namespace cute;++/////////////////////////////////////////////////////////////////////////////////////////////////++template <class DispatchPolicy_, bool CausalMask_, bool FullMask_, bool CachedKV_, bool PagedKV_,+ class TiledMMAQK_, // Tiling for Q*K GEMM+ class TiledMMAPV_, // Tiling for P*V GEMM+ int VTiles_, // # of tiles in V dimension+ class TensorQ_, // Global Q/K/V tensors+ class TensorK_, class TensorV_, class TensorK_cache_, class TensorV_cache_,+ class TiledCopyQ_ = void, // Optional TiledCopy for loading Q+ class TiledCopyK_ = void, // Optional TiledCopy for loading K+ class TiledCopyV_ = void, // Optional TiledCopy for loading V+ class TiledCopyK_cache_ = void,+ class TiledCopyV_cache_ = void> // Optional TiledCopy for loading V_cache+struct SAGEV1FwdMainloop {+ static_assert(cutlass::detail::dependent_false<DispatchPolicy_>, "Could not find a mainloop specialization.");+};++/////////////////////////////////////////////////////////////////////////////////////////////////++template <int Stages, bool CausalMask_, bool FullMask_, bool CachedKV_, bool PagedKV_, class TiledMMAQK_,+ class TiledMMAPV_, int VTiles_, class TensorQ_, class TensorK_, class TensorV_, class TensorK_cache_,+ class TensorV_cache_, class TiledCopyQ_, class TiledCopyK_, class TiledCopyV_, class TiledCopyK_cache_,+ class TiledCopyV_cache_>+struct SAGEV1FwdMainloop<sage::XeDefault<Stages>, CausalMask_, FullMask_, CachedKV_, PagedKV_, TiledMMAQK_, TiledMMAPV_,+ VTiles_, TensorQ_, TensorK_, TensorV_, TensorK_cache_, TensorV_cache_, TiledCopyQ_,+ TiledCopyK_, TiledCopyV_, TiledCopyK_cache_, TiledCopyV_cache_> {+ //+ // Type Aliases+ //+ using TiledMMAQK = TiledMMAQK_;+ using TiledMMAPV = TiledMMAPV_;+ using TileShapeQK = decltype(TiledMMAQK{}.tile_mnk());+ using TileShapePV = decltype(TiledMMAPV{}.tile_mnk());+ static constexpr int VTiles = VTiles_;+ using SubgroupLayoutQK = decltype(TiledMMAQK{}.get_atom_layout_mnk());+ using SGPerWG = decltype(product(take<1, 4>(shape(typename TiledMMAQK::ThrLayoutVMNK{}))));++ using TensorQ = TensorQ_;+ using TensorK = TensorK_;+ using TensorV = TensorV_;++ using TensorQ2D = decltype(TensorQ_{}(append<rank_v<TensorQ_>>(make_coord(_, _), 0)));+ using TensorK2D = decltype(TensorK_{}(append<rank_v<TensorK_>>(make_coord(_, _), 0)));+ using TensorV2D = decltype(TensorV_{}(append<rank_v<TensorV_>>(make_coord(_, _), 0)));++ using TiledCopyQ =+ conditional_t<is_void_v<TiledCopyQ_>, decltype(make_block_2d_copy_A(TiledMMAQK{}, TensorQ2D{})), TiledCopyQ_>;+ using TiledCopyK =+ conditional_t<is_void_v<TiledCopyK_>, decltype(make_block_2d_copy_B(TiledMMAQK{}, TensorK2D{})), TiledCopyK_>;+ using TiledCopyV =+ conditional_t<is_void_v<TiledCopyV_>, decltype(make_block_2d_copy_B(TiledMMAPV{}, TensorV2D{})), TiledCopyV_>;+ using TensorK_cache = TensorK_cache_;+ using TensorV_cache = TensorV_cache_;+ using TensorK_cache2D = decltype(TensorK_cache_{}(append<rank_v<TensorK_cache_>>(make_coord(_, _), 0)));+ using TensorV_cache2D = decltype(TensorV_cache_{}(append<rank_v<TensorV_cache_>>(make_coord(_, _), 0)));+ using TiledCopyK_cache =+ conditional_t<is_void_v<TiledCopyK_cache_>, decltype(make_block_2d_copy_B(TiledMMAQK{}, TensorK_cache2D{})),+ TiledCopyK_cache_>;+ using TiledCopyV_cache =+ conditional_t<is_void_v<TiledCopyV_cache_>, decltype(make_block_2d_copy_B(TiledMMAPV{}, TensorV_cache2D{})),+ TiledCopyV_cache_>;++ // TODO: static_asserts on TiledMMAPV here...++ //+ // Accumulator types+ //+ // FragS: accumulator for Q*K MMA+ // FragO: accumulator for P*V MMAs.+ // Note: v mode may be split into multiple pieces+ // to reduce register pressure.+ // Frag*Row types are reductions of the corresponding Frag* types+ // over rows.+ //+ template <typename TiledMMA>+ using FragC = decltype(TiledMMA{}.get_slice(0).partition_sg_fragment_C(+ make_identity_tensor(select<0, 1>(TiledMMA{}.tile_mnk()))));++ using FragS = FragC<TiledMMAQK>;+ using FragSRow = decltype(reduce<1>(FragS{}, sycl::plus<void>{}));+ using ElementS = typename TiledMMAQK::ValTypeD;+ using ElementM = typename TiledMMAQK::ValTypeA;+ using ElementP = float;+ using FragP = decltype(make_subgroup_tensor(make_fragment_like<ElementP>(typename FragS::Base{}),+ decltype(FragS{}.tv_layout()){}));++ using FragPRow = decltype(reduce<1>(FragP{}, sycl::plus<void>{}));+ using SingleFragA = FragC<TiledMMAPV>; // (atom val,q',v')+ using FragA = expand_sg_fragment_t<SingleFragA, 1, VTiles>; // (atom val,q',v',VV)+ using FragARow = decltype(reduce<1>(FragA{}, sycl::plus<void>{}));+ using ElementA = typename TiledMMAPV::ValTypeD;++ static constexpr bool CausalMask = CausalMask_;+ static constexpr bool CachedKV = CachedKV_;+ static constexpr bool PagedKV = PagedKV_;++ // User-facing arguments+ struct Arguments {+ float const scale;+ float const* mask = nullptr;+ int scale_block_size = 0; // if non-zero, apply scaling in blocks of this size (for block-sparse attention)+ float const* qscale = nullptr;+ float const* kscale = nullptr;+ int const* ptr_page_table = nullptr;+ int page_size = 0;+ int const* num_pages_per_seq = nullptr;+ };++ // Kernel-facing parameters+ using Params = Arguments;++ // SLM data+ struct SharedStorage {};++ Params params;++ //+ // Methods+ //++ SAGEV1FwdMainloop(Params const& params_, SharedStorage&) : params(params_) {}++ static constexpr Params to_underlying_arguments(Arguments const& args, void* /* workspace */) {+ constexpr double kLog2e = 1.4426950408889634074; // log_2(e)+ float val = args.scale * static_cast<float>(kLog2e);+ return Params{val, args.mask, args.scale_block_size, args.qscale,+ args.kscale, args.ptr_page_table, args.page_size, args.num_pages_per_seq};+ }++ CUTLASS_HOST_DEVICE static bool can_implement(Arguments const&) { return true; }++ CUTLASS_DEVICE+ int get_physical_k_tile(int K, int l_coord, int seq_len_kv_cache) {+ int next_page_logical_idx = K * get<1>(TileShapeQK{}) / params.page_size;+ // get<1>(TileShapeQK{}) usually smaller than page_size.+ // assuming page_size is multiple of get<1>(TileShapeQK{})+ int tiles_per_page = params.page_size / get<1>(TileShapeQK{});+ int batch_offset =+ params.num_pages_per_seq ? params.num_pages_per_seq[l_coord] : l_coord * (seq_len_kv_cache / params.page_size);++ return params.ptr_page_table[batch_offset + next_page_logical_idx] * tiles_per_page + K % tiles_per_page;+ }++ template <typename QVCoord>+ CUTLASS_DEVICE void operator()(TensorQ2D const& Q_2D, // (q,d)+ TensorK2D const& K_2D, // (k,d)+ TensorV2D const& V_2D, // (d,k)+ FragA& tArA, // Output accumulator (q,v)+ FragARow& tA_max, // Softmax row-wise max accumulator+ FragARow& tA_sum, // Softmax row-wise sum accumulator+ QVCoord blk_qv, // WG tile indices: (Q,V)+ int blk_k0, // K block range: [K0,K1)+ int blk_k1,+ int total_blk, // Total # of K blocks+ int thr_id, int seq_len, int seq_len_kv_cache, int l_coord, float* scaleQ,+ float* scaleK, int full_tile_offset, int discard_seq_coord,+ TensorK_cache2D const& K_cache_2D = TensorK_cache2D{},+ TensorV_cache2D const& V_cache_2D = TensorV_cache2D{}) {+ using namespace sycl::ext::oneapi::this_work_item;++ // Short dimension names:+ // q = sequence len dimension for Q+ // k = sequence len dimension for K+ // d = head size dimension for K/Q+ // v = head size dimension for V+ // VV = MMA tile indices for V+ // Capital letters (Q, K, ...) refer to WG block indices.+ // Primed letters (q', k', ...) refer to atom block indices.++ auto tile_shape_v = make_shape(get<1>(TileShapePV{}) * C<VTiles>{}, get<2>(TileShapePV{}));++ /* Create proxy coordinate tensors for Q/K/P/V */+ Tensor cQ = make_identity_tensor(Q_2D.shape()); // (q,d)+ Tensor cK = make_identity_tensor(K_2D.shape()); // (k,d)+ Tensor cV = make_identity_tensor(V_2D.shape()); // (v,k)+ Tensor cK_cache = make_identity_tensor(K_cache_2D.shape()); // (k,d)+ Tensor cV_cache = make_identity_tensor(V_cache_2D.shape()); // (v,k)+ Tensor cP = make_identity_tensor(take<0, 2>(TileShapeQK{})); // (q,k)++ /* Partition global tensors into workgroup tiles */+ Tensor gQ = local_tile(cQ, TileShapeQK{}, append(blk_qv, _), Step<_1, X, _1>{}); // (q,d,D)+ Tensor gK = local_tile(cK, TileShapeQK{}, make_coord(_, _, _), Step<X, _1, _1>{}); // (k,d,K,D)+ Tensor gV = local_tile(cV, tile_shape_v, make_coord(get<1>(blk_qv), _)); // (v,k,K)+ Tensor gV_split = local_tile(gV, TileShapePV{}, make_coord(_, _, 0), Step<X, _1, _1>{}); // (v,k,VV,K)++ Tensor gK_cache = local_tile(cK_cache, TileShapeQK{}, make_coord(_, _, _), Step<X, _1, _1>{}); // (k,d,K,D)+ Tensor gV_cache = local_tile(cV_cache, tile_shape_v, make_coord(get<1>(blk_qv), _)); // (v,k,K)+ Tensor gV_cache_split = local_tile(gV_cache, TileShapePV{}, make_coord(_, _, 0), Step<X, _1, _1>{}); // (v,k,VV,K)++ /* Create global -> register copies */+ TiledCopyQ copy_q{Q_2D};+ TiledCopyK copy_k{K_2D};+ TiledCopyV copy_v{V_2D};+ TiledCopyK_cache copy_k_cache{K_cache_2D};+ TiledCopyV_cache copy_v_cache{V_cache_2D};++ /* Create MMAs */+ TiledMMAQK mma_qk{};+ TiledMMAPV mma_pv{};++ /* Slice TiledCopy/TiledMMA operations down to to work-item level */+ auto thr_copy_q = copy_q.get_slice(thr_id);+ auto thr_copy_k = copy_k.get_slice(thr_id);+ auto thr_copy_v = copy_v.get_slice(thr_id);+ auto thr_copy_k_cache = copy_k_cache.get_slice(thr_id);+ auto thr_copy_v_cache = copy_v_cache.get_slice(thr_id);+ auto thr_mma_qk = mma_qk.get_slice(thr_id);+ auto thr_mma_pv = mma_pv.get_slice(thr_id);++ /* Partition coordinate tensors for copy */+ auto tQgQ = thr_copy_q.partition_S(gQ); // (atom_val,q',d',D)+ auto tKgK = thr_copy_k.partition_S(gK); // (atom_val,k',d',K,D)+ auto tVgV = thr_copy_v.partition_S(gV_split); // (atom_val,v',k',VV,K)+ auto tKgK_cache = thr_copy_k_cache.partition_S(gK_cache);+ auto tVgV_cache = thr_copy_v_cache.partition_S(gV_cache_split);++ /* Create register fragments for MMA and copies */+ auto tQrQ = thr_copy_q.partition_sg_fragment_D(gQ(_, _, 0));+ auto tSrQ = thr_mma_qk.partition_sg_fragment_A(gQ(_, _, 0));++ auto tKrK = thr_copy_k.partition_sg_fragment_D(gK(_, _, 0, 0));+ auto tSrK = thr_mma_qk.partition_sg_fragment_B(gK(_, _, 0, 0));++ auto tSrS = thr_mma_qk.partition_sg_fragment_C(cP);++ FragP tPrS = make_subgroup_tensor(make_fragment_like<ElementP>(tSrS.tensor()), tSrS.tv_layout());++ auto tArP = thr_mma_pv.partition_sg_fragment_A(cP);++ auto tVrV = thr_copy_v.partition_sg_fragment_D(gV_split(_, _, 0, 0));+ auto tArV = thr_mma_pv.partition_sg_fragment_B(gV_split(_, _, 0, 0));++ /* Create TiledCopy objects for prefetches */+ auto prefetch_q = make_block_2d_prefetch(copy_q);+ auto prefetch_k = make_block_2d_prefetch(copy_k);+ auto prefetch_v = make_block_2d_prefetch(copy_v);+ auto prefetch_k_cache = make_block_2d_prefetch(copy_k_cache);+ auto prefetch_v_cache = make_block_2d_prefetch(copy_v_cache);++ /* Partition global tensors for prefetch */+ auto pQgQ = prefetch_q.get_slice(thr_id).partition_S(gQ);+ auto pKgK = prefetch_k.get_slice(thr_id).partition_S(gK);+ auto pVgV = prefetch_v.get_slice(thr_id).partition_S(gV_split);+ auto pKgK_cache = prefetch_k_cache.get_slice(thr_id).partition_S(gK_cache);+ auto pVgV_cache = prefetch_v_cache.get_slice(thr_id).partition_S(gV_cache_split);++ // ------+ // Kernel+ // ------++ /* Initialization steps for first block: Q/K prefetch, O init */+ /* TODO: limit D prefetch for large head size, and reorder K prefetches */+ int kblocks_cache = ceil_div(seq_len_kv_cache, get<1>(TileShapeQK{}));+ for (int D = 0; D < size<3>(pQgQ); D++) {+ prefetch(prefetch_q, pQgQ(_, _, _, D));+ }+ for (int D = 0; D < size<4>(pKgK); D++) {+ CUTLASS_PRAGMA_UNROLL+ for (int K = 0; K < Stages; K++) {+ if (K < kblocks_cache) {+ if constexpr (PagedKV) {+ int physical_K_tile = get_physical_k_tile(K, l_coord, seq_len_kv_cache);+ prefetch(prefetch_k_cache, pKgK_cache(_, _, _, physical_K_tile, D));+ } else {+ prefetch(prefetch_k_cache, pKgK_cache(_, _, _, K, D));+ }+ } else {+ prefetch(prefetch_k, pKgK(_, _, _, K - kblocks_cache, D));+ }+ }+ }+ if (blk_k0 == 0) {+ clear(tArA);+ fill(tA_max, cutlass::platform::numeric_limits<ElementA>::lowest());+ clear(tA_sum);+ }++ /* Check if */+ bool check_remainder_k = (seq_len % get<1>(TileShapeQK{}) != 0);+ int q_sg_tile = get<0>(shape_div(TileShapeQK{}, shape(SubgroupLayoutQK{})));++ float dq_scale = params.scale_block_size+ ? scaleQ[(get<0>(blk_qv) * get<0>(TileShapeQK{}) + get_sub_group_id() * q_sg_tile) /+ params.scale_block_size] *+ params.scale+ : 1.f;+ /* Main loop body */+ auto mainloop_body = [&](auto cached_k, int K, auto& copy_k_cur, auto& copy_v_cur, auto& prefetch_v_cur,+ auto& tKgK_cur, auto& tVgV_cur, auto& pVgV_cur) {+ /* Split barrier to keep threads together */+ barrier_arrive(ScopeWorkgroup);+ constexpr bool is_cache = decltype(cached_k)::value;++ int k_idx;+ if constexpr (is_cache) {+ k_idx = K;+ if constexpr (PagedKV) {+ k_idx = get_physical_k_tile(K, l_coord, seq_len_kv_cache);+ }+ } else {+ k_idx = K - kblocks_cache;+ }+ int scalek_idx = params.scale_block_size ? K * get<1>(TileShapeQK{}) / params.scale_block_size : 0;+ /* GEMM 1: S = K * Q */+ clear(tSrS);+ CUTLASS_PRAGMA_UNROLL+ for (int D = 0; D < size<4>(tKgK); D++) {+ copy(copy_q, tQgQ(_, _, _, D), tQrQ);+ copy(copy_k_cur, tKgK_cur(_, _, _, k_idx, D), tKrK);+ reorder(tQrQ, tSrQ);+ reorder(tKrK, tSrK);++ cute::gemm(mma_qk, tSrQ, tSrK, tSrS);+ }++ /* V prefetch for GEMM 2 */+ CUTLASS_PRAGMA_UNROLL+ for (int VV = 0; VV < VTiles; VV++) {+ prefetch(prefetch_v_cur, pVgV_cur(_, _, _, VV, k_idx));+ }+ reorder(tSrS, tPrS);+ if constexpr (!CausalMask) {+ if (params.scale_block_size) {+ if (params.scale_block_size == 1) {+ // Need to get global col and row indices to mask the elements+ Tensor cPgP = make_identity_tensor(make_shape(seq_len, seq_len));+ Tensor gP = local_tile(cPgP, take<0, 2>(TileShapeQK{}), make_coord(get<0>(blk_qv), K));+ auto cS_thread = thr_mma_qk.partition_C(gP);+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tPrS.size(); ++i) {+ int row_idx = get<0>(cS_thread(i));+ int col_idx = get<1>(cS_thread(i));+ tPrS(i) *= ElementP(scaleQ[row_idx] * scaleK[col_idx]);+ }+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tPrS.size(); ++i) {+ tPrS(i) *= params.scale;+ }+ } else {+ float _scale = dq_scale * scaleK[scalek_idx];+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tPrS.size(); i++) tPrS(i) *= _scale;+ }+ }+ } else {+ float _scale = dq_scale * scaleK[scalek_idx];+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tPrS.size(); i++) tPrS(i) *= _scale;+ }++ /* Causal masking - only in non-cache mode */+ if constexpr (!is_cache && CausalMask) {+ if (K == total_blk - 1) {+ // Need to get global col and row indices to mask the elements+ Tensor cPgP = make_identity_tensor(make_shape(seq_len, seq_len));+ Tensor gP = local_tile(cPgP, take<0, 2>(TileShapeQK{}), make_coord(get<0>(blk_qv), K));+ auto cS_thread = thr_mma_qk.partition_C(gP);+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tPrS.size(); ++i) {+ int row_idx = get<0>(cS_thread(i));+ int col_idx = get<1>(cS_thread(i));+ if (col_idx - seq_len_kv_cache - full_tile_offset > row_idx - discard_seq_coord) {+ tPrS(i) = ElementP(-INFINITY);+ }+ }+ }+ } else {+ if constexpr (FullMask_) {+ // Need to get global col and row indices to mask the elements+ Tensor cPgP = make_identity_tensor(make_shape(seq_len, seq_len));+ Tensor gP = local_tile(cPgP, take<0, 2>(TileShapeQK{}), make_coord(get<0>(blk_qv), K));+ auto cS_thread = thr_mma_qk.partition_C(gP);+ int row_idx_begin = get<0>(cS_thread(0));+ int row_idx_end = row_idx_begin + q_sg_tile;+ int col_idx_begin = get<1>(cS_thread(0));+ int col_idx_end = col_idx_begin + get<1>(TileShapeQK{});+ if (row_idx_end <= seq_len && col_idx_end <= seq_len) {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tPrS.size(); ++i) {+ int row_idx = get<0>(cS_thread(i));+ int col_idx = get<1>(cS_thread(i));+ tPrS(i) += ElementP(params.mask[col_idx + row_idx * seq_len]);+ }+ } else {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tPrS.size(); ++i) {+ int row_idx = get<0>(cS_thread(i));+ int col_idx = get<1>(cS_thread(i));+ tPrS(i) += (row_idx < seq_len && col_idx < seq_len) ? ElementP(params.mask[col_idx + row_idx * seq_len])+ : ElementP(-INFINITY);+ }+ }+ }+ }+ /* k masking for remainder tiles */+ if constexpr (!is_cache) {+ if (check_remainder_k && K == total_blk - 1) {+ FragPRow k_rem_mask;+ int k_val = get<0>(tKgK_cur(0, 0, 0, k_idx, 0)) + kblocks_cache * get<1>(TileShapeQK{});+ int k = k_val + get_sub_group().get_local_id()[0];+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < k_rem_mask.size(); i++, k += intel::sg_size) {+ k_rem_mask(i) = (k < seq_len) ? ElementP(sycl::nan(0u)) : ElementP(-INFINITY);+ }+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tPrS.size(); i++) {+ tPrS(i) = sycl::fmin(tPrS(i), broadcast<1>(k_rem_mask, tPrS, i));+ }+ }+ }++ /* Apply softmax and scaling (tA rescaling fused into GEMM2 VTile loop) */+ auto rescale = softmax(K == blk_k0, tPrS, tA_max, tA_sum);+ reorder(tPrS, tArP);++ /* GEMM 2: A += P * V, split in v dimension.+ tArA rescaling is fused to per-VTile */+ CUTLASS_PRAGMA_UNROLL+ for (int VV = 0; VV < VTiles; VV++) {+ copy(copy_v_cur, tVgV_cur(_, _, _, VV, k_idx), tVrV);+ reorder(tVrV, tArV);+ if (K != blk_k0) {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tArA.size() / VTiles; i++) tArA(_, _, _, VV)(i) *= broadcast<0>(rescale, tArA, i);+ }++ cute::gemm(mma_pv, tArP, tArV, tArA(_, _, _, VV));+ }++ /* K prefetch */+ int K_next = K + Stages;+ for (int D = 0; D < size<4>(pKgK); D++) {+ if constexpr (is_cache) {+ bool is_cache_next = K_next < kblocks_cache;+ int physical_K_next = K_next;+ if constexpr (PagedKV) {+ if (is_cache_next) {+ physical_K_next = get_physical_k_tile(K_next, l_coord, seq_len_kv_cache);+ }+ }+ if (is_cache_next) {+ prefetch(prefetch_k_cache, pKgK_cache(_, _, _, physical_K_next, D));+ } else {+ prefetch(prefetch_k, pKgK(_, _, _, K_next - kblocks_cache, D));+ }+ } else {+ prefetch(prefetch_k, pKgK(_, _, _, K_next - kblocks_cache, D));+ }+ }++ barrier_wait(ScopeWorkgroup);+ };++ /* Main loop, blocked in k. */+ if constexpr (CachedKV) {+ for (int K = blk_k0; K < kblocks_cache; K++) {+ mainloop_body(std::bool_constant<true>{}, K, copy_k_cache, copy_v_cache, prefetch_v_cache, tKgK_cache,+ tVgV_cache, pVgV_cache);+ }+ }++ for (int K = (blk_k0 > kblocks_cache ? blk_k0 : kblocks_cache); K < blk_k1; K++) {+ mainloop_body(std::bool_constant<false>{}, K, copy_k, copy_v, prefetch_v, tKgK, tVgV, pVgV);+ }+ }++ // Single step of blocked softmax.+ CUTLASS_DEVICE+ FragARow softmax(bool first_block, // First softmax block?+ FragP& tS, // Softmax src/dst block+ FragARow& tS_max, // Softmax row-wise max accumulator+ FragARow& tS_sum) { // Softmax row-wise sum accumulator+ /* Compute row-wise maxima for this block */++ auto tS_bmax = reduce<1>(tS, sycl::maximum{});++ FragARow rescale;+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tS_max.size(); i++) {+ ElementA new_max = sycl::max(tS_max(i), ElementA(tS_bmax(i)));+ rescale(i) = sycl::native::exp2(tS_max(i) - new_max);+ tS_max(i) = new_max;+ }++ /* Scale S and subtract maxima, then exponentiate */+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tS.size(); i++) tS(i) = sycl::native::exp2(tS(i) - broadcast<0>(tS_max, tS, i));++ /* Rescale existing S sums */+ if (!first_block) {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tS_sum.size(); i++) {+ tS_sum(i) *= rescale(i);+ }+ }++ /* Update sums */+ auto tS_bsum = reduce<1>(tS, sycl::plus<void>{});+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tS_sum.size(); i++) tS_sum(i) += tS_bsum(i);++ return rescale;+ }+};+} // namespace cutlass::fmha::collective++/////////////////////////////////////////////////////////////////////////////////////////////////\ No newline at end of filediff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sdpa_fwd_mainloop.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sdpa_fwd_mainloop.hppnew file mode 100644index 000000000..c8826cbff--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/stla/xe_sdpa_fwd_mainloop.hpp@@ -0,0 +1,511 @@+/***************************************************************************************************+ * Copyright (C) 2025 Intel Corporation, All rights reserved.+ * SPDX-License-Identifier: BSD-3-Clause+ *+ * Redistribution and use in source and binary forms, with or without+ * modification, are permitted provided that the following conditions are met:+ *+ * 1. Redistributions of source code must retain the above copyright notice, this+ * list of conditions and the following disclaimer.+ *+ * 2. Redistributions in binary form must reproduce the above copyright notice,+ * this list of conditions and the following disclaimer in the documentation+ * and/or other materials provided with the distribution.+ *+ * 3. Neither the name of the copyright holder nor the names of its+ * contributors may be used to endorse or promote products derived from+ * this software without specific prior written permission.+ *+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+ *+ **************************************************************************************************/++#pragma once++#include "cutlass/cutlass.h"+#include "cutlass/gemm/dispatch_policy.hpp"++#include "cute/algorithm/functional.hpp"+#include "cute/algorithm/gemm.hpp"+#include "cute/algorithm/subgroup_algorithms.hpp"+#include "cute/atom/mma_atom.hpp"++namespace cutlass::sdpa {++template <int Stages>+class XeDefault {}; // Default FMHA mainloop, P in registers.++}; // namespace cutlass::sdpa++namespace cutlass::fmha::collective {++using namespace cute;++/////////////////////////////////////////////////////////////////////////////////////////////////++template <class DispatchPolicy_, bool CausalMask_, bool FullMask_, bool CachedKV_, bool PagedKV_,+ class TiledMMAQK_, // Tiling for Q*K GEMM+ class TiledMMAPV_, // Tiling for P*V GEMM+ int VTiles_, // # of tiles in V dimension+ class TensorQ_, // Global Q/K/V tensors+ class TensorK_, class TensorV_, class TensorK_cache_, class TensorV_cache_,+ class TiledCopyQ_ = void, // Optional TiledCopy for loading Q+ class TiledCopyK_ = void, // Optional TiledCopy for loading K+ class TiledCopyV_ = void, // Optional TiledCopy for loading V+ class TiledCopyK_cache_ = void,+ class TiledCopyV_cache_ = void> // Optional TiledCopy for loading V_cache+struct SDPAFwdMainloop {+ static_assert(cutlass::detail::dependent_false<DispatchPolicy_>, "Could not find a mainloop specialization.");+};++/////////////////////////////////////////////////////////////////////////////////////////////////++template <int Stages, bool FullMask_, bool CausalMask_, bool CachedKV_, bool PagedKV_, class TiledMMAQK_,+ class TiledMMAPV_, int VTiles_, class TensorQ_, class TensorK_, class TensorV_, class TensorK_cache_,+ class TensorV_cache_, class TiledCopyQ_, class TiledCopyK_, class TiledCopyV_, class TiledCopyK_cache_,+ class TiledCopyV_cache_>+struct SDPAFwdMainloop<sdpa::XeDefault<Stages>, CausalMask_, FullMask_, CachedKV_, PagedKV_, TiledMMAQK_, TiledMMAPV_,+ VTiles_, TensorQ_, TensorK_, TensorV_, TensorK_cache_, TensorV_cache_, TiledCopyQ_, TiledCopyK_,+ TiledCopyV_, TiledCopyK_cache_, TiledCopyV_cache_> {+ //+ // Type Aliases+ //+ using TiledMMAQK = TiledMMAQK_;+ using TiledMMAPV = TiledMMAPV_;+ using TileShapeQK = decltype(TiledMMAQK{}.tile_mnk());+ using TileShapePV = decltype(TiledMMAPV{}.tile_mnk());+ static constexpr int VTiles = VTiles_;+ using SubgroupLayoutQK = decltype(TiledMMAQK{}.get_atom_layout_mnk());+ using SGPerWG = decltype(product(take<1, 4>(shape(typename TiledMMAQK::ThrLayoutVMNK{}))));++ using TensorQ = TensorQ_;+ using TensorK = TensorK_;+ using TensorV = TensorV_;++ using TensorQ2D = decltype(TensorQ_{}(append<rank_v<TensorQ_>>(make_coord(_, _), 0)));+ using TensorK2D = decltype(TensorK_{}(append<rank_v<TensorK_>>(make_coord(_, _), 0)));+ using TensorV2D = decltype(TensorV_{}(append<rank_v<TensorV_>>(make_coord(_, _), 0)));++ using TiledCopyQ =+ conditional_t<is_void_v<TiledCopyQ_>, decltype(make_block_2d_copy_A(TiledMMAQK{}, TensorQ2D{})), TiledCopyQ_>;+ using TiledCopyK =+ conditional_t<is_void_v<TiledCopyK_>, decltype(make_block_2d_copy_B(TiledMMAQK{}, TensorK2D{})), TiledCopyK_>;+ using TiledCopyV =+ conditional_t<is_void_v<TiledCopyV_>, decltype(make_block_2d_copy_B(TiledMMAPV{}, TensorV2D{})), TiledCopyV_>;+ using TensorK_cache = TensorK_cache_;+ using TensorV_cache = TensorV_cache_;+ using TensorK_cache2D = decltype(TensorK_cache_{}(append<rank_v<TensorK_cache_>>(make_coord(_, _), 0)));+ using TensorV_cache2D = decltype(TensorV_cache_{}(append<rank_v<TensorV_cache_>>(make_coord(_, _), 0)));+ using TiledCopyK_cache =+ conditional_t<is_void_v<TiledCopyK_cache_>, decltype(make_block_2d_copy_B(TiledMMAQK{}, TensorK_cache2D{})),+ TiledCopyK_cache_>;+ using TiledCopyV_cache =+ conditional_t<is_void_v<TiledCopyV_cache_>, decltype(make_block_2d_copy_B(TiledMMAPV{}, TensorV_cache2D{})),+ TiledCopyV_cache_>;++ // TODO: static_asserts on TiledMMAPV here...++ //+ // Accumulator types+ //+ // FragS: accumulator for Q*K MMA+ // FragO: accumulator for P*V MMAs.+ // Note: v mode may be split into multiple pieces+ // to reduce register pressure.+ // Frag*Row types are reductions of the corresponding Frag* types+ // over rows.+ //+ template <typename TiledMMA>+ using FragC = decltype(TiledMMA{}.get_slice(0).partition_sg_fragment_C(+ make_identity_tensor(select<0, 1>(TiledMMA{}.tile_mnk()))));++ using FragS = FragC<TiledMMAQK>;+ using FragSRow = decltype(reduce<1>(FragS{}, sycl::plus<void>{}));+ using ElementS = typename TiledMMAQK::ValTypeD;+ using ElementM = typename TiledMMAQK::ValTypeA;++ using SingleFragA = FragC<TiledMMAPV>; // (atom val,q',v')+ using FragA = expand_sg_fragment_t<SingleFragA, 1, VTiles>; // (atom val,q',v',VV)+ using FragARow = decltype(reduce<1>(FragA{}, sycl::plus<void>{}));+ using ElementA = typename TiledMMAPV::ValTypeD;++ static constexpr bool CausalMask = CausalMask_;+ static constexpr bool CachedKV = CachedKV_;+ static constexpr bool PagedKV = PagedKV_;++ // User-facing arguments+ struct Arguments {+ ElementS const scale;+ ElementM const* mask = nullptr;+ int const* ptr_page_table = nullptr;+ int page_size = 0;+ int const* num_pages_per_seq = nullptr;+ };++ // Kernel-facing parameters+ using Params = Arguments;++ // SLM data+ struct SharedStorage {};++ Params params;++ //+ // Methods+ //++ SDPAFwdMainloop(Params const& params_, SharedStorage&) : params(params_) {}++ static constexpr Params to_underlying_arguments(Arguments const& args, void* /* workspace */) {+ constexpr double kLog2e = 1.4426950408889634074; // log_2(e)+ ElementS val = args.scale * static_cast<ElementS>(kLog2e);+ return Params{val, args.mask, args.ptr_page_table, args.page_size, args.num_pages_per_seq};+ }++ CUTLASS_HOST_DEVICE static bool can_implement(Arguments const&) { return true; }++ CUTLASS_DEVICE+ int get_physical_k_tile(int K, int l_coord, int seq_len_kv_cache) {+ int next_page_logical_idx = K * get<1>(TileShapeQK{}) / params.page_size;+ // get<1>(TileShapeQK{}) usually smaller than page_size.+ // assuming page_size is multiple of get<1>(TileShapeQK{})+ int tiles_per_page = params.page_size / get<1>(TileShapeQK{});+ int batch_offset =+ params.num_pages_per_seq ? params.num_pages_per_seq[l_coord] : l_coord * (seq_len_kv_cache / params.page_size);++ return params.ptr_page_table[batch_offset + next_page_logical_idx] * tiles_per_page + K % tiles_per_page;+ }++ template <typename QVCoord>+ CUTLASS_DEVICE void operator()(TensorQ2D const& Q_2D, // (q,d)+ TensorK2D const& K_2D, // (k,d)+ TensorV2D const& V_2D, // (d,k)+ FragA& tArA, // Output accumulator (q,v)+ FragARow& tA_max, // Softmax row-wise max accumulator+ FragARow& tA_sum, // Softmax row-wise sum accumulator+ QVCoord blk_qv, // WG tile indices: (Q,V)+ int blk_k0, // K block range: [K0,K1)+ int blk_k1,+ int total_blk, // Total # of K blocks+ int thr_id, int seq_len, int seq_len_kv_cache, int l_coord, int full_tile_offset,+ int discard_seq_coord, TensorK_cache2D const& K_cache_2D = TensorK_cache2D{},+ TensorV_cache2D const& V_cache_2D = TensorV_cache2D{}) {+ using namespace sycl::ext::oneapi::this_work_item;++ // Short dimension names:+ // q = sequence len dimension for Q+ // k = sequence len dimension for K+ // d = head size dimension for K/Q+ // v = head size dimension for V+ // VV = MMA tile indices for V+ // Capital letters (Q, K, ...) refer to WG block indices.+ // Primed letters (q', k', ...) refer to atom block indices.++ auto tile_shape_v = make_shape(get<1>(TileShapePV{}) * C<VTiles>{}, get<2>(TileShapePV{}));++ /* Create proxy coordinate tensors for Q/K/P/V */+ Tensor cQ = make_identity_tensor(Q_2D.shape()); // (q,d)+ Tensor cK = make_identity_tensor(K_2D.shape()); // (k,d)+ Tensor cV = make_identity_tensor(V_2D.shape()); // (v,k)+ Tensor cK_cache = make_identity_tensor(K_cache_2D.shape()); // (k,d)+ Tensor cV_cache = make_identity_tensor(V_cache_2D.shape()); // (v,k)+ Tensor cP = make_identity_tensor(take<0, 2>(TileShapeQK{})); // (q,k)++ /* Partition global tensors into workgroup tiles */+ Tensor gQ = local_tile(cQ, TileShapeQK{}, append(blk_qv, _), Step<_1, X, _1>{}); // (q,d,D)+ Tensor gK = local_tile(cK, TileShapeQK{}, make_coord(_, _, _), Step<X, _1, _1>{}); // (k,d,K,D)+ Tensor gV = local_tile(cV, tile_shape_v, make_coord(get<1>(blk_qv), _)); // (v,k,K)+ Tensor gV_split = local_tile(gV, TileShapePV{}, make_coord(_, _, 0), Step<X, _1, _1>{}); // (v,k,VV,K)++ Tensor gK_cache = local_tile(cK_cache, TileShapeQK{}, make_coord(_, _, _), Step<X, _1, _1>{}); // (k,d,K,D)+ Tensor gV_cache = local_tile(cV_cache, tile_shape_v, make_coord(get<1>(blk_qv), _)); // (v,k,K)+ Tensor gV_cache_split = local_tile(gV_cache, TileShapePV{}, make_coord(_, _, 0), Step<X, _1, _1>{}); // (v,k,VV,K)++ /* Create global -> register copies */+ TiledCopyQ copy_q{Q_2D};+ TiledCopyK copy_k{K_2D};+ TiledCopyV copy_v{V_2D};+ TiledCopyK_cache copy_k_cache{K_cache_2D};+ TiledCopyV_cache copy_v_cache{V_cache_2D};++ /* Create MMAs */+ TiledMMAQK mma_qk{};+ TiledMMAPV mma_pv{};++ /* Slice TiledCopy/TiledMMA operations down to to work-item level */+ auto thr_copy_q = copy_q.get_slice(thr_id);+ auto thr_copy_k = copy_k.get_slice(thr_id);+ auto thr_copy_v = copy_v.get_slice(thr_id);+ auto thr_copy_k_cache = copy_k_cache.get_slice(thr_id);+ auto thr_copy_v_cache = copy_v_cache.get_slice(thr_id);+ auto thr_mma_qk = mma_qk.get_slice(thr_id);+ auto thr_mma_pv = mma_pv.get_slice(thr_id);++ /* Partition coordinate tensors for copy */+ auto tQgQ = thr_copy_q.partition_S(gQ); // (atom_val,q',d',D)+ auto tKgK = thr_copy_k.partition_S(gK); // (atom_val,k',d',K,D)+ auto tVgV = thr_copy_v.partition_S(gV_split); // (atom_val,v',k',VV,K)+ auto tKgK_cache = thr_copy_k_cache.partition_S(gK_cache);+ auto tVgV_cache = thr_copy_v_cache.partition_S(gV_cache_split);++ /* Create register fragments for MMA and copies */+ auto tQrQ = thr_copy_q.partition_sg_fragment_D(gQ(_, _, 0));+ auto tSrQ = thr_mma_qk.partition_sg_fragment_A(gQ(_, _, 0));++ auto tKrK = thr_copy_k.partition_sg_fragment_D(gK(_, _, 0, 0));+ auto tSrK = thr_mma_qk.partition_sg_fragment_B(gK(_, _, 0, 0));++ auto tSrS = thr_mma_qk.partition_sg_fragment_C(cP);+ auto tArP = thr_mma_pv.partition_sg_fragment_A(cP);++ auto tVrV = thr_copy_v.partition_sg_fragment_D(gV_split(_, _, 0, 0));+ auto tArV = thr_mma_pv.partition_sg_fragment_B(gV_split(_, _, 0, 0));++ /* Create TiledCopy objects for prefetches */+ auto prefetch_q = make_block_2d_prefetch(copy_q);+ auto prefetch_k = make_block_2d_prefetch(copy_k);+ auto prefetch_v = make_block_2d_prefetch(copy_v);+ auto prefetch_k_cache = make_block_2d_prefetch(copy_k_cache);+ auto prefetch_v_cache = make_block_2d_prefetch(copy_v_cache);++ /* Partition global tensors for prefetch */+ auto pQgQ = prefetch_q.get_slice(thr_id).partition_S(gQ);+ auto pKgK = prefetch_k.get_slice(thr_id).partition_S(gK);+ auto pVgV = prefetch_v.get_slice(thr_id).partition_S(gV_split);+ auto pKgK_cache = prefetch_k_cache.get_slice(thr_id).partition_S(gK_cache);+ auto pVgV_cache = prefetch_v_cache.get_slice(thr_id).partition_S(gV_cache_split);++ // ------+ // Kernel+ // ------++ /* Initialization steps for first block: Q/K prefetch, O init */+ /* TODO: limit D prefetch for large head size, and reorder K prefetches */+ int kblocks_cache = ceil_div(seq_len_kv_cache, get<1>(TileShapeQK{}));+ for (int D = 0; D < size<3>(pQgQ); D++) {+ prefetch(prefetch_q, pQgQ(_, _, _, D));+ }+ for (int D = 0; D < size<4>(pKgK); D++) {+ CUTLASS_PRAGMA_UNROLL+ for (int K = 0; K < Stages; K++) {+ if (K < kblocks_cache) {+ if constexpr (PagedKV) {+ int physical_K_tile = get_physical_k_tile(K, l_coord, seq_len_kv_cache);+ prefetch(prefetch_k_cache, pKgK_cache(_, _, _, physical_K_tile, D));+ } else {+ prefetch(prefetch_k_cache, pKgK_cache(_, _, _, K, D));+ }+ } else {+ prefetch(prefetch_k, pKgK(_, _, _, K - kblocks_cache, D));+ }+ }+ }+ if (blk_k0 == 0) {+ clear(tArA);+ fill(tA_max, cutlass::platform::numeric_limits<ElementA>::lowest());+ clear(tA_sum);+ }++ /* Check if */+ bool check_remainder_k = (seq_len % get<1>(TileShapeQK{}) != 0);+ int q_sg_tile = get<0>(shape_div(TileShapeQK{}, shape(SubgroupLayoutQK{})));++ /* Main loop body */+ auto mainloop_body = [&](auto cached_k, int K, auto& copy_k_cur, auto& copy_v_cur, auto& prefetch_v_cur,+ auto& tKgK_cur, auto& tVgV_cur, auto& pVgV_cur) {+ /* Split barrier to keep threads together */+ barrier_arrive(ScopeWorkgroup);+ constexpr bool is_cache = decltype(cached_k)::value;++ int k_idx;+ if constexpr (is_cache) {+ k_idx = K;+ if constexpr (PagedKV) {+ k_idx = get_physical_k_tile(K, l_coord, seq_len_kv_cache);+ }+ } else {+ k_idx = K - kblocks_cache;+ }++ /* GEMM 1: S = K * Q */+ clear(tSrS);+ CUTLASS_PRAGMA_UNROLL+ for (int D = 0; D < size<4>(tKgK); D++) {+ copy(copy_q, tQgQ(_, _, _, D), tQrQ);+ copy(copy_k_cur, tKgK_cur(_, _, _, k_idx, D), tKrK);+ reorder(tQrQ, tSrQ);+ reorder(tKrK, tSrK);++ cute::gemm(mma_qk, tSrQ, tSrK, tSrS);+ }++ /* V prefetch for GEMM 2 */+ CUTLASS_PRAGMA_UNROLL+ for (int VV = 0; VV < VTiles; VV++) {+ prefetch(prefetch_v_cur, pVgV_cur(_, _, _, VV, k_idx));+ }+ /* Causal masking - only in non-cache mode */+ if constexpr (!is_cache && CausalMask) {+ if (K == total_blk - 1) {+ // Need to get global col and row indices to mask the elements+ Tensor cPgP = make_identity_tensor(make_shape(seq_len, seq_len));+ Tensor gP = local_tile(cPgP, take<0, 2>(TileShapeQK{}), make_coord(get<0>(blk_qv), K));+ auto cS_thread = thr_mma_qk.partition_C(gP);+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tSrS.size(); ++i) {+ int row_idx = get<0>(cS_thread(i));+ int col_idx = get<1>(cS_thread(i));+ if (col_idx - seq_len_kv_cache - full_tile_offset > row_idx - discard_seq_coord) {+ tSrS(i) = ElementS(-INFINITY);+ }+ }+ }+ } else {+ if constexpr (FullMask_) {+ // Need to get global col and row indices to mask the elements+ Tensor cPgP = make_identity_tensor(make_shape(seq_len, seq_len));+ Tensor gP = local_tile(cPgP, take<0, 2>(TileShapeQK{}), make_coord(get<0>(blk_qv), K));+ auto cS_thread = thr_mma_qk.partition_C(gP);+ int row_idx_begin = get<0>(cS_thread(0));+ int row_idx_end = row_idx_begin + q_sg_tile;+ int col_idx_begin = get<1>(cS_thread(0));+ int col_idx_end = col_idx_begin + get<1>(TileShapeQK{});+ if (row_idx_end <= seq_len && col_idx_end <= seq_len) {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tSrS.size(); ++i) {+ int row_idx = get<0>(cS_thread(i));+ int col_idx = get<1>(cS_thread(i));+ tSrS(i) += ElementS(params.mask[col_idx + row_idx * seq_len]);+ }+ } else {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tSrS.size(); ++i) {+ int row_idx = get<0>(cS_thread(i));+ int col_idx = get<1>(cS_thread(i));+ tSrS(i) += (row_idx < seq_len && col_idx < seq_len) ? ElementS(params.mask[col_idx + row_idx * seq_len])+ : ElementS(-INFINITY);+ }+ }+ }+ }+ /* k masking for remainder tiles */+ if constexpr (!is_cache) {+ if (check_remainder_k && K == total_blk - 1) {+ FragSRow k_rem_mask;+ int k_val = get<0>(tKgK_cur(0, 0, 0, k_idx, 0)) + kblocks_cache * get<1>(TileShapeQK{});+ int k = k_val + get_sub_group().get_local_id()[0];+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < k_rem_mask.size(); i++, k += intel::sg_size) {+ k_rem_mask(i) = (k < seq_len) ? ElementS(sycl::nan(0u)) : ElementS(-INFINITY);+ }+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tSrS.size(); i++) {+ tSrS(i) = sycl::fmin(tSrS(i), broadcast<1>(k_rem_mask, tSrS, i));+ }+ }+ }++ /* Apply softmax and scaling (tA rescaling fused into GEMM2 VTile loop) */+ auto rescale = softmax(K == blk_k0, tSrS, tA_max, tA_sum);+ reorder(tSrS, tArP);++ /* GEMM 2: A += P * V, split in v dimension.+ tArA rescaling is fused to per-VTile */+ CUTLASS_PRAGMA_UNROLL+ for (int VV = 0; VV < VTiles; VV++) {+ copy(copy_v_cur, tVgV_cur(_, _, _, VV, k_idx), tVrV);+ reorder(tVrV, tArV);+ if (K != blk_k0) {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tArA.size() / VTiles; i++) tArA(_, _, _, VV)(i) *= broadcast<0>(rescale, tArA, i);+ }++ cute::gemm(mma_pv, tArP, tArV, tArA(_, _, _, VV));+ }++ /* K prefetch */+ int K_next = K + Stages;+ for (int D = 0; D < size<4>(pKgK); D++) {+ if constexpr (is_cache) {+ bool is_cache_next = K_next < kblocks_cache;+ int physical_K_next = K_next;+ if constexpr (PagedKV) {+ if (is_cache_next) {+ physical_K_next = get_physical_k_tile(K_next, l_coord, seq_len_kv_cache);+ }+ }+ if (is_cache_next) {+ prefetch(prefetch_k_cache, pKgK_cache(_, _, _, physical_K_next, D));+ } else {+ prefetch(prefetch_k, pKgK(_, _, _, K_next - kblocks_cache, D));+ }+ } else {+ prefetch(prefetch_k, pKgK(_, _, _, K_next - kblocks_cache, D));+ }+ }+ barrier_wait(ScopeWorkgroup);+ };++ /* Main loop, blocked in k. */+ if constexpr (CachedKV) {+ for (int K = blk_k0; K < kblocks_cache; K++) {+ mainloop_body(std::bool_constant<true>{}, K, copy_k_cache, copy_v_cache, prefetch_v_cache, tKgK_cache,+ tVgV_cache, pVgV_cache);+ }+ }++ for (int K = (blk_k0 > kblocks_cache ? blk_k0 : kblocks_cache); K < blk_k1; K++) {+ mainloop_body(std::bool_constant<false>{}, K, copy_k, copy_v, prefetch_v, tKgK, tVgV, pVgV);+ }+ }++ // Single step of blocked softmax.+ CUTLASS_DEVICE+ FragSRow softmax(bool first_block, // First softmax block?+ FragS& tS, // Softmax src/dst block+ FragSRow& tS_max, // Softmax row-wise max accumulator+ FragSRow& tS_sum) { // Softmax row-wise sum accumulator+ /* Compute row-wise maxima for this block */+ auto tS_bmax = reduce<1>(tS, sycl::maximum{});++ FragSRow rescale;+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tS_max.size(); i++) {+ ElementS new_max = sycl::max(tS_max(i), params.scale * tS_bmax(i));+ rescale(i) = sycl::native::exp2(tS_max(i) - new_max);+ tS_max(i) = new_max;+ }++ /* Scale S and subtract maxima, then exponentiate */+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tS.size(); i++) tS(i) = sycl::native::exp2(params.scale * tS(i) - broadcast<0>(tS_max, tS, i));++ /* Rescale existing S sums */+ if (!first_block) {+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tS_sum.size(); i++) {+ tS_sum(i) *= rescale(i);+ }+ }++ /* Update sums */+ auto tS_bsum = reduce<1>(tS, sycl::plus<void>{});+ CUTLASS_PRAGMA_UNROLL+ for (int i = 0; i < tS_sum.size(); i++) tS_sum(i) += tS_bsum(i);++ return rescale;+ }+};++} // namespace cutlass::fmha::collective++/////////////////////////////////////////////////////////////////////////////////////////////////diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hppnew file mode 100644index 000000000..37752f79a--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_common.hpp@@ -0,0 +1,69 @@+// SYCL-TLA Flash Attention Common Definitions+// Copyright (C) 2026 Intel Corporation+// SPDX-License-Identifier: MIT++#pragma once++#include <cstddef>+#include <cstdint>+#include <stdexcept>+#include <string>+#include "bestla/bestla.h"+#ifdef ARK_XPU+#include <sycl/sycl.hpp>+#endif++namespace ark {++#if defined(ARK_SYCL_TLA)++/**+ * @brief MOE GEMM Grouped GEMM+ * Implementation in moe_gemm.cpp+ *+ * @param q SYCL queue+ * @param activations Input activations [total_tokens, K]+ * @param weights Expert weights [num_experts, N, K]+ * @param scales Optional scales (nullptr if not used)+ * @param outputs Output [total_tokens, N]+ * @param dtype Data type (FP16/BF16)+ * @param N Output feature dimension+ * @param K Input feature dimension+ * @param num_tokens_per_expert Array of token counts per expert [num_experts]+ * @param num_experts Number of experts+ */+void moe_gemm(sycl::queue* q, void* activations, void* weights, void* scales, void* outputs, BTLA_DTYPE dtype, int N,+ int K, int* num_tokens_per_expert, int num_experts);++// ========================================================================+// Public API+// ========================================================================+/**+ * @brief Flash Attention Prefill (FP16)+ *+ * @param q SYCL queue+ * @param Q_ptr Pointer to Q tensor [B, Hq, Sq, D]+ * @param K_ptr Pointer to K tensor [B, Hkv, Skv, D]+ * @param V_ptr Pointer to V tensor [B, Hkv, Skv, D]+ * @param O_ptr Pointer to output tensor [B, Hq, Sq, D], fp32+ * @param mask Pointer to attention mask tensor [B, 1, Sq, Skv], uint8 (0 for valid, 1 for masked)+ * @param q_dtype Q/K/V data type (FP16)+ * @param batch Batch size+ * @param num_heads_q Number of query heads+ * @param num_heads_kv Number of KV heads+ * @param seq_len_q Query sequence length+ * @param seq_len_kv KV sequence length+ * @param head_dim Head dimension (64 or 128)+ * @param softmax_scale Softmax scale factor+ * @param is_causal Whether to apply causal mask+ */+void sdpa_impl(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, BTLA_DTYPE q_dtype,+ int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv, int head_dim,+ float softmax_scale, bool is_causal);++void sdpa_impl_qks8_pvhalf(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask, int scale_block_size, void* qscale,+ void* kscale, int batch, int num_heads_q, int num_heads_kv, int seq_len_q,+ int seq_len_kv, int head_dim, float softmax_scale, bool is_causal);+#endif // ARK_SYCL_TLA++} // namespace arkdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe.hppnew file mode 100644index 000000000..cff5626de--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_moe.hpp@@ -0,0 +1,153 @@+// SYCL-TLA MOE GEMM Wrapper+// Based on sycl-tla/examples/12_xe20_moe_gemm_cute_interface+//+// Copyright (C) 2026 Intel Corporation+// SPDX-License-Identifier: MIT++#pragma once++#include <cstddef>+#include <cstdint>+#include <stdexcept>+#include <string>++#ifdef ARK_XPU+#include <sycl/sycl.hpp>+#endif++#if defined(ARK_SYCL_TLA)++#include <type_traits>++#include "cutlass/util/device_memory.h"+#include "cutlass/util/packed_stride.hpp"+#include "cutlass/util/GPU_Clock.hpp"+#include "cutlass/util/sycl_event_manager.hpp"++// CUTLASS-SYCL/CuTe compat layer+#include "cute/util/compat.hpp"+#include <sycl/ext/intel/experimental/grf_size_properties.hpp>++#include "cute/tensor.hpp"++#include "cutlass/kernel_hardware_info.h"+#include "cutlass/platform/platform.h"++// MOE kernel headers from sycl-tla example 12+#include "moe_grouped_gemm.hpp"+#include "moe_tile_scheduler.hpp"++#pragma clang diagnostic ignored "-Wpass-failed"+#pragma clang diagnostic ignored "-Wdeprecated-declarations"++#endif // ARK_SYCL_TLA++#include "sycl_tla_common.hpp"++namespace ark {++#if defined(ARK_SYCL_TLA)++namespace moe_detail {++using namespace cute;+using namespace MoE;++// Helper to choose TiledMMA based on element types+template <class TA, class TB>+auto choose_tiled_mma(TA* A, TB* B) {+ using TA_non_CV = cutlass::platform::remove_cv_t<TA>;+ using TB_non_CV = cutlass::platform::remove_cv_t<TB>;+ auto op = XE_DPAS_TT<8, float, TA_non_CV, TB_non_CV>{};++ using WGTile = Shape<_256, _128, _32>; // 256x128 WG tile size+ using SGLayout = Layout<Shape<_8, _2, _1>, Stride<_2, _1, _0>>; // 8x2 SG tiling, n-major++ using MMA = typename TiledMMAHelper<MMA_Atom<decltype(op)>, Layout<WGTile>, SGLayout>::TiledMMA;++ return MMA{};+}++// Unique kernel name tag+template <typename EA, typename EB, typename ED, char layoutA, char layoutB>+class MoEGemmKernel;++// MOE GEMM launcher template+template <char layoutA, char layoutB, typename ElementA, typename ElementB, typename ElementS, typename ElementD>+void moe_gemm_launcher(sycl::queue* q, const ElementA* activations, const ElementB* weights, const ElementS* scales,+ ElementD* outputs, const int gemm_n, const int gemm_k, int* num_rows_per_expert_device,+ const int num_experts) {+ compat::set_default_queue(*q);++ int sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(0);+ cutlass::KernelHardwareInfo hw_info{0, sm_count};++ auto dummy_problem_shape = cute::Shape<int, int, int>{1, gemm_k, gemm_n};+ auto dummy_group_problem_shape =+ cutlass::gemm::GroupProblemShape<Shape<int, int, int>>{1, &dummy_problem_shape, nullptr};++ using TileShape = Shape<_256, _128, _32>;+ using ClusterShape = Shape<_1, _1, _1>;++ auto scheduler_params = PersistentTileSchedulerXeMoE<ProblemShape>::to_underlying_arguments(+ dummy_group_problem_shape, TileShape{}, ClusterShape{}, hw_info,+ PersistentTileSchedulerXeMoE<ProblemShape>::Arguments{1, RasterOrderOptions::AlongN});++ auto group_distribution = PersistentTileSchedulerXeMoE<ProblemShape>::get_grid_shape(+ scheduler_params, dummy_group_problem_shape, TileShape{}, ClusterShape{}, hw_info,+ PersistentTileSchedulerXeMoE<ProblemShape>::Arguments{1, RasterOrderOptions::AlongN});++ auto mma = choose_tiled_mma(activations, weights);+ auto MaxThreadsPerWorkgroup = size(mma);+ dim3 local_range{static_cast<unsigned int>(MaxThreadsPerWorkgroup), 1, 1};++ sycl::range<3> local = {local_range.z, local_range.y, local_range.x};+ sycl::range<3> groups = {group_distribution.z, group_distribution.y, group_distribution.x};+ sycl::range<3> global = {local[0] * groups[0], local[1] * groups[1], local[2] * groups[2]};++ namespace syclex = sycl::ext::oneapi::experimental;+ namespace intelex = sycl::ext::intel::experimental;++ syclex::properties kernel_props{syclex::sub_group_size<16>, intelex::grf_size<256>};++ auto event = q->parallel_for<MoEGemmKernel<ElementA, ElementB, ElementD, layoutA, layoutB>>(+ sycl::nd_range<3>(global, local), kernel_props, [=](auto) {+ MoE::MoEGEMM<XE_LOAD_2D<16, 32, 32, 16>, XE_LOAD_2D_VNNI<16, 32, 16, 16>, XE_STORE_2D<16, 8, 32>, 'R', 'R',+ 'R'>(activations, weights, scales, outputs, mma, num_rows_per_expert_device, num_experts, gemm_n,+ gemm_k, scheduler_params);+ });++ EventManager::getInstance().addEvent(event);+ event.wait();+}++} // namespace moe_detail++// Public MOE GEMM API+inline void moe_gemm(sycl::queue* q, void* activations, void* weights, void* scales, void* outputs, BTLA_DTYPE dtype,+ int N, int K, int* num_tokens_per_expert, int num_experts) {+ switch (dtype) {+ case BTLA_DTYPE::BF16: {+ using Element = cutlass::bfloat16_t;+ moe_detail::moe_gemm_launcher<'R', 'R', Element, Element, Element, Element>(+ q, static_cast<const Element*>(activations), static_cast<const Element*>(weights),+ static_cast<const Element*>(scales), static_cast<Element*>(outputs), N, K, num_tokens_per_expert,+ num_experts);+ break;+ }+ case BTLA_DTYPE::F16: {+ using Element = cutlass::half_t;+ moe_detail::moe_gemm_launcher<'R', 'R', Element, Element, Element, Element>(+ q, static_cast<const Element*>(activations), static_cast<const Element*>(weights),+ static_cast<const Element*>(scales), static_cast<Element*>(outputs), N, K, num_tokens_per_expert,+ num_experts);+ break;+ }+ default:+ throw std::runtime_error("moe_gemm: unsupported dtype, only BF16/FP16 supported");+ }+}++#endif // ARK_SYCL_TLA++} // namespace arkdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hppnew file mode 100644index 000000000..9aaac19f3--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_sdpa.hpp@@ -0,0 +1,946 @@+// SYCL-TLA Flash Attention Wrapper (Prefill + Decode)+// Provides both prefill and decode entry points while accounting for convert_type ODR concerns in sycl-tla+//+// Copyright (C) 2026 Intel Corporation+// SPDX-License-Identifier: MIT++#pragma once++#include <cstddef>+#include <cstdint>+#include <cute/util/compat/memory.hpp>+#include <iostream>+#include <stdexcept>+#include <string>+#include <sycl/aliases.hpp>+#include "bestla/bestla.h"+#ifdef ARK_XPU+#include <sycl/sycl.hpp>+#endif++#if defined(ARK_SYCL_TLA)+#include "cutlass/epilogue/collective/default_epilogue.hpp"+#include "cutlass/gemm/device/gemm_universal_adapter.h"+#include "cutlass/util/packed_stride.hpp"+#include "stla/xe_sdpa_fwd_mainloop.hpp"+#include "stla/xe_sagev1_fwd_mainloop.hpp"+#include "stla/xe_sage_fwd_kernel.hpp"+#include "flash_attention_v2/collective/fmha_fusion.hpp"+#include "flash_attention_v2/collective/xe_fmha_fwd_mainloop.hpp"+#include "flash_attention_v2/kernel/xe_fmha_fwd_kernel.hpp"+#include "flash_attention_v2/kernel/xe_tile_scheduler.hpp"+#include "cutlass/util/GPU_Clock.hpp"+#include "cutlass/util/sycl_event_manager.hpp"+#include <cute/tensor.hpp>+#include <random>++#include "helper.h"+#include "cutlass/util/command_line.h"+#include "cutlass/util/device_memory.h"+#include "cutlass/util/reference/device/gemm_complex.h"+#include "cutlass/util/reference/device/tensor_compare.h"+#include "sycl_common.hpp"++#include <sycl/ext/intel/experimental/grf_size_properties.hpp>++#endif // ARK_SYCL_TLA++namespace ark {++/// Flash Attention data type codes (matches Python side)+enum class FlashAttnDtype : int {+ FP16 = 0,+ BF16 = 1,+ FP32 = 2,+ FP8_E4M3 = 3,+ FP8_E5M2 = 4,+};++#if defined(ARK_SYCL_TLA)++namespace detail {++using namespace cute;++// Command line options parsing+struct Options {+ const void *q = nullptr, *k = nullptr, *v = nullptr;+ void* mask = nullptr;+ void* o = nullptr;+ int scale_block_size = 0;+ const void *qscale = nullptr, *kscale = nullptr;+ const void *block_K = nullptr, *block_V = nullptr;+ const int *page_table = nullptr, *num_pages_per_seq = nullptr;+ bool is_causal = false;+ bool varlen = false;+ bool use_paged_kv = false;+ int batch = 0, num_heads_q = 0, num_heads_kv = 0, seq_len_qo = 0, seq_len_kv = 0, seq_len_kv_cache = 0, page_size = 0,+ head_size_qk = 0, head_size_vo = 0;+ int total_seqlen_q = 0, total_seqlen_kv = 0, total_seqlen_kv_cache = 0;+ int max_seqlen_q = 0, max_seqlen_kv = 0, max_seqlen_kv_cache = 0;+ float softmax_scale = 0.0f;+ bool persistent = false;++ void print(std::ostream& os = std::cout) const {+ os << std::boolalpha << "Options {\n"+ << " q: " << q << "\n"+ << " k: " << k << "\n"+ << " v: " << v << "\n"+ << " o: " << o << "\n"+ << " mask: " << mask << "\n"+ << " block_K: " << block_K << "\n"+ << " block_V: " << block_V << "\n"+ << " page_table: " << page_table << "\n"+ << " num_pages_per_seq: " << num_pages_per_seq << "\n"+ << " is_causal: " << is_causal << "\n"+ << " varlen: " << varlen << "\n"+ << " use_paged_kv: " << use_paged_kv << "\n"+ << " batch: " << batch << "\n"+ << " num_heads_q: " << num_heads_q << "\n"+ << " num_heads_kv: " << num_heads_kv << "\n"+ << " seq_len_qo: " << seq_len_qo << "\n"+ << " seq_len_kv: " << seq_len_kv << "\n"+ << " seq_len_kv_cache: " << seq_len_kv_cache << "\n"+ << " page_size: " << page_size << "\n"+ << " head_size_qk: " << head_size_qk << "\n"+ << " head_size_vo: " << head_size_vo << "\n"+ << " total_seqlen_q: " << total_seqlen_q << "\n"+ << " total_seqlen_kv: " << total_seqlen_kv << "\n"+ << " total_seqlen_kv_cache: " << total_seqlen_kv_cache << "\n"+ << " max_seqlen_q: " << max_seqlen_q << "\n"+ << " max_seqlen_kv: " << max_seqlen_kv << "\n"+ << " max_seqlen_kv_cache: " << max_seqlen_kv_cache << "\n"+ << " softmax_scale: " << softmax_scale << "\n"+ << " persistent: " << persistent << "\n"+ << "}\n";+ }+};++// 3 input matrices: (K)eys, (Q)ueries and (V)alues.+using LayoutQ = cutlass::layout::RowMajor;+using LayoutK = cutlass::layout::ColumnMajor;+using LayoutV = cutlass::layout::RowMajor;+using LayoutO = cutlass::layout::RowMajor;++template <class FMHAKernel, bool isVarLen = false>+struct KernelRunner {+ using StrideQ = typename FMHAKernel::StrideQ;+ using StrideK = typename FMHAKernel::StrideK;+ using StrideV = typename FMHAKernel::StrideV;+ using StrideO = typename FMHAKernel::StrideO;++ using ElementQ = typename FMHAKernel::ElementQ;+ using ElementK = typename FMHAKernel::ElementK;+ using ElementV = typename FMHAKernel::ElementV;+ using ElementO = typename FMHAKernel::ElementO;++ using CollectiveMainloop = typename FMHAKernel::CollectiveMainloop;+ using ElementS = typename CollectiveMainloop::ElementS;++ using ProblemShapeType = cutlass::fmha::kernel::FMHAProblemShape<isVarLen>;++ //+ // Data members+ //++ /// Initialization+ StrideQ stride_Q;+ StrideK stride_K;+ StrideV stride_V;+ StrideK stride_K_cache;+ StrideV stride_V_cache;+ StrideO stride_O;+ uint64_t seed = 0;++ //+ // Methods+ //++ // Note that the GemmUniversalAdapter currently doesn't support flash attention, which is why this+ // secondary `run` function is required to launch the kernel.+ static void run(typename FMHAKernel::Params params) {+ namespace syclex = sycl::ext::oneapi::experimental;+ namespace intelex = sycl::ext::intel::experimental;++ dim3 const block = FMHAKernel::get_block_shape();+ dim3 const grid = FMHAKernel::get_grid_shape(params);++ // configure smem size and carveout+ int smem_size = FMHAKernel::SharedStorageSize;++ const auto sycl_block = compat::dim3(block.x, block.y, block.z);+ const auto sycl_grid = compat::dim3(grid.x, grid.y, grid.z);++ // Launch parameters depend on whether SYCL compiler supports work-group scratch memory extension+ compat::experimental::launch_properties launch_props{+ syclex::work_group_scratch_size(smem_size),+ };+ compat::experimental::kernel_properties kernel_props{syclex::sub_group_size<cute::intel::sg_size>,+ intelex::grf_size<256>};+ compat::experimental::launch_policy policy{sycl_grid, sycl_block, launch_props, kernel_props};+ auto event = compat::experimental::launch<cutlass::device_kernel<FMHAKernel>, FMHAKernel>(policy, params);++ EventManager::getInstance().addEvent(event);+ }++ template <class ProblemShape>+ auto initialize_varlen(const ProblemShape& problem_size, const Options& options) {+ ProblemShape problem_size_for_init = problem_size;+ get<0>(problem_size_for_init) = 1;+ get<3>(problem_size_for_init) = options.total_seqlen_q;+ get<4>(problem_size_for_init) = options.total_seqlen_kv;+ get<5>(problem_size_for_init) = options.total_seqlen_kv_cache;++ ProblemShapeType problem_size_for_launch;+ problem_size_for_launch.batch = get<0>(problem_size);+ problem_size_for_launch.num_heads_q = get<1>(problem_size);+ problem_size_for_launch.num_heads_kv = get<2>(problem_size);+ problem_size_for_launch.seq_len_qo = cutlass::fmha::collective::VariableLength{options.max_seqlen_q};+ problem_size_for_launch.seq_len_kv = cutlass::fmha::collective::VariableLength{options.max_seqlen_kv};+ problem_size_for_launch.seq_len_kv_cache = cutlass::fmha::collective::VariableLength{options.max_seqlen_kv_cache};+ problem_size_for_launch.head_size_qk = get<6>(problem_size);+ problem_size_for_launch.head_size_vo = get<7>(problem_size);++ return cute::make_tuple(problem_size_for_init, problem_size_for_launch);+ }++ ProblemShapeType initialize(const Options& options) {+ auto problem_shape_in =+ cute::make_tuple(options.batch, options.num_heads_q, options.num_heads_kv, options.seq_len_qo,+ options.seq_len_kv, options.seq_len_kv_cache, options.head_size_qk, options.head_size_vo);+ ProblemShapeType shape;++ decltype(problem_shape_in) problem_size;++ if constexpr (isVarLen) {+ auto [problem_shape_init, problem_shape_launch] = initialize_varlen(problem_shape_in, options);+ problem_size = problem_shape_init;+ shape = problem_shape_launch;+ } else {+ problem_size = problem_shape_in;+ shape.batch = options.batch;+ shape.num_heads_q = options.num_heads_q;+ shape.num_heads_kv = options.num_heads_kv;+ shape.seq_len_qo = options.seq_len_qo;+ shape.seq_len_kv = options.seq_len_kv;+ shape.seq_len_kv_cache = options.seq_len_kv_cache;+ shape.head_size_qk = options.head_size_qk;+ shape.head_size_vo = options.head_size_vo;+ }++ auto [batch, num_heads_q, num_heads_kv, seq_len_qo, seq_len_kv, seq_len_kv_cache, head_size_qk, head_size_vo] =+ problem_size;+ auto shape_Q = cute::make_shape(seq_len_qo, head_size_qk, num_heads_q, batch);+ auto shape_K = cute::make_shape(seq_len_kv, head_size_qk, num_heads_kv, batch);+ auto shape_V = cute::make_shape(head_size_vo, seq_len_kv, num_heads_kv, batch);+ auto shape_K_cache = cute::make_shape(seq_len_kv_cache, head_size_qk, num_heads_kv, batch);+ auto shape_V_cache = cute::make_shape(head_size_vo, seq_len_kv_cache, num_heads_kv, batch);+ auto shape_O = cute::make_shape(seq_len_qo, head_size_vo, num_heads_q, batch);++ stride_Q = cutlass::make_cute_packed_stride(StrideQ{}, shape_Q);+ stride_K = cutlass::make_cute_packed_stride(StrideK{}, shape_K);+ stride_V = cutlass::make_cute_packed_stride(StrideV{}, shape_V);+ stride_K_cache = cutlass::make_cute_packed_stride(StrideK{}, shape_K_cache);+ stride_V_cache = cutlass::make_cute_packed_stride(StrideV{}, shape_V_cache);+ stride_O = cutlass::make_cute_packed_stride(StrideO{}, shape_O);++ if constexpr (isVarLen) {+ // shape.seq_len_qo.cumulative_length = device_cumulative_seqlen_q.get();+ // shape.seq_len_kv.cumulative_length = device_cumulative_seqlen_kv.get();+ // shape.seq_len_kv_cache.cumulative_length = device_cumulative_seqlen_kv_cache.get();+ }+ return shape;+ }++ cutlass::Status run(const Options& options, const cutlass::KernelHardwareInfo& hw_info) {+ ProblemShapeType shape = initialize(options);++ typename FMHAKernel::Arguments arguments{+ {+ shape,+ static_cast<const FMHAKernel::ElementQ*>(options.q),+ stride_Q,+ static_cast<const FMHAKernel::ElementK*>(options.k),+ stride_K,+ static_cast<const FMHAKernel::ElementV*>(options.v),+ stride_V,+ static_cast<FMHAKernel::ElementO*>(options.o),+ stride_O,+ static_cast<const FMHAKernel::ElementK*>(options.block_K),+ stride_K_cache,+ static_cast<const FMHAKernel::ElementV*>(options.block_V),+ stride_V_cache,+ },+ {options.softmax_scale, static_cast<FMHAKernel::ElementQ*>(options.mask),+ options.use_paged_kv ? options.page_table : nullptr, options.use_paged_kv ? options.page_size : 0,+ options.use_paged_kv ? options.num_pages_per_seq : nullptr},+ {},+ hw_info};+ // Define device-global scratch memory+ size_t workspace_size = FMHAKernel::get_workspace_size(arguments);+ cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);+ if (!FMHAKernel::can_implement(arguments)) {+ std::cout << "Invalid Problem Size: " << options.batch << 'x' << options.num_heads_q << 'x' << options.seq_len_qo+ << 'x' << options.seq_len_kv << 'x' << options.head_size_qk << 'x' << options.head_size_vo+ << (options.is_causal ? "xCausal" : "xNonCausal") << std::endl;+ return cutlass::Status::kErrorInvalidProblem;+ }++ // Initialize the workspace+ CUTLASS_CHECK(FMHAKernel::initialize_workspace(arguments, workspace.get()));++ // Convert host-side arguments to device-side arguments to be passed to the kernel+ auto params = FMHAKernel::to_underlying_arguments(arguments, workspace.get());++ run(params);++ return cutlass::Status::kSuccess;+ }+};++template <bool Causal, typename TileShapeQK, typename TileShapePV, typename TileShapeOutput, typename SubgroupLayoutQK,+ typename SubgroupLayoutPV_, /* void -> default */+ int PipelineStages, bool persistent, typename ElementQ = bfloat16_t, typename ElementK = bfloat16_t,+ typename ElementV = bfloat16_t, typename ElementO = ElementQ,+ typename MMAOperation_ = void, /* void -> default */+ typename StrideQ = Stride<int, _1, int, int>, typename StrideK = Stride<int, _1, int, int>,+ typename StrideV = Stride<_1, int, int, int>, typename StrideO = Stride<int, _1, int, int>,+ typename GmemTiledCopyQ = void, /* void -> default block 2D */+ typename GmemTiledCopyK = void, typename GmemTiledCopyV = void, typename GmemTiledCopyO = void>+struct FMHAConfig {+ static constexpr int SGTileQ = get<0>(shape_div(TileShapeQK{}, shape(SubgroupLayoutQK{})))();+ using MMAOperation = cute::conditional_t<+ is_void_v<MMAOperation_>,+ typename cute::conditional_t<+ cute::is_same_v<ElementQ, cutlass::float_e5m2_t> || cute::is_same_v<ElementQ, cutlass::float_e4m3_t>,+ XE_DPAS_TT<cute::gcd(SGTileQ, 8), float, half_t>, XE_DPAS_TT<cute::gcd(SGTileQ, 8), float, ElementQ>>,+ MMAOperation_>;+ using SubgroupLayoutPV =+ cute::conditional_t<is_void_v<SubgroupLayoutPV_>,+ decltype(cutlass::fmha::collective::get_sg_layout_pv(SubgroupLayoutQK{})), SubgroupLayoutPV_>;++ template <bool isVarLen, bool CachedKV, bool PagedKV, class Scheduler>+ static int run(const Options& options) {+ //+ // Run examples+ //++ // The KernelHardwareInfo struct holds the number of EUs on the GPU with a given device ID. This+ // information is used by the underlying kernel.+ cutlass::KernelHardwareInfo hw_info;+ hw_info.sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id);++ using ProblemShapeType = cutlass::fmha::kernel::FMHAProblemShape<isVarLen>;++ using TiledMMAQK = typename TiledMMAHelper<MMA_Atom<MMAOperation>, Layout<TileShapeQK>, SubgroupLayoutQK>::TiledMMA;+ using TiledMMAPV = typename TiledMMAHelper<MMA_Atom<MMAOperation>, Layout<TileShapePV>, SubgroupLayoutPV>::TiledMMA;++ static_assert(get<0>(TileShapeOutput{}) == get<0>(TileShapePV{}),+ "Output tile and P*V tile have different sizes in Q dimension");+ constexpr int VTiles = get<1>(TileShapeOutput{}) / get<1>(TileShapePV{});++ auto make_dummy_tensor = [&](auto val, auto stride) {+ return make_tensor(make_gmem_ptr(&val), make_layout(repeat<rank_v<decltype(stride)>>(1), stride));+ };++ using TensorQ = decltype(make_dummy_tensor(ElementQ{}, StrideQ{}));+ using TensorK = decltype(make_dummy_tensor(ElementK{}, StrideK{}));+ using TensorV = decltype(make_dummy_tensor(ElementV{}, StrideV{}));+ using TensorO = decltype(make_dummy_tensor(ElementO{}, StrideO{}));+ using TensorK_cache = TensorK;+ using TensorV_cache = TensorV;+ using GmemTiledCopyK_cache = GmemTiledCopyK;+ using GmemTiledCopyV_cache = GmemTiledCopyV;+ using MainloopDispatchPolicy = cutlass::sdpa::XeDefault<PipelineStages>;+ if constexpr (Causal) {+ // Mainloop+ using CollectiveMainloop =+ cutlass::fmha::collective::SDPAFwdMainloop<MainloopDispatchPolicy, Causal, false, CachedKV, PagedKV,+ TiledMMAQK, TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,+ TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,+ GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;++ // Epilogue+ using CollectiveEpilogue =+ cutlass::fmha::collective::FMHAFwdEpilogue<CollectiveMainloop, TileShapeOutput, TensorO, GmemTiledCopyO>;++ static_assert(!(persistent & Causal), "persistent SDPA kernel not support Causal yet");+ using FMHAKernel = conditional_t<+ is_same_v<Scheduler, cutlass::fmha::kernel::XeFHMAIndividualPersistentTileScheduler>,+ cutlass::fmha::kernel::XeFMHAFwdDynamicSplitKernel<ProblemShapeType, CollectiveMainloop, CollectiveEpilogue,+ Scheduler>,+ cutlass::fmha::kernel::XeFMHAFwdKernel<ProblemShapeType, CollectiveMainloop, CollectiveEpilogue, Scheduler>>;++ KernelRunner<FMHAKernel, isVarLen> runner;++ CUTLASS_CHECK(runner.run(options, hw_info));+ } else {+ if (options.mask) {+ // Mainloop+ using CollectiveMainloop =+ cutlass::fmha::collective::SDPAFwdMainloop<MainloopDispatchPolicy, Causal, true, CachedKV, PagedKV,+ TiledMMAQK, TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,+ TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,+ GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;++ // Epilogue+ using CollectiveEpilogue =+ cutlass::fmha::collective::FMHAFwdEpilogue<CollectiveMainloop, TileShapeOutput, TensorO, GmemTiledCopyO>;++ static_assert(!(persistent & Causal), "persistent SDPA kernel not support Causal yet");+ using FMHAKernel =+ conditional_t<is_same_v<Scheduler, cutlass::fmha::kernel::XeFHMAIndividualPersistentTileScheduler>,+ cutlass::fmha::kernel::XeFMHAFwdDynamicSplitKernel<ProblemShapeType, CollectiveMainloop,+ CollectiveEpilogue, Scheduler>,+ cutlass::fmha::kernel::XeFMHAFwdKernel<ProblemShapeType, CollectiveMainloop,+ CollectiveEpilogue, Scheduler>>;++ KernelRunner<FMHAKernel, isVarLen> runner;++ CUTLASS_CHECK(runner.run(options, hw_info));+ } else {+ // Mainloop+ using CollectiveMainloop =+ cutlass::fmha::collective::SDPAFwdMainloop<MainloopDispatchPolicy, Causal, false, CachedKV, PagedKV,+ TiledMMAQK, TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,+ TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,+ GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;++ // Epilogue+ using CollectiveEpilogue =+ cutlass::fmha::collective::FMHAFwdEpilogue<CollectiveMainloop, TileShapeOutput, TensorO, GmemTiledCopyO>;++ static_assert(!(persistent & Causal), "persistent SDPA kernel not support Causal yet");+ using FMHAKernel =+ conditional_t<is_same_v<Scheduler, cutlass::fmha::kernel::XeFHMAIndividualPersistentTileScheduler>,+ cutlass::fmha::kernel::XeFMHAFwdDynamicSplitKernel<ProblemShapeType, CollectiveMainloop,+ CollectiveEpilogue, Scheduler>,+ cutlass::fmha::kernel::XeFMHAFwdKernel<ProblemShapeType, CollectiveMainloop,+ CollectiveEpilogue, Scheduler>>;++ KernelRunner<FMHAKernel, isVarLen> runner;++ CUTLASS_CHECK(runner.run(options, hw_info));+ }+ }++ return 0;+ }++ static int run(const Options& options) {+ bool cached_kv = options.seq_len_kv_cache > 0;+ if constexpr (persistent) {+ if (options.use_paged_kv || options.seq_len_kv_cache > 0) {+ std::cerr+ << "Error: Persistent kernel does not support paged/cached KV cache (use_paged_kv or seq_len_kv_cache > 0)."+ << std::endl;+ return -1;+ }+ return run<false, false, false, cutlass::fmha::kernel::XeFHMAIndividualPersistentTileScheduler>(options);+ } else if (options.use_paged_kv && !options.varlen) {+ throw std::runtime_error("Paged KV without varlen is not supported yet");+ // return run<false, true, true, cutlass::fmha::kernel::XeFHMAIndividualTileScheduler>(options);+ } else if (!options.use_paged_kv && options.varlen && !cached_kv) {+ throw std::runtime_error("Varlen without cached KV is not supported yet");+ // return run<true, false, false, cutlass::fmha::kernel::XeFHMAIndividualTileScheduler>(options);+ } else if (!options.use_paged_kv && !options.varlen && !cached_kv) {+ return run<false, false, false, cutlass::fmha::kernel::XeFHMAIndividualTileScheduler>(options);+ } else if (!options.use_paged_kv && options.varlen && cached_kv) {+ throw std::runtime_error("Varlen with cached KV but without paged KV is not supported yet");+ // return run<true, true, false, cutlass::fmha::kernel::XeFHMAIndividualTileScheduler>(options);+ } else if (!options.use_paged_kv && !options.varlen && cached_kv) {+ throw std::runtime_error("Cached KV without varlen is not supported yet");+ // return run<false, true, false, cutlass::fmha::kernel::XeFHMAIndividualTileScheduler>(options);+ } else {+ throw std::runtime_error("The combination of options is not supported yet");+ // return run<true, true, true, cutlass::fmha::kernel::XeFHMAIndividualTileScheduler>(options);+ }+ }+};++template <class FMHAKernel, bool isVarLen = false>+struct SageKernelRunner {+ using StrideQ = typename FMHAKernel::StrideQ;+ using StrideK = typename FMHAKernel::StrideK;+ using StrideV = typename FMHAKernel::StrideV;+ using StrideO = typename FMHAKernel::StrideO;++ using ElementQ = typename FMHAKernel::ElementQ;+ using ElementK = typename FMHAKernel::ElementK;+ using ElementV = typename FMHAKernel::ElementV;+ using ElementO = typename FMHAKernel::ElementO;++ using CollectiveMainloop = typename FMHAKernel::CollectiveMainloop;+ using ElementS = typename CollectiveMainloop::ElementS;++ using ProblemShapeType = cutlass::fmha::kernel::SageProblemShape<isVarLen>;++ //+ // Data members+ //++ /// Initialization+ StrideQ stride_Q;+ StrideK stride_K;+ StrideV stride_V;+ StrideK stride_K_cache;+ StrideV stride_V_cache;+ StrideO stride_O;+ uint64_t seed = 0;++ //+ // Methods+ //++ // Note that the GemmUniversalAdapter currently doesn't support flash attention, which is why this+ // secondary `run` function is required to launch the kernel.+ static void run(typename FMHAKernel::Params params) {+ namespace syclex = sycl::ext::oneapi::experimental;+ namespace intelex = sycl::ext::intel::experimental;++ dim3 const block = FMHAKernel::get_block_shape();+ dim3 const grid = FMHAKernel::get_grid_shape(params);++ // configure smem size and carveout+ int smem_size = FMHAKernel::SharedStorageSize;++ const auto sycl_block = compat::dim3(block.x, block.y, block.z);+ const auto sycl_grid = compat::dim3(grid.x, grid.y, grid.z);++ // Launch parameters depend on whether SYCL compiler supports work-group scratch memory extension+ compat::experimental::launch_properties launch_props{+ syclex::work_group_scratch_size(smem_size),+ };+ compat::experimental::kernel_properties kernel_props{syclex::sub_group_size<cute::intel::sg_size>,+ intelex::grf_size<256>};+ compat::experimental::launch_policy policy{sycl_grid, sycl_block, launch_props, kernel_props};+ auto event = compat::experimental::launch<cutlass::device_kernel<FMHAKernel>, FMHAKernel>(policy, params);++ EventManager::getInstance().addEvent(event);+ }++ template <class ProblemShape>+ auto initialize_varlen(const ProblemShape& problem_size, const Options& options) {+ ProblemShape problem_size_for_init = problem_size;+ get<0>(problem_size_for_init) = 1;+ get<3>(problem_size_for_init) = options.total_seqlen_q;+ get<4>(problem_size_for_init) = options.total_seqlen_kv;+ get<5>(problem_size_for_init) = options.total_seqlen_kv_cache;++ ProblemShapeType problem_size_for_launch;+ problem_size_for_launch.batch = get<0>(problem_size);+ problem_size_for_launch.num_heads_q = get<1>(problem_size);+ problem_size_for_launch.num_heads_kv = get<2>(problem_size);+ problem_size_for_launch.seq_len_qo = cutlass::fmha::collective::VariableLength{options.max_seqlen_q};+ problem_size_for_launch.seq_len_kv = cutlass::fmha::collective::VariableLength{options.max_seqlen_kv};+ problem_size_for_launch.seq_len_kv_cache = cutlass::fmha::collective::VariableLength{options.max_seqlen_kv_cache};+ problem_size_for_launch.head_size_qk = get<6>(problem_size);+ problem_size_for_launch.head_size_vo = get<7>(problem_size);++ return cute::make_tuple(problem_size_for_init, problem_size_for_launch);+ }++ ProblemShapeType initialize(const Options& options) {+ auto problem_shape_in =+ cute::make_tuple(options.batch, options.num_heads_q, options.num_heads_kv, options.seq_len_qo,+ options.seq_len_kv, options.seq_len_kv_cache, options.head_size_qk, options.head_size_vo);+ ProblemShapeType shape;++ decltype(problem_shape_in) problem_size;++ if constexpr (isVarLen) {+ auto [problem_shape_init, problem_shape_launch] = initialize_varlen(problem_shape_in, options);+ problem_size = problem_shape_init;+ shape = problem_shape_launch;+ } else {+ problem_size = problem_shape_in;+ shape.batch = options.batch;+ shape.num_heads_q = options.num_heads_q;+ shape.num_heads_kv = options.num_heads_kv;+ shape.seq_len_qo = options.seq_len_qo;+ shape.seq_len_kv = options.seq_len_kv;+ shape.seq_len_kv_cache = options.seq_len_kv_cache;+ shape.head_size_qk = options.head_size_qk;+ shape.head_size_vo = options.head_size_vo;+ }++ auto [batch, num_heads_q, num_heads_kv, seq_len_qo, seq_len_kv, seq_len_kv_cache, head_size_qk, head_size_vo] =+ problem_size;+ auto shape_Q = cute::make_shape(seq_len_qo, head_size_qk, num_heads_q, batch);+ auto shape_K = cute::make_shape(seq_len_kv, head_size_qk, num_heads_kv, batch);+ auto shape_V = cute::make_shape(head_size_vo, seq_len_kv, num_heads_kv, batch);+ auto shape_K_cache = cute::make_shape(seq_len_kv_cache, head_size_qk, num_heads_kv, batch);+ auto shape_V_cache = cute::make_shape(head_size_vo, seq_len_kv_cache, num_heads_kv, batch);+ auto shape_O = cute::make_shape(seq_len_qo, head_size_vo, num_heads_q, batch);++ stride_Q = cutlass::make_cute_packed_stride(StrideQ{}, shape_Q);+ stride_K = cutlass::make_cute_packed_stride(StrideK{}, shape_K);+ stride_V = cutlass::make_cute_packed_stride(StrideV{}, shape_V);+ stride_K_cache = cutlass::make_cute_packed_stride(StrideK{}, shape_K_cache);+ stride_V_cache = cutlass::make_cute_packed_stride(StrideV{}, shape_V_cache);+ stride_O = cutlass::make_cute_packed_stride(StrideO{}, shape_O);++ if constexpr (isVarLen) {+ // shape.seq_len_qo.cumulative_length = device_cumulative_seqlen_q.get();+ // shape.seq_len_kv.cumulative_length = device_cumulative_seqlen_kv.get();+ // shape.seq_len_kv_cache.cumulative_length = device_cumulative_seqlen_kv_cache.get();+ }+ return shape;+ }++ cutlass::Status run(const Options& options, const cutlass::KernelHardwareInfo& hw_info) {+ ProblemShapeType shape = initialize(options);++ typename FMHAKernel::Arguments arguments{+ {+ shape,+ static_cast<const FMHAKernel::ElementQ*>(options.q),+ stride_Q,+ static_cast<const FMHAKernel::ElementK*>(options.k),+ stride_K,+ static_cast<const FMHAKernel::ElementV*>(options.v),+ stride_V,+ static_cast<FMHAKernel::ElementO*>(options.o),+ stride_O,+ static_cast<const FMHAKernel::ElementK*>(options.block_K),+ stride_K_cache,+ static_cast<const FMHAKernel::ElementV*>(options.block_V),+ stride_V_cache,+ },+ {options.softmax_scale, static_cast<float*>(options.mask), options.scale_block_size,+ static_cast<const float*>(options.qscale), static_cast<const float*>(options.kscale),+ options.use_paged_kv ? options.page_table : nullptr, options.use_paged_kv ? options.page_size : 0,+ options.use_paged_kv ? options.num_pages_per_seq : nullptr},+ {},+ hw_info};+ // Define device-global scratch memory+ size_t workspace_size = FMHAKernel::get_workspace_size(arguments);+ cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);+ if (!FMHAKernel::can_implement(arguments)) {+ std::cout << "Invalid Problem Size: " << options.batch << 'x' << options.num_heads_q << 'x' << options.seq_len_qo+ << 'x' << options.seq_len_kv << 'x' << options.head_size_qk << 'x' << options.head_size_vo+ << (options.is_causal ? "xCausal" : "xNonCausal") << std::endl;+ return cutlass::Status::kErrorInvalidProblem;+ }++ // Initialize the workspace+ CUTLASS_CHECK(FMHAKernel::initialize_workspace(arguments, workspace.get()));++ // Convert host-side arguments to device-side arguments to be passed to the kernel+ auto params = FMHAKernel::to_underlying_arguments(arguments, workspace.get());++ run(params);++ return cutlass::Status::kSuccess;+ }+};++template <bool Causal, typename TileShapeQK, typename TileShapePV, typename TileShapeOutput, typename SubgroupLayoutQK,+ typename SubgroupLayoutPV_, /* void -> default */+ int PipelineStages, bool persistent, typename ElementQ = bfloat16_t, typename ElementK = bfloat16_t,+ typename ElementV = bfloat16_t, typename ElementO = ElementQ,+ typename MMAOperation_ = void, /* void -> default */+ typename StrideQ = Stride<int, _1, int, int>, typename StrideK = Stride<int, _1, int, int>,+ typename StrideV = Stride<_1, int, int, int>, typename StrideO = Stride<int, _1, int, int>,+ typename GmemTiledCopyQ = void, /* void -> default block 2D */+ typename GmemTiledCopyK = void, typename GmemTiledCopyV = void, typename GmemTiledCopyO = void>+struct SageConfig {+ static constexpr int SGTileQ = get<0>(shape_div(TileShapeQK{}, shape(SubgroupLayoutQK{})))();+ using MMAOperation =+ cute::conditional_t<is_void_v<MMAOperation_>, XE_DPAS_TT<cute::gcd(SGTileQ, 8), int32_t, int8_t>, MMAOperation_>;+ using MMAOperationPV = cute::conditional_t<is_void_v<MMAOperation_>,+ XE_DPAS_TT<cute::gcd(SGTileQ, 8), float, cute::half_t>, MMAOperation_>;+ using SubgroupLayoutPV =+ cute::conditional_t<is_void_v<SubgroupLayoutPV_>,+ decltype(cutlass::fmha::collective::get_sg_layout_pv(SubgroupLayoutQK{})), SubgroupLayoutPV_>;++ template <bool isVarLen, bool CachedKV, bool PagedKV, class Scheduler>+ static int run(const Options& options) {+ //+ // Run examples+ //++ // The KernelHardwareInfo struct holds the number of EUs on the GPU with a given device ID. This+ // information is used by the underlying kernel.+ cutlass::KernelHardwareInfo hw_info;+ hw_info.sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id);++ using ProblemShapeType = cutlass::fmha::kernel::SageProblemShape<isVarLen>;++ using TiledMMAQK = typename TiledMMAHelper<MMA_Atom<MMAOperation>, Layout<TileShapeQK>, SubgroupLayoutQK>::TiledMMA;+ using TiledMMAPV =+ typename TiledMMAHelper<MMA_Atom<MMAOperationPV>, Layout<TileShapePV>, SubgroupLayoutPV>::TiledMMA;++ static_assert(get<0>(TileShapeOutput{}) == get<0>(TileShapePV{}),+ "Output tile and P*V tile have different sizes in Q dimension");+ constexpr int VTiles = get<1>(TileShapeOutput{}) / get<1>(TileShapePV{});++ auto make_dummy_tensor = [&](auto val, auto stride) {+ return make_tensor(make_gmem_ptr(&val), make_layout(repeat<rank_v<decltype(stride)>>(1), stride));+ };++ using TensorQ = decltype(make_dummy_tensor(ElementQ{}, StrideQ{}));+ using TensorK = decltype(make_dummy_tensor(ElementK{}, StrideK{}));+ using TensorV = decltype(make_dummy_tensor(ElementV{}, StrideV{}));+ using TensorO = decltype(make_dummy_tensor(ElementO{}, StrideO{}));+ using TensorK_cache = TensorK;+ using TensorV_cache = TensorV;+ using GmemTiledCopyK_cache = GmemTiledCopyK;+ using GmemTiledCopyV_cache = GmemTiledCopyV;++ // Mainloop+ using MainloopDispatchPolicy = cutlass::sage::XeDefault<PipelineStages>;+ if constexpr (Causal) {+ using CollectiveMainloop =+ cutlass::fmha::collective::SAGEV1FwdMainloop<MainloopDispatchPolicy, Causal, false, CachedKV, PagedKV,+ TiledMMAQK, TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,+ TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,+ GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;++ // Epilogue+ using CollectiveEpilogue =+ cutlass::fmha::collective::FMHAFwdEpilogue<CollectiveMainloop, TileShapeOutput, TensorO, GmemTiledCopyO>;++ static_assert(!(persistent & Causal), "persistent SDPA kernel not support Causal yet");+ using FMHAKernel =+ cutlass::fmha::kernel::XeSageFwdKernel<ProblemShapeType, CollectiveMainloop, CollectiveEpilogue, Scheduler>;++ SageKernelRunner<FMHAKernel, isVarLen> runner;++ CUTLASS_CHECK(runner.run(options, hw_info));+ } else {+ if (options.mask) {+ using CollectiveMainloop =+ cutlass::fmha::collective::SAGEV1FwdMainloop<MainloopDispatchPolicy, Causal, true, CachedKV, PagedKV,+ TiledMMAQK, TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,+ TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,+ GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;++ // Epilogue+ using CollectiveEpilogue =+ cutlass::fmha::collective::FMHAFwdEpilogue<CollectiveMainloop, TileShapeOutput, TensorO, GmemTiledCopyO>;++ static_assert(!(persistent & Causal), "persistent SDPA kernel not support Causal yet");+ using FMHAKernel =+ cutlass::fmha::kernel::XeSageFwdKernel<ProblemShapeType, CollectiveMainloop, CollectiveEpilogue, Scheduler>;++ SageKernelRunner<FMHAKernel, isVarLen> runner;++ CUTLASS_CHECK(runner.run(options, hw_info));+ } else {+ using CollectiveMainloop =+ cutlass::fmha::collective::SAGEV1FwdMainloop<MainloopDispatchPolicy, Causal, false, CachedKV, PagedKV,+ TiledMMAQK, TiledMMAPV, VTiles, TensorQ, TensorK, TensorV,+ TensorK_cache, TensorV_cache, GmemTiledCopyQ, GmemTiledCopyK,+ GmemTiledCopyV, GmemTiledCopyK_cache, GmemTiledCopyV_cache>;++ // Epilogue+ using CollectiveEpilogue =+ cutlass::fmha::collective::FMHAFwdEpilogue<CollectiveMainloop, TileShapeOutput, TensorO, GmemTiledCopyO>;++ static_assert(!(persistent & Causal), "persistent SDPA kernel not support Causal yet");+ using FMHAKernel =+ cutlass::fmha::kernel::XeSageFwdKernel<ProblemShapeType, CollectiveMainloop, CollectiveEpilogue, Scheduler>;++ SageKernelRunner<FMHAKernel, isVarLen> runner;++ CUTLASS_CHECK(runner.run(options, hw_info));+ }+ }+ return 0;+ }++ static int run(const Options& options) {+ bool cached_kv = options.seq_len_kv_cache > 0;+ if constexpr (persistent) {+ if (options.use_paged_kv || options.seq_len_kv_cache > 0) {+ std::cerr+ << "Error: Persistent kernel does not support paged/cached KV cache (use_paged_kv or seq_len_kv_cache > 0)."+ << std::endl;+ return -1;+ }+ return run<false, false, false, cutlass::fmha::kernel::XeFHMAIndividualPersistentTileScheduler>(options);+ } else if (options.use_paged_kv && !options.varlen) {+ throw std::runtime_error("Paged KV without varlen is not supported yet");+ // return run<false, true, true, cutlass::fmha::kernel::XeFHMAIndividualTileScheduler>(options);+ } else if (!options.use_paged_kv && options.varlen && !cached_kv) {+ throw std::runtime_error("Varlen without cached KV is not supported yet");+ // return run<true, false, false, cutlass::fmha::kernel::XeFHMAIndividualTileScheduler>(options);+ } else if (!options.use_paged_kv && !options.varlen && !cached_kv) {+ return run<false, false, false, cutlass::fmha::kernel::XeFHMAIndividualTileScheduler>(options);+ } else if (!options.use_paged_kv && options.varlen && cached_kv) {+ throw std::runtime_error("Varlen with cached KV but without paged KV is not supported yet");+ // return run<true, true, false, cutlass::fmha::kernel::XeFHMAIndividualTileScheduler>(options);+ } else if (!options.use_paged_kv && !options.varlen && cached_kv) {+ throw std::runtime_error("Cached KV without varlen is not supported yet");+ // return run<false, true, false, cutlass::fmha::kernel::XeFHMAIndividualTileScheduler>(options);+ } else {+ throw std::runtime_error("The combination of options is not supported yet");+ // return run<true, true, true, cutlass::fmha::kernel::XeFHMAIndividualTileScheduler>(options);+ }+ }+};++// ========================================================================+// Prefill Kernel Launch+// ========================================================================++template <typename ElementQ, typename ElementK, typename ElementV, bool persistent = false>+inline int launch_prefill_kernel_128(Options const& options) {+ constexpr int PipelineStages = 2;+ constexpr int PipelineStages1 = 2;+ using ShapeQK = Shape<_256, _32, _32>;+ using ShapePV = Shape<_256, _32, _32>;+ using ShapeOut = Shape<_256, _128>;+ using SubgroupLayoutQK = Layout<Shape<_16, _1, _1>>;+ using ShapeQK1 = Shape<_256, _32, _32>;+ using ShapePV1 = Shape<_256, _32, _32>;+ using ShapeOut1 = Shape<_256, _128>;+ using SubgroupLayoutQK1 = Layout<Shape<_16, _1, _1>>;+ return options.is_causal ? FMHAConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,+ /*persistent=*/false, ElementQ, ElementK, ElementV>::run(options)+ : FMHAConfig<false, ShapeQK1, ShapePV1, ShapeOut1, SubgroupLayoutQK1, void, PipelineStages1,+ /*persistent=*/persistent, ElementQ, ElementK, ElementV>::run(options);+}++template <typename ElementQ, typename ElementK, typename ElementV>+inline int launch_sage_prefill_kernel_128(Options const& options) {+ constexpr int PipelineStages = 1;+ constexpr int PipelineStages1 = 1;+ using ShapeQK = Shape<_256, _64, _32>;+ using ShapePV = Shape<_256, _32, _64>;+ using ShapeOut = Shape<_256, _128>;+ using SubgroupLayoutQK = Layout<Shape<_16, _1, _1>>;+ using ShapeQK1 = Shape<_256, _64, _32>;+ using ShapePV1 = Shape<_256, _32, _64>;+ using ShapeOut1 = Shape<_256, _128>;+ using SubgroupLayoutQK1 = Layout<Shape<_16, _1, _1>>;+ return options.is_causal ? SageConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,+ /*persistent=*/false, ElementQ, ElementK, ElementV, ElementV>::run(options)+ : SageConfig<false, ShapeQK1, ShapePV1, ShapeOut1, SubgroupLayoutQK1, void, PipelineStages1,+ /*persistent=*/false, ElementQ, ElementK, ElementV, ElementV>::run(options);+}++template <typename ElementQ, typename ElementK, typename ElementV>+inline int launch_sage_prefill_kernel_64(Options const& options) {+ constexpr int PipelineStages = 2;+ constexpr int PipelineStages1 = 2;+ using ShapeQK = Shape<_128, _64, _32>;+ using ShapePV = Shape<_128, _32, _64>;+ using ShapeOut = Shape<_128, _64>;+ using SubgroupLayoutQK = Layout<Shape<_8, _1, _1>>;+ using SubgroupLayoutPV = void;+ return options.is_causal+ ? SageConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, SubgroupLayoutPV, PipelineStages,+ /*persistent=*/false, ElementQ, ElementK, ElementV, ElementV>::run(options)+ : SageConfig<false, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, SubgroupLayoutPV, PipelineStages1,+ /*persistent=*/false, ElementQ, ElementK, ElementV, ElementV>::run(options);+}++template <typename ElementQ, typename ElementK, typename ElementV, bool persistent = false>+inline int launch_prefill_kernel_64(Options const& options) {+ constexpr int PipelineStages = 1;+ using ShapeQK = Shape<_256, _64, _32>;+ using ShapePV = Shape<_256, _32, _64>;+ using ShapeOut = Shape<_256, _64>;+ using SubgroupLayoutQK = Layout<Shape<_16, _1, _1>>;+ using ShapeQK1 = Shape<_128, _64, _32>;+ using ShapePV1 = Shape<_128, _32, _64>;+ using ShapeOut1 = Shape<_128, _64>;+ using SubgroupLayoutQK1 = Layout<Shape<_8, _1, _1>>;+ return options.is_causal ? FMHAConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,+ /*persistent=*/false, ElementQ, ElementK, ElementV>::run(options)+ : FMHAConfig<false, ShapeQK1, ShapePV1, ShapeOut1, SubgroupLayoutQK1, void, PipelineStages,+ /*persistent=*/persistent, ElementQ, ElementK, ElementV>::run(options);+}++template <typename ElementQ, typename ElementK, typename ElementV, bool persistent = false>+inline int launch_prefill_kernel_192(Options const& options) {+ constexpr int PipelineStages = 2;+ using ShapeQK = Shape<_256, _64, _32>;+ using ShapePV = Shape<_256, _32, _64>;+ using ShapeOut = Shape<_256, _192>;+ using SubgroupLayoutQK = Layout<Shape<_32, _1, _1>>;+ return options.is_causal ? FMHAConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,+ /*persistent=*/false, ElementQ, ElementK, ElementV>::run(options)+ : FMHAConfig<false, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,+ /*persistent=*/persistent, ElementQ, ElementK, ElementV>::run(options);+}++template <typename ElementQ, typename ElementK, typename ElementV, bool persistent = false>+inline int launch_prefill_kernel_96(Options const& options) {+ constexpr int PipelineStages = 2;+ using ShapeQK = Shape<_128, _64, _32>;+ using ShapePV = Shape<_128, _32, _64>;+ using ShapeOut = Shape<_128, _96>;+ using SubgroupLayoutQK = Layout<Shape<_8, _1, _1>>;+ return options.is_causal ? FMHAConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,+ /*persistent=*/false, ElementQ, ElementK, ElementV>::run(options)+ : FMHAConfig<false, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,+ /*persistent=*/persistent, ElementQ, ElementK, ElementV>::run(options);+}++template <typename ElementQ, typename ElementK, typename ElementV, bool persistent = false>+inline int launch_decode_kernel_128(Options const& options) {+ constexpr int PipelineStages = 1;+ using NUM_SG = std::conditional_t<persistent, _16, _8>;+ using KV_TILE_SIZE = std::conditional_t<persistent, _256, _512>;+ using ShapeQK = Shape<_1, KV_TILE_SIZE, _64>;+ using ShapePV = Shape<_1, _32, KV_TILE_SIZE>;+ using ShapeOut = Shape<_1, _128>;+ using SubgroupLayoutQK = Layout<Shape<_1, NUM_SG, _1>>;+ return options.is_causal ? FMHAConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,+ /*persistent=*/persistent, ElementQ, ElementK, ElementV>::run(options)+ : FMHAConfig<false, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,+ /*persistent=*/persistent, ElementQ, ElementK, ElementV>::run(options);+}++template <typename ElementQ, typename ElementK, typename ElementV, bool persistent = false>+inline int launch_decode_kernel_64(Options const& options) {+ constexpr int PipelineStages = 1;+ using NUM_SG = std::conditional_t<persistent, _16, _8>;+ using KV_TILE_SIZE = std::conditional_t<persistent, _256, _512>;+ using ShapeQK = Shape<_1, KV_TILE_SIZE, _64>;+ using ShapePV = Shape<_1, _32, KV_TILE_SIZE>;+ using ShapeOut = Shape<_1, _64>;+ using SubgroupLayoutQK = Layout<Shape<_1, NUM_SG, _1>>;+ return options.is_causal ? FMHAConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,+ /*persistent=*/persistent, ElementQ, ElementK, ElementV>::run(options)+ : FMHAConfig<false, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,+ /*persistent=*/persistent, ElementQ, ElementK, ElementV>::run(options);+}++template <typename ElementQ, typename ElementK, typename ElementV, bool persistent = false>+inline int launch_decode_kernel_96(Options const& options) {+ constexpr int PipelineStages = 1;+ using NUM_SG = std::conditional_t<persistent, _16, _8>;+ using KV_TILE_SIZE = std::conditional_t<persistent, _256, _512>;+ using ShapeQK = Shape<_1, KV_TILE_SIZE, _64>;+ using ShapePV = Shape<_1, _32, KV_TILE_SIZE>;+ using ShapeOut = Shape<_1, _96>;+ using SubgroupLayoutQK = Layout<Shape<_1, NUM_SG, _1>>;+ return options.is_causal ? FMHAConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,+ /*persistent=*/persistent, ElementQ, ElementK, ElementV>::run(options)+ : FMHAConfig<false, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,+ /*persistent=*/persistent, ElementQ, ElementK, ElementV>::run(options);+}++template <typename ElementQ, typename ElementK, typename ElementV, bool persistent = false>+inline int launch_decode_kernel_192(Options const& options) {+ constexpr int PipelineStages = 1;+ using NUM_SG = std::conditional_t<persistent, _16, _8>;+ using KV_TILE_SIZE = std::conditional_t<persistent, _256, _512>;+ using ShapeQK = Shape<_1, KV_TILE_SIZE, _64>;+ using ShapePV = Shape<_1, _32, KV_TILE_SIZE>;+ using ShapeOut = Shape<_1, _192>;+ using SubgroupLayoutQK = Layout<Shape<_1, NUM_SG, _1>>;+ return options.is_causal ? FMHAConfig<true, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,+ /*persistent=*/persistent, ElementQ, ElementK, ElementV>::run(options)+ : FMHAConfig<false, ShapeQK, ShapePV, ShapeOut, SubgroupLayoutQK, void, PipelineStages,+ /*persistent=*/persistent, ElementQ, ElementK, ElementV>::run(options);+}++} // namespace detail+#endif // ARK_SYCL_TLA++} // namespace arkdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_wrapper.hppnew file mode 100644index 000000000..10b309328--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/sycl_tla_wrapper.hpp@@ -0,0 +1,647 @@+// SYCL-TLA Flash Attention Wrapper (Header-Only)+// This file wraps sycl-tla flash attention kernels for use in ark.cpp+//+// MIT license+// Copyright (C) 2026 Intel Corporation+// SPDX-License-Identifier: MIT++#pragma once++#include <cstddef>+#include <cstdint>+#include <stdexcept>+#include <string>++#ifdef ARK_XPU+#include <sycl/sycl.hpp>+#endif++#if defined(ARK_SYCL_TLA)++#include <type_traits>++#include "cutlass/util/device_memory.h"+#include "cutlass/util/packed_stride.hpp"++// FlashAttention-v2 (Xe) building blocks (from sycl-tla).+#include "cutlass/epilogue/collective/default_epilogue.hpp"+#include "cutlass/epilogue/fusion/xe_callbacks.hpp"+#include "flash_attention_v2/collective/fmha_fusion.hpp"+#include "flash_attention_v2/kernel/tile_scheduler.hpp"++// FlashAttention-v2 Prefill kernels+#include "flash_attention_v2/kernel/xe_flash_attn_prefill.hpp"+#include "flash_attention_v2/collective/xe_flash_attn_prefill_epilogue.hpp"+#include "flash_attention_v2/collective/xe_flash_attn_prefill_softmax_epilogue.hpp"++// FlashAttention-v2 Decode kernels+// NOTE: Decode is temporarily disabled due to convert_type ODR violation in sycl-tla+// Both prefill_mma.hpp and decode_mma.hpp define the same convert_type function+// TODO: Enable when upstream sycl-tla fixes this issue+// #include "flash_attention_v2/kernel/xe_flash_attn_decode.hpp"+// #include "flash_attention_v2/collective/xe_flash_attn_decode_epilogue.hpp"+// #include "flash_attention_v2/collective/xe_flash_attn_decode_softmax_epilogue.hpp"+#define ARK_FLASH_ATTN_DECODE_DISABLED 1++// CUTLASS-SYCL/CuTe compat layer+#include "cute/util/compat.hpp"+#include "cute/tensor.hpp"++// sycl-tla example utilities+#include "helper.h"+#include "sycl_common.hpp"++#endif // ARK_SYCL_TLA++namespace ark {++/// Flash Attention data type codes (matches Python side)+enum class FlashAttnDtype : int {+ FP16 = 0,+ BF16 = 1,+ FP32 = 2,+ FP8_E4M3 = 3,+ FP8_E5M2 = 4,+};++#if defined(ARK_SYCL_TLA)++namespace detail {++// Import cute namespace for convenience in kernel builders+using namespace cute;++// Helper to get subgroup size from kernel+template <typename Kernel, typename = void>+struct KernelSubgroupSize {+ static constexpr int value = Kernel::DispatchPolicy::SubgroupSize;+};++template <typename Kernel>+struct KernelSubgroupSize<Kernel, std::void_t<decltype(Kernel::SubgroupSize)>> {+ static constexpr int value = Kernel::SubgroupSize;+};++// ========================================================================+// Prefill Kernel Launch+// ========================================================================++template <typename KernelT>+inline sycl::event launch_prefill_kernel(typename KernelT::Params const& kernel_params) {+ compat::dim3 const block = KernelT::get_block_shape();+ compat::dim3 const grid = KernelT::get_grid_shape(kernel_params);+ int smem_size = KernelT::SharedStorageSize;++#if !defined(SYCL_EXT_ONEAPI_WORK_GROUP_SCRATCH_MEMORY)+ using namespace compat::experimental;+ return launch<cutlass::device_kernel<KernelT>>(+ launch_policy{grid,+ block,+ local_mem_size{static_cast<std::size_t>(smem_size)},+ kernel_properties{sycl_exp::sub_group_size<KernelSubgroupSize<KernelT>::value>}},+ kernel_params);+#else+ compat::experimental::launch_properties launch_props{+ sycl::ext::oneapi::experimental::work_group_scratch_size(smem_size),+ };+ compat::experimental::kernel_properties kernel_props{+ sycl::ext::oneapi::experimental::sub_group_size<KernelSubgroupSize<KernelT>::value>,+ };+ compat::experimental::launch_policy policy{grid, block, launch_props, kernel_props};+ return compat::experimental::launch<cutlass::device_kernel<KernelT>, KernelT>(policy, kernel_params);+#endif+}++// ========================================================================+// Prefill Kernel Builder Templates+// ========================================================================++// FP16 Prefill Kernel+template <int HeadDim, bool Causal>+struct PrefillKernelBuilderFP16 {+ using ElementInput = cutlass::half_t;+ using ElementOutput = float;+ using ElementAccumulator = float;+ using ElementComputeEpilogue = float;++ using LayoutQ = cutlass::layout::RowMajor;+ using LayoutK = cutlass::layout::ColumnMajor;+ using LayoutV = cutlass::layout::RowMajor;+ using LayoutO = cutlass::layout::RowMajor;++ static constexpr int PipelineStages = 2;++ using MMAOperation = XE_8x16x16_F32F16F16F32_TT;+ using GmemTiledCopyQ = XE_2D_U16x8x16_LD_N;+ using GmemTiledCopyK = XE_2D_U16x16x16_LD_T;+ using GmemTiledCopyV = XE_2D_U16x32x32_LD_V;+ using GmemTiledCopyO = XE_2D_U32x8x16_ST_N;++ // Shape format: [CTA_M, CTA_N, CTA_K] - matches sycl-tla examples+ using ShapeQK = std::conditional_t<HeadDim == 64, Shape<_128, _64, _64>, Shape<_128, _64, _64>>;+ using ShapePV = std::conditional_t<HeadDim == 64, Shape<_128, _32, _64>, Shape<_128, _32, _64>>;+ // ShapeOutput must be rank-3: [CTA_M_QO, CTA_N_VO (HeadDim), CTA_K_PV]+ using ShapeOutput = std::conditional_t<HeadDim == 64, Shape<_128, _64, _64>, Shape<_128, _128, _64>>;+ using SubgroupLayout = std::conditional_t<HeadDim == 64, Layout<Shape<_8, _1, _1>>, Layout<Shape<_16, _1, _1>>>;++ using EpilogueDispatchPolicy = cutlass::epilogue::IntelXeXMX16;+ using GEMMDispatchPolicy = cutlass::gemm::MainloopIntelXeXMX16<PipelineStages>;++ using CollectiveEpilogue = cutlass::flash_attention::collective::FlashPrefillEpilogue<+ EpilogueDispatchPolicy,+ MMAOperation,+ ShapeOutput,+ SubgroupLayout,+ ElementComputeEpilogue,+ ElementOutput,+ cutlass::gemm::TagToStrideC_t<LayoutO>,+ ElementOutput,+ GmemTiledCopyO>;++ using CollectiveSoftmaxEpilogue =+ cutlass::flash_attention::collective::FlashPrefillSoftmaxEpilogue<Causal, EpilogueDispatchPolicy, ElementAccumulator>;++ using ProblemShape = cute::tuple<int, int, int, int, int, int, int>;++ using CollectiveMainloop = cutlass::flash_attention::collective::FlashPrefillMma<+ GEMMDispatchPolicy,+ ProblemShape,+ ElementInput,+ cutlass::gemm::TagToStrideA_t<LayoutQ>,+ ElementInput,+ cutlass::gemm::TagToStrideB_t<LayoutK>,+ ElementInput,+ cutlass::gemm::TagToStrideB_t<LayoutV>,+ MMAOperation,+ ShapeQK,+ ShapePV,+ SubgroupLayout,+ GmemTiledCopyQ,+ GmemTiledCopyK,+ GmemTiledCopyV,+ Causal>;++ using Kernel = cutlass::flash_attention::kernel::FMHAPrefill<+ ProblemShape,+ CollectiveMainloop,+ CollectiveSoftmaxEpilogue,+ CollectiveEpilogue,+ cutlass::flash_attention::IndividualScheduler>;+};++#ifndef ARK_FLASH_ATTN_DECODE_DISABLED+// ========================================================================+// Decode Kernel Builder Templates+// ========================================================================++template <int KVTile, int NumSGs>+struct DecodeTileConfigH64 {+ using ShapeQK = cute::Shape<cute::_1, cute::Int<KVTile>, cute::_64>;+ using ShapePV = cute::Shape<cute::_1, cute::_32, cute::Int<KVTile>>;+ using ShapeOutput = cute::Shape<cute::_1, cute::_64, cute::Int<KVTile>>;+ using SubgroupLayout = cute::Layout<cute::Shape<cute::Int<NumSGs>, cute::_1, cute::_1>,+ cute::Stride<cute::_1, cute::_1, cute::_1>>;+};++template <int KVTile, int NumSGs>+struct DecodeTileConfigH128 {+ using ShapeQK = cute::Shape<cute::_1, cute::Int<KVTile>, cute::_64>;+ using ShapePV = cute::Shape<cute::_1, cute::_32, cute::Int<KVTile>>;+ using ShapeOutput = cute::Shape<cute::_1, cute::_128, cute::Int<KVTile>>;+ using SubgroupLayout = cute::Layout<cute::Shape<cute::Int<NumSGs>, cute::_1, cute::_1>,+ cute::Stride<cute::_1, cute::_1, cute::_1>>;+};++template <int HeadDim>+using DecodeTileConfig = std::conditional_t<HeadDim == 64,+ DecodeTileConfigH64<512, 8>,+ DecodeTileConfigH128<512, 8>>;++// FP16 Decode Kernel+template <int HeadDim, bool Causal>+struct DecodeKernelBuilderFP16 {+ using ElementInput = cutlass::half_t;+ using ElementOutput = float;+ using ElementAccumulator = float;++ using LayoutQ = cutlass::layout::RowMajor;+ using LayoutK = cutlass::layout::ColumnMajor;+ using LayoutV = cutlass::layout::RowMajor;+ using LayoutO = cutlass::layout::RowMajor;++ using GEMMDispatchPolicy = cutlass::gemm::MainloopIntelXeXMX16<2>;+ using EpilogueDispatchPolicy = cutlass::epilogue::IntelXeXMX16;+ using MMAOperation = cute::XE_1x16x16_F32F16F16F32_TT;++ using Tile = DecodeTileConfig<HeadDim>;++ using GmemTiledCopyQ = cute::XE_2D_U16x1x16_LD_N;+ using GmemTiledCopyK = cute::XE_2D_U16x16x16_LD_T;+ using GmemTiledCopyV = cute::XE_2D_U16x32x32_LD_V;+ using GmemTiledCopyO = cute::XE_2D_U32x1x16_ST_N;++ using CollectiveEpilogue = cutlass::flash_attention::collective::FlashDecodeEpilogue<+ EpilogueDispatchPolicy,+ MMAOperation,+ typename Tile::ShapeOutput,+ typename Tile::SubgroupLayout,+ ElementAccumulator,+ ElementOutput,+ cutlass::gemm::TagToStrideC_t<LayoutO>,+ ElementOutput,+ GmemTiledCopyO>;++ using CollectiveSoftmaxEpilogue =+ cutlass::flash_attention::collective::FlashDecodeSoftmaxEpilogue<Causal, EpilogueDispatchPolicy, ElementAccumulator>;++ using ProblemShapeType = cute::tuple<int, int, int, int, int, int, int, int>;++ using CollectiveMainloop = cutlass::flash_attention::collective::FlashDecodeMma<+ GEMMDispatchPolicy,+ ProblemShapeType,+ ElementInput,+ cutlass::gemm::TagToStrideA_t<LayoutQ>,+ ElementInput,+ cutlass::gemm::TagToStrideB_t<LayoutK>,+ ElementInput,+ cutlass::gemm::TagToStrideB_t<LayoutV>,+ MMAOperation,+ typename Tile::ShapeQK,+ typename Tile::ShapePV,+ typename Tile::SubgroupLayout,+ GmemTiledCopyQ,+ GmemTiledCopyK,+ GmemTiledCopyV,+ Causal,+ false>;++ using Kernel = cutlass::flash_attention::kernel::FMHADecode<+ ProblemShapeType,+ CollectiveMainloop,+ CollectiveSoftmaxEpilogue,+ CollectiveEpilogue>;+};++// ========================================================================+// Decode Kernel Launch+// ========================================================================++template <typename KernelT>+inline sycl::event launch_decode_kernel(typename KernelT::Params const& params) {+ compat::dim3 const block = KernelT::get_block_shape();+ compat::dim3 const grid = KernelT::get_grid_shape(params);+ int smem_size = KernelT::SharedStorageSize;++#if !defined(SYCL_EXT_ONEAPI_WORK_GROUP_SCRATCH_MEMORY)+ using namespace compat::experimental;+ return launch<cutlass::device_kernel<KernelT>>(+ launch_policy{grid,+ block,+ local_mem_size{static_cast<std::size_t>(smem_size)},+ kernel_properties{sycl_exp::sub_group_size<KernelSubgroupSize<KernelT>::value>}},+ params);+#else+ compat::experimental::launch_properties launch_props{+ sycl::ext::oneapi::experimental::work_group_scratch_size(smem_size),+ };+ compat::experimental::kernel_properties kernel_props{+ sycl::ext::oneapi::experimental::sub_group_size<KernelSubgroupSize<KernelT>::value>,+ };+ compat::experimental::launch_policy policy{grid, block, launch_props, kernel_props};+ return compat::experimental::launch<cutlass::device_kernel<KernelT>, KernelT>(policy, params);+#endif+}+#endif // ARK_FLASH_ATTN_DECODE_DISABLED++// ========================================================================+// Prefill Implementation+// ========================================================================++template <typename KernelBuilder>+inline void run_prefill_impl(+ sycl::queue* q,+ void* Q_ptr,+ void* K_ptr,+ void* V_ptr,+ void* O_ptr,+ int batch,+ int num_heads_q,+ int num_heads_kv,+ int seq_len_q,+ int seq_len_kv,+ int head_dim,+ float softmax_scale) {+ using namespace cute;+ using Kernel = typename KernelBuilder::Kernel;+ using ElementInput = typename KernelBuilder::ElementInput;+ using ElementOutput = typename KernelBuilder::ElementOutput;+ using ProblemShape = typename Kernel::ProblemShape;++ compat::set_default_queue(*q);++ typename Kernel::StrideQ stride_Q = cutlass::make_cute_packed_stride(+ typename Kernel::StrideQ{}, make_shape(seq_len_q, head_dim, batch * num_heads_q));+ typename Kernel::StrideK stride_K = cutlass::make_cute_packed_stride(+ typename Kernel::StrideK{}, make_shape(seq_len_kv, head_dim, batch * num_heads_kv));+ typename Kernel::StrideV stride_V = cutlass::make_cute_packed_stride(+ typename Kernel::StrideV{}, make_shape(head_dim, seq_len_kv, batch * num_heads_kv));+ typename Kernel::StrideO stride_O = cutlass::make_cute_packed_stride(+ typename Kernel::StrideO{}, make_shape(seq_len_q, head_dim, batch * num_heads_q));++ // Problem shape: (batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, head_dim_v)+ ProblemShape problem_shape = make_tuple(batch, num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, head_dim);++ // Hardware info+ cutlass::KernelHardwareInfo hw_info;+ hw_info.sm_count = 64; // Will be auto-detected if needed++ typename Kernel::Arguments args{+ cutlass::gemm::GemmUniversalMode::kGemm,+ problem_shape,+ {(ElementInput*)Q_ptr, stride_Q,+ (ElementInput*)K_ptr, stride_K,+ (ElementInput*)V_ptr, stride_V},+ {softmax_scale},+ {(ElementOutput*)O_ptr, stride_O},+ hw_info};++ // Get workspace size and allocate+ size_t workspace_size = Kernel::get_workspace_size(args);+ cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);++ // Initialize workspace if needed+ Kernel::initialize_workspace(args, workspace.get());++ typename Kernel::Params params = Kernel::to_underlying_arguments(args, workspace.get());+ launch_prefill_kernel<Kernel>(params).wait();+}++template <int HeadDim, bool Causal>+inline void dispatch_prefill_dtype(+ sycl::queue* q,+ void* Q_ptr,+ void* K_ptr,+ void* V_ptr,+ void* O_ptr,+ FlashAttnDtype dtype,+ int batch,+ int num_heads_q,+ int num_heads_kv,+ int seq_len_q,+ int seq_len_kv,+ float softmax_scale) {+ switch (dtype) {+ case FlashAttnDtype::FP16:+ run_prefill_impl<PrefillKernelBuilderFP16<HeadDim, Causal>>(+ q, Q_ptr, K_ptr, V_ptr, O_ptr, batch, num_heads_q, num_heads_kv,+ seq_len_q, seq_len_kv, HeadDim, softmax_scale);+ break;+ default:+ throw std::runtime_error("Unsupported dtype for flash_attn_prefill (only FP16 supported)");+ }+}++template <int HeadDim>+inline void dispatch_prefill_causal(+ sycl::queue* q,+ void* Q_ptr,+ void* K_ptr,+ void* V_ptr,+ void* O_ptr,+ FlashAttnDtype dtype,+ int batch,+ int num_heads_q,+ int num_heads_kv,+ int seq_len_q,+ int seq_len_kv,+ float softmax_scale,+ bool is_causal) {+ if (is_causal) {+ dispatch_prefill_dtype<HeadDim, true>(+ q, Q_ptr, K_ptr, V_ptr, O_ptr, dtype, batch, num_heads_q, num_heads_kv,+ seq_len_q, seq_len_kv, softmax_scale);+ } else {+ dispatch_prefill_dtype<HeadDim, false>(+ q, Q_ptr, K_ptr, V_ptr, O_ptr, dtype, batch, num_heads_q, num_heads_kv,+ seq_len_q, seq_len_kv, softmax_scale);+ }+}++#ifndef ARK_FLASH_ATTN_DECODE_DISABLED+// ========================================================================+// Decode Implementation+// ========================================================================++template <typename KernelBuilder>+inline void run_decode_impl(+ sycl::queue* q,+ void* Q_ptr,+ void* K_new_ptr,+ void* V_new_ptr,+ void* K_cache_ptr,+ void* V_cache_ptr,+ void* O_ptr,+ int batch,+ int num_heads_q,+ int num_heads_kv,+ int seq_len_kv_cache,+ int head_dim,+ float softmax_scale) {+ using namespace cute;+ using Kernel = typename KernelBuilder::Kernel;+ using ElementInput = typename KernelBuilder::ElementInput;++ compat::set_default_queue(*q);++ const int seq_qo = 1;+ const int seq_kv_new = 1;++ typename Kernel::StrideQ stride_Q = cutlass::make_cute_packed_stride(+ typename Kernel::StrideQ{}, make_shape(seq_qo, head_dim, batch * num_heads_q));+ typename Kernel::StrideK stride_K = cutlass::make_cute_packed_stride(+ typename Kernel::StrideK{}, make_shape(seq_kv_new, head_dim, batch * num_heads_kv));+ typename Kernel::StrideV stride_V = cutlass::make_cute_packed_stride(+ typename Kernel::StrideV{}, make_shape(head_dim, seq_kv_new, batch * num_heads_kv));+ typename Kernel::StrideK stride_K_cache = cutlass::make_cute_packed_stride(+ typename Kernel::StrideK{}, make_shape(seq_len_kv_cache, head_dim, batch * num_heads_kv));+ typename Kernel::StrideV stride_V_cache = cutlass::make_cute_packed_stride(+ typename Kernel::StrideV{}, make_shape(head_dim, seq_len_kv_cache, batch * num_heads_kv));+ typename Kernel::StrideO stride_O = cutlass::make_cute_packed_stride(+ typename Kernel::StrideO{}, make_shape(seq_qo, head_dim, batch * num_heads_q));++ typename Kernel::Arguments args{+ make_tuple(batch, num_heads_q, num_heads_kv, seq_qo, seq_kv_new, head_dim, seq_len_kv_cache, /*page_size=*/0),+ {(ElementInput*)Q_ptr, stride_Q,+ (ElementInput*)K_new_ptr, stride_K,+ (ElementInput*)V_new_ptr, stride_V,+ (ElementInput*)K_cache_ptr, stride_K_cache,+ (ElementInput*)V_cache_ptr, stride_V_cache,+ softmax_scale,+ nullptr, // page_table+ 0}, // page_table_batch+ {},+ {(float*)O_ptr, stride_O}};++ typename Kernel::Params params = Kernel::to_underlying_arguments(args);+ launch_decode_kernel<Kernel>(params).wait();+}++template <int HeadDim, bool Causal>+inline void dispatch_decode_dtype(+ sycl::queue* q,+ void* Q_ptr,+ void* K_new_ptr,+ void* V_new_ptr,+ void* K_cache_ptr,+ void* V_cache_ptr,+ void* O_ptr,+ FlashAttnDtype dtype,+ int batch,+ int num_heads_q,+ int num_heads_kv,+ int seq_len_kv_cache,+ float softmax_scale) {+ switch (dtype) {+ case FlashAttnDtype::FP16:+ run_decode_impl<DecodeKernelBuilderFP16<HeadDim, Causal>>(+ q, Q_ptr, K_new_ptr, V_new_ptr, K_cache_ptr, V_cache_ptr, O_ptr,+ batch, num_heads_q, num_heads_kv, seq_len_kv_cache, HeadDim, softmax_scale);+ break;+ default:+ throw std::runtime_error("Unsupported dtype for flash_attn_decode (only FP16 supported)");+ }+}++template <int HeadDim>+inline void dispatch_decode_causal(+ sycl::queue* q,+ void* Q_ptr,+ void* K_new_ptr,+ void* V_new_ptr,+ void* K_cache_ptr,+ void* V_cache_ptr,+ void* O_ptr,+ FlashAttnDtype dtype,+ int batch,+ int num_heads_q,+ int num_heads_kv,+ int seq_len_kv_cache,+ float softmax_scale,+ bool is_causal) {+ if (is_causal) {+ dispatch_decode_dtype<HeadDim, true>(+ q, Q_ptr, K_new_ptr, V_new_ptr, K_cache_ptr, V_cache_ptr, O_ptr,+ dtype, batch, num_heads_q, num_heads_kv, seq_len_kv_cache, softmax_scale);+ } else {+ dispatch_decode_dtype<HeadDim, false>(+ q, Q_ptr, K_new_ptr, V_new_ptr, K_cache_ptr, V_cache_ptr, O_ptr,+ dtype, batch, num_heads_q, num_heads_kv, seq_len_kv_cache, softmax_scale);+ }+}+#endif // ARK_FLASH_ATTN_DECODE_DISABLED++} // namespace detail++// ========================================================================+// Public API+// ========================================================================++/**+ * @brief Flash Attention Prefill (FP16)+ *+ * @param q SYCL queue+ * @param Q_ptr Pointer to Q tensor [B, Hq, Sq, D]+ * @param K_ptr Pointer to K tensor [B, Hkv, Skv, D]+ * @param V_ptr Pointer to V tensor [B, Hkv, Skv, D]+ * @param O_ptr Pointer to output tensor [B, Hq, Sq, D], fp32+ * @param q_dtype Q/K/V data type (FP16)+ * @param batch Batch size+ * @param num_heads_q Number of query heads+ * @param num_heads_kv Number of KV heads+ * @param seq_len_q Query sequence length+ * @param seq_len_kv KV sequence length+ * @param head_dim Head dimension (64 or 128)+ * @param softmax_scale Softmax scale factor+ * @param is_causal Whether to apply causal mask+ */+inline void flash_attn_prefill(+ sycl::queue* q,+ void* Q_ptr,+ void* K_ptr,+ void* V_ptr,+ void* O_ptr,+ FlashAttnDtype q_dtype,+ int batch,+ int num_heads_q,+ int num_heads_kv,+ int seq_len_q,+ int seq_len_kv,+ int head_dim,+ float softmax_scale,+ bool is_causal) {+ if (head_dim == 64) {+ detail::dispatch_prefill_causal<64>(+ q, Q_ptr, K_ptr, V_ptr, O_ptr, q_dtype, batch, num_heads_q, num_heads_kv,+ seq_len_q, seq_len_kv, softmax_scale, is_causal);+ } else if (head_dim == 128) {+ detail::dispatch_prefill_causal<128>(+ q, Q_ptr, K_ptr, V_ptr, O_ptr, q_dtype, batch, num_heads_q, num_heads_kv,+ seq_len_q, seq_len_kv, softmax_scale, is_causal);+ } else {+ throw std::runtime_error("flash_attn_prefill: unsupported head_dim (must be 64 or 128)");+ }+}++#ifndef ARK_FLASH_ATTN_DECODE_DISABLED+/**+ * @brief Flash Attention Decode (FP16)+ *+ * @param q SYCL queue+ * @param Q_ptr Pointer to Q tensor [B, Hq, 1, D]+ * @param K_new_ptr Pointer to new K tensor [B, Hkv, 1, D]+ * @param V_new_ptr Pointer to new V tensor [B, Hkv, 1, D]+ * @param K_cache_ptr Pointer to K cache [B, Hkv, Scache, D] (nullptr if no cache)+ * @param V_cache_ptr Pointer to V cache [B, Hkv, Scache, D] (nullptr if no cache)+ * @param O_ptr Pointer to output tensor [B, Hq, 1, D], fp32+ * @param q_dtype Q/K/V data type (FP16)+ * @param batch Batch size+ * @param num_heads_q Number of query heads+ * @param num_heads_kv Number of KV heads+ * @param seq_len_kv_cache KV cache sequence length (0 if no cache)+ * @param head_dim Head dimension (64 or 128)+ * @param softmax_scale Softmax scale factor+ * @param is_causal Whether to apply causal mask+ */+inline void flash_attn_decode(+ sycl::queue* q,+ void* Q_ptr,+ void* K_new_ptr,+ void* V_new_ptr,+ void* K_cache_ptr,+ void* V_cache_ptr,+ void* O_ptr,+ FlashAttnDtype q_dtype,+ int batch,+ int num_heads_q,+ int num_heads_kv,+ int seq_len_kv_cache,+ int head_dim,+ float softmax_scale,+ bool is_causal) {+ if (head_dim == 64) {+ detail::dispatch_decode_causal<64>(+ q, Q_ptr, K_new_ptr, V_new_ptr, K_cache_ptr, V_cache_ptr, O_ptr,+ q_dtype, batch, num_heads_q, num_heads_kv, seq_len_kv_cache, softmax_scale, is_causal);+ } else if (head_dim == 128) {+ detail::dispatch_decode_causal<128>(+ q, Q_ptr, K_new_ptr, V_new_ptr, K_cache_ptr, V_cache_ptr, O_ptr,+ q_dtype, batch, num_heads_q, num_heads_kv, seq_len_kv_cache, softmax_scale, is_causal);+ } else {+ throw std::runtime_error("flash_attn_decode: unsupported head_dim (must be 64 or 128)");+ }+}+#endif // ARK_FLASH_ATTN_DECODE_DISABLED++#endif // ARK_SYCL_TLA++} // namespace arkdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hppnew file mode 100644index 000000000..cc60e5d7a--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/utils.hpp@@ -0,0 +1,217 @@+//+// MIT license+// Copyright (C) 2024 Intel Corporation+// SPDX-License-Identifier: MIT+//++//+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.+// See https://llvm.org/LICENSE.txt for license information.+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception+//++#pragma once+#include <cstdlib>+#include <array>+#include <unordered_map>+#include <dnnl.hpp>+#include "bestla/bestla_wrapper.h"+#if ARK_XPU+#include "bestla/sycl/sycl_wrapper.h"+#include <dnnl_sycl.hpp>+#else+namespace sycl {+typedef void queue;+}+#endif++#define LOG_LINE() printf("%s:L%d\n", __FUNCTION__, __LINE__);++namespace ark {++struct env_params {+ int verbose = 2; // trace 0/ debug 1/ info 2/ warn 3/ error 4/ critical 5/ off 6+ int auto_s8 = 0;+ static env_params* Instance() {+ static env_params instance;+ return &instance;+ }+ env_params() {+ env_i("ARK_VERBOSE", verbose);+ env_i("ARK_AUTO_S8", auto_s8);+ }+ static inline void env_i(const char* envstr, int& default_) {+ const char* log_level_env = getenv(envstr);+ if (log_level_env != nullptr) default_ = std::stoi(log_level_env);+ }+};++using UUIDArray = std::array<unsigned char, 16>;+// 高性能 UUID 哈希器+struct UUIDHasher {+ size_t operator()(const UUIDArray& uuid) const {+ // 将 16 字节视为两个 64 位整数进行处理,性能远高于逐字节遍历+ const uint64_t* p = reinterpret_cast<const uint64_t*>(uuid.data());+ uint64_t h1 = p[0];+ uint64_t h2 = p[1];++ // 使用类似 splitmix64 的混合逻辑+ h1 ^= h1 >> 33;+ h1 *= 0xff51afd7ed558ccdLLU;+ h2 ^= h2 >> 33;+ h2 *= 0xc4ceb9fe1a85ec53LLU;++ return static_cast<size_t>(h1 ^ h2);+ }+};++template <typename T>+static inline constexpr dnnl::memory::data_type to_dt() {+ if constexpr (std::is_same_v<T, float>)+ return dnnl::memory::data_type::f32;+ else if constexpr (std::is_same_v<T, bestla::utils::fp16>)+ return dnnl::memory::data_type::f16;+ else if constexpr (std::is_same_v<T, int8_t>)+ return dnnl::memory::data_type::s8;+ else if constexpr (std::is_same_v<T, uint8_t>)+ return dnnl::memory::data_type::u8;+ else if constexpr (std::is_same_v<T, bestla::utils::bf16>)+ return dnnl::memory::data_type::bf16;+ else+ static_assert(0);+}++static inline constexpr dnnl::memory::data_type to_dt(BTLA_DTYPE bt) {+ switch (bt) {+ case BTLA_DTYPE::F32:+ return dnnl::memory::data_type::f32;+ case BTLA_DTYPE::F16:+ return dnnl::memory::data_type::f16;+ case BTLA_DTYPE::S8:+ return dnnl::memory::data_type::s8;+ case BTLA_DTYPE::U8:+ return dnnl::memory::data_type::u8;+ case BTLA_DTYPE::BF16:+ return dnnl::memory::data_type::bf16;+ default:+ return dnnl::memory::data_type::f32;+ }+}++class DnnlContext {+ public:+ static DnnlContext* Instance() {+ static DnnlContext instance;+ return &instance;+ }++ dnnl::engine* get_eng(sycl::queue* q) {+ auto key = check_dnnl_device(q);+ return &dev_engine_map[key];+ }++ dnnl::stream* get_stream(sycl::queue* q) {+ auto key = check_dnnl_device(q);+ return &dev_stream_map[key];+ }++ size_t check_dnnl_device(sycl::queue* q) {+ size_t key = 0;+ if (q == nullptr) {+ if (dev_engine_map.find(key) == dev_engine_map.end()) {+ dev_engine_map[key] = dnnl::engine(dnnl::engine::kind::cpu, 0);+ dev_stream_map[key] = dnnl::stream(dev_engine_map[key]);+ }+ return key;+ }+#if ARK_XPU+ auto uuid = q->get_device().get_info<sycl::ext::intel::info::device::uuid>();+ key = UUIDHasher{}(uuid);+ if (dev_engine_map.find(key) == dev_engine_map.end()) {+ sycl::device dev = q->get_device();+ // Get the context associated with the queue+ sycl::context ctx = q->get_context();+ dev_engine_map[key] = dnnl::sycl_interop::make_engine(dev, ctx);+ dev_stream_map[key] = dnnl::sycl_interop::make_stream(dev_engine_map[key], *q);+ }+#endif+ return key;+ }++ dnnl::memory get_scratch_mem(dnnl::memory::desc _md, sycl::queue* q) {+ auto key = check_dnnl_device(q);+ auto ptr = get_scratch_ptr(_md.get_size(), 0, q, key);+ return dnnl::memory(_md, dev_engine_map[key], ptr);+ }++ void* get_scratch_mem(size_t _size, size_t buf_loc, sycl::queue* q) {+ auto key = check_dnnl_device(q);+ auto ptr = get_scratch_ptr(_size, buf_loc, q, key);+ return ptr;+ }++ void* get_scratch_ptr(size_t _size, size_t buf_loc, sycl::queue* q, size_t key) {+ if (_size <= 0 || buf_loc >= MaxLocNum) return nullptr;++ auto it = dev_mem_ptr_map[buf_loc].find(key);+ if (it == dev_mem_ptr_map[buf_loc].end()) {+ dev_mem_size_map[buf_loc][key] = _size;+#if ARK_XPU+ auto newptr = sycl::aligned_alloc_device<int8_t>(128, _size, *q);+#else+ auto newptr = (int8_t*)malloc(_size);+#endif+ dev_mem_ptr_map[buf_loc][key] = newptr;+ return newptr;+ } else {+ auto pre_size = dev_mem_size_map[buf_loc][key];+ if (pre_size < _size) {+ auto pre_ptr = it->second;+#if ARK_XPU+ sycl::free(pre_ptr, *q);+#else+ free(pre_ptr);+#endif+ dev_mem_size_map[buf_loc][key] = _size;+#if ARK_XPU+ auto newptr = sycl::aligned_alloc_device<int8_t>(128, _size, *q);+#else+ auto newptr = (int8_t*)malloc(_size);+#endif+ dev_mem_ptr_map[buf_loc][key] = newptr;+ return newptr;+ }+ }+ return it->second;+ }+ typedef std::unordered_map<size_t, size_t> msize_t;+ typedef std::unordered_map<size_t, int8_t*> mptr_t;+ static constexpr int MaxLocNum = 8;+ std::array<msize_t, MaxLocNum> dev_mem_size_map;+ std::array<mptr_t, MaxLocNum> dev_mem_ptr_map;+ std::unordered_map<size_t, dnnl::engine> dev_engine_map;+ std::unordered_map<size_t, dnnl::stream> dev_stream_map;+};++struct QuantParam {+ int n;+ int k;+ int blocksize;+ BTLA_DTYPE compute_type;+ BTLA_DTYPE weight_type;+ BTLA_DTYPE scale_type;+ bool asym;++ QuantParam(int _n, int _k, int _blocksize, int _ct, int _wt, int _st, bool _asym)+ : n(_n),+ k(_k),+ blocksize(_blocksize),+ compute_type((BTLA_DTYPE)_ct),+ weight_type((BTLA_DTYPE)_wt),+ scale_type((BTLA_DTYPE)_st),+ asym(_asym) {}++ inline int blks() { return k / blocksize; }+};++} // namespace ark\ No newline at end of filediff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hppnew file mode 100644index 000000000..682a9085b--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_wrapper.hpp@@ -0,0 +1,837 @@+//+// MIT license+// Copyright (C) 2024 Intel Corporation+// SPDX-License-Identifier: MIT+//++//+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.+// See https://llvm.org/LICENSE.txt for license information.+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception+//++#pragma once+#include <cmath>+#include <cstdio>+#include "utils.hpp"+#include "dnnl_wrapper.hpp"+#include "sycl_tla_common.hpp"++namespace ark {++class XpuWrapper {+ public:+ static inline size_t get_weight_qbits(QuantParam* p) { return bestla::utils::bestla_dtype_bits(p->weight_type); }++ static inline size_t get_weight_qbytes(QuantParam* p) { return get_weight_qbits(p) * p->k * p->n / 8; }++ static inline float get_weight_fullrange(QuantParam* p) { return float(1u << (get_weight_qbits(p) - 1)); }++ static inline size_t get_packw_qsize(QuantParam* p) {+ size_t size = 0;+ if (p->weight_type == BTLA_DTYPE::S2 || p->weight_type == BTLA_DTYPE::S4) {+ size += get_weight_qbytes(p);+ }+ if (p->weight_type == BTLA_DTYPE::S8) {+ size += p->k * p->n;+ }+ if (p->weight_type == BTLA_DTYPE::F8_E4M3 || p->weight_type == BTLA_DTYPE::F8_E5M2) {+ size += p->k * p->n; // FP8 is stored as uint8_t, same size as int8+ }+ return size;+ }++ static inline size_t get_scale_size_bytes(QuantParam* p) { return bestla::utils::bestla_dtype_bytes(p->scale_type); }++ static inline size_t get_scale_size(QuantParam* p) {+ using namespace bestla::utils;+ return p->blks() * p->n * bestla_dtype_bytes(p->scale_type);+ }++ static inline size_t get_zp_size(QuantParam* p) {+ using namespace bestla::utils;+ if (!p->asym) return 0;+ return p->blks() * p->n;+ }++ static inline size_t get_scalext_size(QuantParam* p) {+ using namespace bestla::utils;+ if (env_params::Instance()->auto_s8 == 0) return 0;+ size_t nblk = 1;+ if (env_params::Instance()->auto_s8 != -1) {+ nblk *= p->k / env_params::Instance()->auto_s8;+ }+ return nblk * p->n * bestla_dtype_bytes(p->scale_type);+ }++ static inline size_t get_packw_size(QuantParam* p) {+ size_t size = get_packw_qsize(p);+ size += get_scale_size(p);+ size += get_zp_size(p);+ size += get_scalext_size(p);+ return size;+ }++ static inline size_t get_scale_offset(QuantParam* p) {+ size_t size = get_packw_qsize(p);+ return size;+ }++ static inline size_t get_zp_offset(QuantParam* p) {+ size_t size = get_scale_offset(p);+ size += get_scale_size(p);+ return size;+ }++ static inline size_t get_scalext_offset(QuantParam* p) {+ size_t size = get_zp_offset(p);+ size += get_zp_size(p);+ return size;+ }++ static bool can_comps8(QuantParam* p) {+ if (p->asym && p->weight_type == BTLA_DTYPE::S8) return false;+ if (p->blocksize == -1 || p->blocksize == p->k) return true;+ if (rescale(p)) return true;+ if (p->blocksize % 64 == 0) return true;+ return false;+ }++ static inline void check_compute_type(QuantParam* p) {+ using namespace bestla;+ switch (p->compute_type) {+ case BTLA_DTYPE::EleBitsUndef: // from low precision to high precision+ [[fallthrough]];+ case BTLA_DTYPE::S8:+ // TODO(Yu): add Xe Arch condition+ if (can_comps8(p)) {+ p->compute_type = BTLA_DTYPE::S8;+ return;+ }+ [[fallthrough]];+ case BTLA_DTYPE::F16:+ p->compute_type = BTLA_DTYPE::F16;+ return;+ [[fallthrough]];+ case BTLA_DTYPE::F32:+ default:+ p->compute_type = BTLA_DTYPE::F32;+ return;+ }+ }++ static void packq_int8(int8_t* raws8, int8_t* blob, QuantParam* p, sycl::queue* q) {+ size_t k = p->k;+ size_t n = p->n;+ auto psrc = raws8;+ constexpr int SG_SIZE = 16;+ auto ker = [&](sycl::handler& cgh) {+ cgh.parallel_for(sycl::nd_range<2>({k, n}, {1, SG_SIZE}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(SG_SIZE)]]+ {+ auto g_0 = it.get_global_id()[0];+ auto g_1 = it.get_global_id()[1];+ int8_t src0 = psrc[g_0 * n + g_1];+ *((int8_t*)blob + g_1 * k + g_0) = src0;+ });+ };+ q->submit(ker);+ }++ static void packq_int4(int8_t* raws8, int8_t* blob, QuantParam* p, sycl::queue* q) {+ size_t k = p->k;+ size_t n = p->n;+ auto psrc = raws8;+ constexpr int SG_SIZE = 16;+ auto ker = [&](sycl::handler& cgh) {+ cgh.parallel_for(sycl::nd_range<2>({k / 2, n}, {1, SG_SIZE}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(SG_SIZE)]]+ {+ auto g_0 = it.get_global_id()[0];+ auto g_1 = it.get_global_id()[1];+ int8_t src0 = psrc[(g_0 * 2 + 0) * n + g_1] + 8;+ int8_t src1 = psrc[(g_0 * 2 + 1) * n + g_1] + 8;+ uint8_t dst = src0 | (src1 << 4);+ *((uint8_t*)blob + g_1 * k / 2 + g_0) = dst;+ });+ };+ q->submit(ker);+ }++ static void packq_int2(int8_t* raws8, int8_t* blob, QuantParam* p, sycl::queue* q) {+ size_t k = p->k;+ size_t n = p->n;+ auto psrc = raws8;+ constexpr int SG_SIZE = 16;+ auto ker = [&](sycl::handler& cgh) {+ cgh.parallel_for(sycl::nd_range<2>({k / 4, n}, {1, SG_SIZE}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(SG_SIZE)]]+ {+ auto g_0 = it.get_global_id()[0];+ auto g_1 = it.get_global_id()[1];+ uint8_t src0 = static_cast<uint8_t>(psrc[(g_0 * 4 + 0) * n + g_1] + 2);+ uint8_t src1 = static_cast<uint8_t>(psrc[(g_0 * 4 + 1) * n + g_1] + 2);+ uint8_t src2 = static_cast<uint8_t>(psrc[(g_0 * 4 + 2) * n + g_1] + 2);+ uint8_t src3 = static_cast<uint8_t>(psrc[(g_0 * 4 + 3) * n + g_1] + 2);+ uint8_t dst = src0 | (src1 << 2) | (src2 << 4) | (src3 << 6);+ *((uint8_t*)blob + g_1 * k / 4 + g_0) = dst;+ });+ };+ q->submit(ker);+ }++ static void packscale(void* scaleptr, int8_t* blobptr, QuantParam* p, sycl::queue* q) {+ size_t k = p->k;+ size_t n = p->n;+ int block = p->blocksize;+ size_t blks = p->blks();+ auto ssize = get_scale_size_bytes(p);+ constexpr int SG_SIZE = 16;+ auto dstptr = blobptr + get_scale_offset(p);+ if (p->scale_type == BTLA_DTYPE::F8_E8M0) {+ auto psrc_f32 = (const float*)scaleptr;+ auto pdst_s8 = (int8_t*)dstptr;+ auto ker = [&](sycl::handler& cgh) {+ cgh.parallel_for(sycl::nd_range<2>({blks, n}, {1, SG_SIZE}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(SG_SIZE)]] {+ auto g_0 = it.get_global_id()[0];+ auto g_1 = it.get_global_id()[1];+ float v = psrc_f32[g_0 * n + g_1];+ int exp = static_cast<int>(sycl::rint(v));+ exp = (exp > 127) ? 127 : exp;+ exp = (exp < -127) ? -127 : exp;+ pdst_s8[g_1 * blks + g_0] = static_cast<int8_t>(exp);+ });+ };+ q->submit(ker);+ return;+ }+ auto psrc = (int8_t*)scaleptr;+ auto ker = [&](sycl::handler& cgh) {+ cgh.parallel_for(sycl::nd_range<2>({blks, n}, {1, SG_SIZE}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(SG_SIZE)]]+ {+ auto g_0 = it.get_global_id()[0];+ auto g_1 = it.get_global_id()[1];+ int8_t tmp[4];+ for (size_t i = 0; i < ssize; i++) {+ tmp[i] = psrc[g_0 * n * ssize + g_1 * ssize + i];+ }+ for (size_t i = 0; i < ssize; i++) {+ *((int8_t*)dstptr + g_0 * ssize + g_1 * blks * ssize + i) = tmp[i];+ }+ });+ };+ q->submit(ker);+ if (rescale(p)) {+#ifdef ARK_RESCALE+ auto scalext_ptr = (int8_t*)blobptr + get_scalext_offset(p);+ int newblock_size = rescale_blocksize(p);+ auto newblks = p->k / newblock_size;+ auto st = p->scale_type;+ if (st == BTLA_DTYPE::F32) {+ using T = float;+ auto ker2 = [&](sycl::handler& cgh) {+ cgh.parallel_for(sycl::nd_range<2>({1, n}, {1, SG_SIZE}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(SG_SIZE)]]+ {+ auto g_1 = it.get_global_id()[1];+ auto sg = it.get_sub_group();+ auto sptr = (T*)scaleptr;+ auto sxtptr = (T*)scalext_ptr;+ for (int j = 0; j < newblks; j++) {+ float maxv = 0.f;+ int start_blk = j * newblock_size / block;+ int end_blk = (j + 1) * newblock_size / block;+ for (int i = start_blk; i < end_blk; i += 1) {+ maxv = std::max(abs(sptr[i * n + g_1]), maxv);+ }+ sxtptr[g_1 * newblks + j] = maxv * get_weight_fullrange(p) / 127.f;+ }+ });+ };+ q->submit(ker2);+ }+ if (st == BTLA_DTYPE::F16) {+ using T = sycl::half;+ auto ker2 = [&](sycl::handler& cgh) {+ cgh.parallel_for(sycl::nd_range<2>({1, n}, {1, SG_SIZE}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(SG_SIZE)]]+ {+ auto g_1 = it.get_global_id()[1];+ auto sg = it.get_sub_group();+ auto sptr = (T*)scaleptr;+ auto sxtptr = (T*)scalext_ptr;+ for (int j = 0; j < newblks; j++) {+ float maxv = 0.f;+ int start_blk = j * newblock_size / block;+ int end_blk = (j + 1) * newblock_size / block;+ for (int i = start_blk; i < end_blk; i += 1) {+ maxv = std::max(abs((float)sptr[i * n + g_1]), maxv);+ }+ sxtptr[g_1 * newblks + j] = maxv * get_weight_fullrange(p) / 127.f;+ }+ });+ };+ q->submit(ker2);+ }+#endif+ }+ }++ static void packzp(int8_t* zpptr, int8_t* blob, QuantParam* p, sycl::queue* q) {+ if (zpptr == nullptr || !p->asym) return;+ size_t k = p->k;+ size_t n = p->n;+ int block = p->blocksize;+ size_t blks = p->blks();+ auto dstptr = (int8_t*)blob + get_zp_offset(p);+ constexpr int SG_SIZE = 16;+ auto ker = [&](sycl::handler& cgh) {+ cgh.parallel_for(sycl::nd_range<2>({blks, n}, {1, SG_SIZE}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(SG_SIZE)]]+ {+ auto g_0 = it.get_global_id()[0];+ auto g_1 = it.get_global_id()[1];+ *(dstptr + g_0 + g_1 * blks) = zpptr[g_0 * n + g_1];+ });+ };+ q->submit(ker);+ }++ static void packq_fp8(int8_t* raws8, int8_t* blob, QuantParam* p, sycl::queue* q) {+ size_t k = p->k;+ size_t n = p->n;+ auto psrc = (uint8_t*)raws8;+ constexpr int SG_SIZE = 16;+ auto ker = [&](sycl::handler& cgh) {+ cgh.parallel_for(sycl::nd_range<2>({k, n}, {1, SG_SIZE}),+ [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(SG_SIZE)]]+ {+ auto g_0 = it.get_global_id()[0];+ auto g_1 = it.get_global_id()[1];+ uint8_t src0 = psrc[g_0 * n + g_1];+ *((uint8_t*)blob + g_1 * k + g_0) = src0;+ });+ };+ q->submit(ker);+ }++ static void packq(int8_t* raws8, void* scaleptr, int8_t* zpptr, int8_t* blob, QuantParam* p, sycl::queue* q) {+ auto type = bestla::utils::bestla_dtype_type(p->weight_type);+ auto typeint = bestla::utils::bestla_dtype_type(BTLA_DTYPE::TypeInt);+ if (type == typeint) {+ // integer data is row-major NT layout+ if (p->weight_type == BTLA_DTYPE::S2) {+ packq_int2(raws8, blob, p, q);+ }+ if (p->weight_type == BTLA_DTYPE::S4) {+ packq_int4(raws8, blob, p, q);+ }+ if (p->weight_type == BTLA_DTYPE::S8) {+ packq_int8(raws8, blob, p, q);+ }+ }+ if (p->weight_type == BTLA_DTYPE::F8_E4M3 || p->weight_type == BTLA_DTYPE::F8_E5M2) {+ // FP8 data is row-major NT layout, same as int8+ packq_fp8(raws8, blob, p, q);+ }+ packscale(scaleptr, blob, p, q);+ packzp(zpptr, blob, p, q);+ }++ static void unpackq(BTLA_DTYPE outt, int8_t* blob, void* optr, QuantParam* p, sycl::queue* q) {+ using namespace bestla;+ using namespace bestla::sycl_prologue_b;+ auto qptr = (uint8_t*)blob;+ auto scale_ptr = blob + get_scale_offset(p);+ auto zp_ptr = p->asym ? blob + get_zp_offset(p) : nullptr;+ int blks = p->blks();+ if (p->weight_type == BTLA_DTYPE::S2) {+ if (outt == BTLA_DTYPE::S8) {+ bool _rescale = rescale(p);+ if (_rescale) {+#ifdef ARK_RESCALE+ auto scalext_ptr = blob + get_scalext_offset(p);+ int newblocksize = rescale_blocksize(p);+ if (p->scale_type == BTLA_DTYPE::F32) {+ using ProB = WeightS2T<float>;+ ProB::template dequantS8<typename ProB::CfgDequantS8Rescale>(+ p->n, p->k, p->blocksize, {qptr, (float*)scale_ptr, blks, nullptr, zp_ptr, (float*)scalext_ptr},+ (int8_t*)optr, q, newblocksize);+ }+ if (p->scale_type == BTLA_DTYPE::F16) {+ using ProB = WeightS2T<sycl::half>;+ ProB::template dequantS8<typename ProB::CfgDequantS8Rescale>(+ p->n, p->k, p->blocksize,+ {qptr, (sycl::half*)scale_ptr, blks, nullptr, zp_ptr, (sycl::half*)scalext_ptr}, (int8_t*)optr, q,+ newblocksize);+ }+#endif+ } else {+ using ProB = WeightS2T<float>;+ ProB::template dequantS8<typename ProB::CfgDequantS8>(+ p->n, p->k, p->blocksize, {qptr, nullptr, blks, nullptr, zp_ptr}, (int8_t*)optr, q);+ }+ return;+ }+ if (p->scale_type == BTLA_DTYPE::F32) {+ using ProB = WeightS2T<float>;+ ProB::template dequant<typename ProB::CfgDequantF32>(+ p->n, p->k, p->blocksize, {qptr, (float*)scale_ptr, blks, nullptr, zp_ptr}, (float*)optr, q);+ }+ if (p->scale_type == BTLA_DTYPE::F16) {+ using ProB = WeightS2T<sycl::half>;+ ProB::template dequant<typename ProB::CfgDequantF16>(+ p->n, p->k, p->blocksize, {qptr, (sycl::half*)scale_ptr, blks, nullptr, zp_ptr}, (sycl::half*)optr, q);+ }+ return;+ }+ if (p->weight_type == BTLA_DTYPE::S4) {+ if (outt == BTLA_DTYPE::S8) {+ bool _rescale = rescale(p);+ if (_rescale) {+#ifdef ARK_RESCALE+ auto scalext_ptr = blob + get_scalext_offset(p);+ int newblocksize = rescale_blocksize(p);+ if (p->scale_type == BTLA_DTYPE::F32) {+ using ProB = WeightS4T<float>;+ ProB::template dequantS8<typename ProB::CfgDequantS8Rescale>(+ p->n, p->k, p->blocksize, {qptr, (float*)scale_ptr, blks, nullptr, zp_ptr, (float*)scalext_ptr},+ (int8_t*)optr, q, newblocksize);+ }+ if (p->scale_type == BTLA_DTYPE::F16) {+ using ProB = WeightS4T<sycl::half>;+ ProB::template dequantS8<typename ProB::CfgDequantS8Rescale>(+ p->n, p->k, p->blocksize,+ {qptr, (sycl::half*)scale_ptr, blks, nullptr, zp_ptr, (sycl::half*)scalext_ptr}, (int8_t*)optr, q,+ newblocksize);+ }+#endif+ } else {+ using ProB = WeightS4T<float>;+ ProB::template dequantS8<typename ProB::CfgDequantS8>(+ p->n, p->k, p->blocksize, {qptr, nullptr, blks, nullptr, zp_ptr}, (int8_t*)optr, q);+ }+ return;+ }+ if (p->scale_type == BTLA_DTYPE::F32) {+ using ProB = WeightS4T<float>;+ ProB::template dequant<typename ProB::CfgDequantF32>(+ p->n, p->k, p->blocksize, {qptr, (float*)scale_ptr, blks, nullptr, zp_ptr}, (float*)optr, q);+ }+ if (p->scale_type == BTLA_DTYPE::F16) {+ using ProB = WeightS4T<sycl::half>;+ ProB::template dequant<typename ProB::CfgDequantF16>(+ p->n, p->k, p->blocksize, {qptr, (sycl::half*)scale_ptr, blks, nullptr, zp_ptr}, (sycl::half*)optr, q);+ }+ return;+ }+ if (p->weight_type == BTLA_DTYPE::S8) {+ if (p->scale_type == BTLA_DTYPE::F32) {+ using ProB = WeightS8T<float>;+ ProB::template dequant<typename ProB::CfgDequantF32>(+ p->n, p->k, p->blocksize, {(int8_t*)qptr, (float*)scale_ptr, blks, nullptr, zp_ptr}, (float*)optr, q);+ }+ if (p->scale_type == BTLA_DTYPE::F16) {+ using ProB = WeightS8T<sycl::half>;+ ProB::template dequant<typename ProB::CfgDequantF16>(+ p->n, p->k, p->blocksize, {(int8_t*)qptr, (sycl::half*)scale_ptr, blks, nullptr, zp_ptr}, (sycl::half*)optr,+ q);+ }+ return;+ }+ if (p->weight_type == BTLA_DTYPE::F8_E4M3 || p->weight_type == BTLA_DTYPE::F8_E5M2) {+ auto e4m3 = p->weight_type == BTLA_DTYPE::F8_E4M3;+ if (p->scale_type != BTLA_DTYPE::F8_E8M0) {+ if (p->scale_type == BTLA_DTYPE::F32) {+ using ST = float;+ if (e4m3) {+ using ProB = WeightF8T<ST, true>;+ ProB::dequant<ProB::Cfg>(p->n, p->k, p->blocksize, {(const uint8_t*)qptr, (ST*)scale_ptr, blks}, (ST*)optr,+ q);+ } else {+ using ProB = WeightF8T<ST, false>;+ ProB::dequant<ProB::Cfg>(p->n, p->k, p->blocksize, {(const uint8_t*)qptr, (ST*)scale_ptr, blks}, (ST*)optr,+ q);+ }+ return;+ }+ if (p->scale_type == BTLA_DTYPE::F16) {+ using ST = sycl::half;+ if (e4m3) {+ using ProB = WeightF8T<ST, true>;+ ProB::dequant<ProB::Cfg>(p->n, p->k, p->blocksize, {(const uint8_t*)qptr, (ST*)scale_ptr, blks}, (ST*)optr,+ q);+ } else {+ using ProB = WeightF8T<ST, false>;+ ProB::dequant<ProB::Cfg>(p->n, p->k, p->blocksize, {(const uint8_t*)qptr, (ST*)scale_ptr, blks}, (ST*)optr,+ q);+ }+ return;+ }+ } else {+ using ST = bestla::utils::f8;+ if (outt == BTLA_DTYPE::F32) {+ using T = float;+ if (e4m3) {+ using ProB = WeightF8T<ST, true>;+ ProB::dequant<ProB::Cfg>(p->n, p->k, p->blocksize, {(const uint8_t*)qptr, (ST*)scale_ptr, blks}, (T*)optr,+ q);+ } else {+ using ProB = WeightF8T<ST, false>;+ ProB::dequant<ProB::Cfg>(p->n, p->k, p->blocksize, {(const uint8_t*)qptr, (ST*)scale_ptr, blks}, (T*)optr,+ q);+ }+ } else if (outt == BTLA_DTYPE::F16) {+ using T = sycl::half;+ if (e4m3) {+ using ProB = WeightF8T<ST, true>;+ ProB::dequant<ProB::Cfg>(p->n, p->k, p->blocksize, {(const uint8_t*)qptr, (ST*)scale_ptr, blks}, (T*)optr,+ q);+ } else {+ using ProB = WeightF8T<ST, false>;+ ProB::dequant<ProB::Cfg>(p->n, p->k, p->blocksize, {(const uint8_t*)qptr, (ST*)scale_ptr, blks}, (T*)optr,+ q);+ }+ }++ return;+ }+ return;+ }+ }++ static int woq_gemv(sycl::queue* q, size_t m, QuantParam* p, const void* matA, const void* blobB, void* matC,+ const void* bias, BTLA_DTYPE outt) {+ if (m > 1) return -2;+ using namespace bestla;+ using namespace bestla::sycl_prologue_b;+ auto qptr = (uint8_t*)blobB;+ auto scale_ptr = (int8_t*)blobB + get_scale_offset(p);+ auto zp_ptr = p->asym ? (int8_t*)blobB + get_zp_offset(p) : nullptr;+ int blks = p->blks();+ if (p->weight_type == BTLA_DTYPE::S2) {+ if (p->scale_type == BTLA_DTYPE::F32) {+ using ST = float;+ using T = float;+ using ProB = WeightS2T<ST>;+ ProB::gemv<ProB::CfgGemvF32>((const T*)matA,+ {(const uint8_t*)qptr, (ST*)scale_ptr, blks, (const T*)bias, zp_ptr}, (T*)matC,+ p->n, p->k, p->blocksize, q);+ }+ if (p->scale_type == BTLA_DTYPE::F16) {+ using ST = sycl::half;+ using ProB = WeightS2T<ST>;+ using T = sycl::half;+ ProB::gemv<ProB::CfgGemvF16>((const T*)matA,+ {(const uint8_t*)qptr, (ST*)scale_ptr, blks, (const T*)bias, zp_ptr}, (T*)matC,+ p->n, p->k, p->blocksize, q);+ }+ if (p->scale_type == BTLA_DTYPE::BF16) {+ return -1;+ }+ return 0;+ }+ if (p->weight_type == BTLA_DTYPE::S4) {+ if (p->scale_type == BTLA_DTYPE::F32) {+ using ST = float;+ using T = float;+ using ProB = WeightS4T<ST>;+ ProB::gemv<ProB::CfgGemvF32>((const T*)matA,+ {(const uint8_t*)qptr, (ST*)scale_ptr, blks, (const T*)bias, zp_ptr}, (T*)matC,+ p->n, p->k, p->blocksize, q);+ }+ if (p->scale_type == BTLA_DTYPE::F16) {+ using ST = sycl::half;+ using ProB = WeightS4T<ST>;+ using T = sycl::half;+ ProB::gemv<ProB::CfgGemvF16>((const T*)matA,+ {(const uint8_t*)qptr, (ST*)scale_ptr, blks, (const T*)bias, zp_ptr}, (T*)matC,+ p->n, p->k, p->blocksize, q);+ }+ if (p->scale_type == BTLA_DTYPE::BF16) {+ return -1;+ }+ return 0;+ }+ if (p->weight_type == BTLA_DTYPE::S8) {+ if (p->scale_type == BTLA_DTYPE::F32) {+ using ST = float;+ using T = float;+ using ProB = WeightS8T<ST>;+ ProB::gemv<ProB::CfgGemvF32>((const T*)matA,+ {(const int8_t*)qptr, (ST*)scale_ptr, blks, (const T*)bias, zp_ptr}, (T*)matC,+ p->n, p->k, p->blocksize, q);+ }+ if (p->scale_type == BTLA_DTYPE::F16) {+ using ST = sycl::half;+ using ProB = WeightS8T<ST>;+ using T = sycl::half;+ ProB::gemv<ProB::CfgGemvF16>((const T*)matA,+ {(const int8_t*)qptr, (ST*)scale_ptr, blks, (const T*)bias, zp_ptr}, (T*)matC,+ p->n, p->k, p->blocksize, q);+ }+ if (p->scale_type == BTLA_DTYPE::BF16) {+ return -1;+ }+ return 0;+ }+ if (p->weight_type == BTLA_DTYPE::F8_E4M3 || p->weight_type == BTLA_DTYPE::F8_E5M2) {+ auto e4m3 = p->weight_type == BTLA_DTYPE::F8_E4M3;+ if (p->scale_type != BTLA_DTYPE::F8_E8M0) {+ if (e4m3) {+ auto constexpr E4M3_T = true;+ if (p->scale_type == BTLA_DTYPE::F32) {+ using T = float;+ using ProB = WeightF8T<T, E4M3_T>;+ ProB::gemv((T*)matA, {(const uint8_t*)qptr, (T*)scale_ptr, blks, bias}, (T*)matC, p->n, p->k, p->blocksize,+ q);+ }+ if (p->scale_type == BTLA_DTYPE::F16) {+ using T = sycl::half;+ using ProB = WeightF8T<T, E4M3_T>;+ ProB::gemv((T*)matA, {(const uint8_t*)qptr, (T*)scale_ptr, blks, bias}, (T*)matC, p->n, p->k, p->blocksize,+ q);+ }+ } else {+ auto constexpr E4M3_T = false;+ if (p->scale_type == BTLA_DTYPE::F32) {+ using T = float;+ using ProB = WeightF8T<T, E4M3_T>;+ ProB::gemv((T*)matA, {(const uint8_t*)qptr, (T*)scale_ptr, blks, bias}, (T*)matC, p->n, p->k, p->blocksize,+ q);+ }+ if (p->scale_type == BTLA_DTYPE::F16) {+ using T = sycl::half;+ using ProB = WeightF8T<T, E4M3_T>;+ ProB::gemv((T*)matA, {(const uint8_t*)qptr, (T*)scale_ptr, blks, bias}, (T*)matC, p->n, p->k, p->blocksize,+ q);+ }+ }+ } else {+ if (e4m3) {+ auto constexpr E4M3_T = true;+ using ST = bestla::utils::f8;+ using ProB = WeightF8T<ST, E4M3_T>;+ if (outt == BTLA_DTYPE::F32) {+ using T = float;+ ProB::gemv((T*)matA, {(const uint8_t*)qptr, (ST*)scale_ptr, blks, bias}, (T*)matC, p->n, p->k, p->blocksize,+ q);+ }+ if (outt == BTLA_DTYPE::F16) {+ using T = sycl::half;+ ProB::gemv((T*)matA, {(const uint8_t*)qptr, (ST*)scale_ptr, blks, bias}, (T*)matC, p->n, p->k, p->blocksize,+ q);+ }+ } else {+ auto constexpr E4M3_T = false;+ using ST = bestla::utils::f8;+ using ProB = WeightF8T<ST, E4M3_T>;+ if (outt == BTLA_DTYPE::F32) {+ using T = float;+ ProB::gemv((T*)matA, {(const uint8_t*)qptr, (ST*)scale_ptr, blks, bias}, (T*)matC, p->n, p->k, p->blocksize,+ q);+ }+ if (outt == BTLA_DTYPE::F16) {+ using T = sycl::half;+ ProB::gemv((T*)matA, {(const uint8_t*)qptr, (ST*)scale_ptr, blks, bias}, (T*)matC, p->n, p->k, p->blocksize,+ q);+ }+ }+ }+ return 0;+ }+ return -1;+ }++ static inline bool rescale(QuantParam* p) {+#ifdef ARK_RESCALE+ if (env_params::Instance()->auto_s8 == -1) return true;+ if (env_params::Instance()->auto_s8 > p->blocksize && p->k % env_params::Instance()->auto_s8 == 0) return true;+#endif+ return false;+ }++ static inline int rescale_blocksize(QuantParam* p) {+#ifdef ARK_RESCALE+ if (env_params::Instance()->auto_s8 == 0) return p->blocksize;+ return env_params::Instance()->auto_s8 == -1 ? p->k : env_params::Instance()->auto_s8;+#else+ return p->blocksize;+#endif+ }++ static void woq_gemm(int m, const void* a, const void* b, void* c, const void* bias, BTLA_DTYPE acdt, QuantParam* p,+ sycl::queue* q) {+ auto ret = woq_gemv(q, m, p, a, b, c, bias, acdt);+ if (ret) {+ auto dnnl_dt = to_dt(acdt);+ check_compute_type(p);+ if (p->compute_type != BTLA_DTYPE::S8) {+ size_t elesize = bestla::utils::bestla_dtype_bytes(acdt);+ size_t total_size = elesize * p->k * p->n;+ auto ptr = DnnlContext::Instance()->get_scratch_mem(total_size, 1, q);+ unpackq(acdt, (int8_t*)b, ptr, p, q);+ DnnlWrapper::gemm(q, m, p->n, p->k, a, dnnl_dt, ptr, dnnl_dt, true, c, dnnl_dt, bias);+ } else {+ auto bptr = (int8_t*)b;+ bool _rescale = rescale(p);+ auto scaleb_ptr = (int8_t*)b + get_scale_offset(p);+ if (p->weight_type == BTLA_DTYPE::S8) {+ scaleb_ptr = (int8_t*)b + get_scale_offset(p);+ } else {+ size_t elesize = 1;+ size_t total_size = elesize * p->k * p->n;+ bptr = (int8_t*)DnnlContext::Instance()->get_scratch_mem(total_size, 2, q);+ unpackq(BTLA_DTYPE::S8, (int8_t*)b, bptr, p, q);+ scaleb_ptr = _rescale ? (int8_t*)b + get_scalext_offset(p) : (int8_t*)b + get_scale_offset(p);+ }+ auto blocksize = rescale_blocksize(p);+ DnnlWrapper::woq_s8(q, m, p->n, p->k, a, bptr, true, c, dnnl_dt, scaleb_ptr, (void*)bias, blocksize);+ }+ }+ }++ // input: num_rows x head_dim matrix, output: int8 quantized matrix + scale (per block)+ // scale: num_rows // block_size+ template <typename T>+ static void sage_dynamic_quant(sycl::queue* q, const T* in_ptr, int8_t* out_ptr, float* scale_ptr, int num_rows,+ int seq, int n_seq_blk, int head_dim, int block_size) {+ int num_blocks = num_rows * n_seq_blk;+ int elems_per_block = block_size * head_dim;++ // Work-group size: use 256 threads (16 sub-groups × 16 lanes).+ // For block_size=1, head_dim=128: 128 elements / 256 threads < 1, so we use smaller WG.+ // For block_size=64, head_dim=128: 8192 elements / 256 threads = 32 elements per thread.+ constexpr int SG_SIZE = 32;+ constexpr int MAX_Reg = 64;+ constexpr int MAX_WG_SIZE = 256;+ constexpr int Unroll = 4;+ if (elems_per_block > MAX_Reg * MAX_WG_SIZE) {+ int wg_size = (elems_per_block <= 256) ? SG_SIZE : 256;+ // Ensure wg_size is a multiple of SG_SIZE+ wg_size = ((wg_size + SG_SIZE - 1) / SG_SIZE) * SG_SIZE;++ q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ int block_id = item.get_group(0);+ int row_id = block_id / n_seq_blk;+ int seq_id = block_id % n_seq_blk;+ int tid = item.get_local_id(0);+ auto wg = item.get_group();+ auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;+ auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;++ int elems_this_wg =+ (seq_id + 1) * block_size <= seq ? elems_per_block : (seq - seq_id * block_size) * head_dim;++ // Phase 1: compute absmax across entire block+ float local_max = 0.0f;+ sycl::vec<T, Unroll> local_data, local_max_vec;+ local_max_vec = sycl::vec<T, Unroll>(0.0f);+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {+ local_data = *(sycl::vec<T, Unroll>*)(&block_in[i]);+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data));+ }+ for (int i = 0; i < Unroll; ++i) {+ local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));+ }+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});++ // Compute scale+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;++ // Store scale (one thread writes)+ if (tid == 0) {+ scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;+ }++ // Phase 2: fused quantize+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll) {+#pragma unroll+ for (int j = 0; j < Unroll; ++j) {+ float val = static_cast<float>(block_in[i + j]) * inv_scale;+ int iv = static_cast<int>(val + (val >= 0.0f ? 0.5f : -0.5f));+ iv = sycl::clamp(iv, -127, 127);+ block_out[i + j] = static_cast<int8_t>(iv);+ }+ }+ });+ } else {+ int wg_size = MAX_WG_SIZE;+ q->parallel_for(sycl::nd_range<1>(num_blocks * wg_size, wg_size),+ [=](sycl::nd_item<1> item) [[intel::reqd_sub_group_size(SG_SIZE)]] {+ int seq_id = item.get_group(0);+ int row_id = seq_id / n_seq_blk;+ seq_id = seq_id % n_seq_blk;+ int tid = item.get_local_id(0);+ auto wg = item.get_group();+ auto* block_in = in_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;+ auto* block_out = out_ptr + (size_t)row_id * seq * head_dim + seq_id * block_size * head_dim;++ // Phase 1: compute absmax across entire block+ float local_max = 0.0f;+ sycl::vec<T, Unroll> local_data[MAX_Reg / Unroll], local_max_vec;+ local_max_vec = sycl::vec<T, Unroll>(0.0f);+ int local_i = 0;+ int elems_this_wg = (seq_id + 1) * block_size <= seq ? block_size * head_dim+ : (seq - seq_id * block_size) * head_dim;+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {+ local_data[local_i] = *(sycl::vec<T, Unroll>*)&block_in[i];+ local_max_vec = sycl::fmax(local_max_vec, sycl::fabs(local_data[local_i]));+ }+#pragma unroll+ for (int i = 0; i < Unroll; ++i) {+ local_max = sycl::fmax(local_max, static_cast<float>(local_max_vec[i]));+ }+ float absmax = sycl::reduce_over_group(wg, local_max, sycl::maximum<float>{});++ // Compute scale+ float inv_scale = (absmax > 0.0f) ? (127.0f / absmax) : 0.0f;++ // Store scale (one thread writes)+ if (tid == 0) {+ scale_ptr[row_id * n_seq_blk + seq_id] = absmax / 127.0f;+ }++ // Phase 2: fused quantize+ local_i = 0;+ for (int i = tid * Unroll; i < elems_this_wg; i += wg_size * Unroll, local_i++) {+ sycl::vec<float, Unroll> val =+ local_data[local_i].template convert<float, sycl::rounding_mode::automatic>();+ val = val * inv_scale;+ val = sycl::round(val);+ val = sycl::clamp(val, -127, 127);+ sycl::vec<int8_t, Unroll> qv = val.template convert<int8_t, sycl::rounding_mode::automatic>();+ *(sycl::vec<int8_t, Unroll>*)(&block_out[i]) = qv;+ }+ });+ }+ }++ static void sagev1(sycl::queue* q, void* Q_ptr, void* K_ptr, void* V_ptr, void* O_ptr, void* mask,+ int scale_block_size, int batch, int num_heads_q, int num_heads_kv, int seq_len_q, int seq_len_kv,+ int head_dim, float softmax_scale, bool is_causal) {+ size_t seq_q_blk = (seq_len_q + scale_block_size - 1) / scale_block_size;+ size_t seq_kv_blk = (seq_len_kv + scale_block_size - 1) / scale_block_size;+ size_t q_size = num_heads_q * seq_len_q * head_dim * batch;+ size_t q_scale_size = num_heads_q * seq_q_blk * batch;+ size_t k_size = num_heads_kv * seq_len_kv * head_dim * batch;+ size_t k_scale_size = num_heads_kv * seq_kv_blk * batch;+ size_t total_size = k_size + q_size + k_scale_size * sizeof(float) + q_scale_size * sizeof(float);+ auto ptr = DnnlContext::Instance()->get_scratch_mem(total_size, 1, q);+ auto q_out_ptr = (int8_t*)ptr;+ auto k_out_ptr = (int8_t*)ptr + q_size;+ auto qscale = (float*)((int8_t*)ptr + q_size + k_size);+ auto kscale = (float*)((int8_t*)ptr + q_size + k_size + q_scale_size * sizeof(float));+ sage_dynamic_quant<sycl::half>(q, (sycl::half*)Q_ptr, (int8_t*)q_out_ptr, (float*)qscale, batch * num_heads_q,+ seq_len_q, seq_q_blk, head_dim, scale_block_size);+ sage_dynamic_quant<sycl::half>(q, (sycl::half*)K_ptr, (int8_t*)k_out_ptr, (float*)kscale, batch * num_heads_kv,+ seq_len_kv, seq_kv_blk, head_dim, scale_block_size);+ ark::sdpa_impl_qks8_pvhalf(q, q_out_ptr, k_out_ptr, V_ptr, O_ptr, mask, scale_block_size, qscale, kscale, batch,+ num_heads_q, num_heads_kv, seq_len_q, seq_len_kv, head_dim, softmax_scale, is_causal);+ }+};++} // namespace arkdiff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/common.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/common.hppnew file mode 100644index 000000000..7f9b397be--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/common.hpp@@ -0,0 +1,46 @@+#pragma once++#include <string>+#include <vector>+#include "../include/dnnl_wrapper.hpp"+#include "bestla/bestla_utils.h"++using namespace bestla;+using fp16 = utils::fp16;+using bf16 = utils::bf16;++struct Context {+ Context() {+#if ARK_XPU+ q = new sycl::queue(sycl::gpu_selector_v);+#else+ q = nullptr;+#endif+ }+ sycl::queue* q = nullptr;+ static Context* Instance() {+ static Context inst;+ return &inst;+ }++ void* allocate(size_t size) {+#if ARK_XPU+ return sycl::aligned_alloc_device<int8_t>(128, size, *q);+#else+ return malloc(size);+#endif+ }++ void deallocate(void* ptr) {+ if (!ptr) return;+#ifdef ARK_XPU+ sycl::free(ptr, *q);+#else+ free(ptr);+#endif+ }+};++#define GETQ() \+ auto ctx = Context::Instance(); \+ auto q = Context::Instance()->q;\ No newline at end of filediff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_gemm.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_gemm.hppnew file mode 100644index 000000000..ae5a17d33--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_gemm.hpp@@ -0,0 +1,67 @@+#pragma once+#include "common.hpp"++struct TestGemm {+ TestGemm() {+ test<float>(128, 128, 128);+#ifdef ARK_XPU+ test<bf16>(128, 128, 128);+ test<fp16>(128, 128, 128);+ test_s8s8<float>(128, 128, 128);+ test_woqs8<float>(128, 128, 128);+#endif+ }++ template <typename T>+ void test(size_t m, size_t n, size_t k) {+ GETQ();+ LOG_LINE();+ auto dt = ark::to_dt<T>();+ auto A = ctx->allocate(m * k * sizeof(T));+ auto B = ctx->allocate(n * k * sizeof(T));+ auto C = ctx->allocate(m * n * sizeof(T));+ ark::DnnlWrapper::gemm(q, m, n, k, A, dt, B, dt, false, C, dt, nullptr);+#ifdef ARK_XPU+ q->wait();+#endif+ ctx->deallocate(A);+ ctx->deallocate(B);+ ctx->deallocate(C);+ }++ template <typename T>+ void test_s8s8(size_t m, size_t n, size_t k) {+ GETQ();+ LOG_LINE();+ auto dt = ark::to_dt<T>();+ auto A = ctx->allocate(m * k * sizeof(int8_t));+ auto B = ctx->allocate(n * k * sizeof(int8_t));+ auto C = ctx->allocate(m * n * sizeof(T));+ auto scaleA = ctx->allocate(1 * sizeof(T));+ auto scaleB = ctx->allocate(n * sizeof(T));+ ark::DnnlWrapper::igemm_s8s8(q, m, n, k, A, B, true, C, dt, scaleA, scaleB, nullptr);+ ctx->deallocate(A);+ ctx->deallocate(B);+ ctx->deallocate(C);+ ctx->deallocate(scaleA);+ ctx->deallocate(scaleB);+ }++ template <typename T>+ void test_woqs8(size_t m, size_t n, size_t k) {+ GETQ();+ LOG_LINE();+ auto dt = ark::to_dt<T>();+ auto A = ctx->allocate(m * k * sizeof(T));+ auto B = ctx->allocate(n * k * sizeof(int8_t));+ auto C = ctx->allocate(m * n * sizeof(T));+ auto scaleB = ctx->allocate(n * sizeof(T));+ ark::DnnlWrapper::woq_s8(q, m, n, k, A, B, true, C, dt, scaleB, nullptr, k);+ ctx->deallocate(A);+ ctx->deallocate(B);+ ctx->deallocate(C);+ ctx->deallocate(scaleB);+ }+};++static TestGemm sTestGemm;\ No newline at end of filediff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_main.cpp b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_main.cppnew file mode 100644index 000000000..9e067c3f0--- /dev/null+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/test/test_main.cpp@@ -0,0 +1,7 @@+#include <iostream>+#include "test_gemm.hpp"++int main() {+ printf("Welcome to ARK TEST\n");+ return 1;+}\ No newline at end of filediff --git a/auto_round_extension/ark/install_kernel.py b/auto_round_extension/ark/install_kernel.pydeleted file mode 100644index ef450e4a2..000000000--- a/auto_round_extension/ark/install_kernel.py+++ /dev/null@@ -1,61 +0,0 @@-# Copyright (c) 2026 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--import re-import subprocess-import sys---def get_torch_minor():- try:- import torch-- m = re.match(r"^(\d+)\.(\d+)", torch.__version__)- return f"{m.group(1)}.{m.group(2)}" if m else None- except ImportError:- return None---def get_auto_round_minor():- try:- import auto_round-- m = re.match(r"^(\d+)\.(\d+)", auto_round.__version__)- return f"{m.group(1)}.{m.group(2)}" if m else None- except ImportError:- return None---# Map torch minor version to kernel version-auto_round_minor = "0.9" if get_auto_round_minor() is None else get_auto_round_minor()-KERNEL_MAP = {- "2.8": f"auto-round-lib~={auto_round_minor}.1.0",- "2.9": f"auto-round-lib~={auto_round_minor}.2.0",- "2.10": f"auto-round-lib~={auto_round_minor}.3.0",-}---def main():- torch_minor = get_torch_minor()- if torch_minor and torch_minor in KERNEL_MAP:- pkg = KERNEL_MAP[torch_minor]- print(f"Detected torch {torch_minor}, installing {pkg} ...")- subprocess.check_call([sys.executable, "-m", "pip", "install", pkg, "--upgrade-strategy", "only-if-needed"])- else:- print("torch not found or no mapping for your version. Installing the latest auto-round-lib ...")- subprocess.check_call([sys.executable, "-m", "pip", "install", "auto-round-lib"])---if __name__ == "__main__":- main()diff --git a/auto_round_extension/ark/qlinear.py b/auto_round_extension/ark/qlinear.pydeleted file mode 100644index b85d51c44..000000000--- a/auto_round_extension/ark/qlinear.py+++ /dev/null@@ -1,267 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--import math--import torch-import torch.nn as nn--from auto_round.export.export_to_awq.utils import dequantize_gemm, reverse_awq_order, unpack_awq-from auto_round.utils import convert_dtype_torch2str, logger--try:- import auto_round_kernel-- ARK_INSTALLED = True- ark = auto_round_kernel.ARK()-except:- ARK_INSTALLED = False--BITS_DTYPE_MAPPING = {- 2: "int2",- 4: "int4",- 8: "int8",-}---class QuantLinear(nn.Module):- QUANT_TYPE = "ark_gptq_nozp"- ZP_BIAS = 0-- def __init__(- self,- bits,- group_size,- sym,- in_features,- out_features,- bias,- weight_dtype=torch.bfloat16,- **kwargs,- ):- super().__init__()- if bits not in [2, 4, 8]:- raise NotImplementedError("Only 2, 4, 8 bits are supported for ARK.")- if not ARK_INSTALLED:- raise ModuleNotFoundError(- "The 'auto_round_kernel' module is required but not installed. "- "Please install the 'auto-round-lib' package, for example:\n"- " pip install auto-round-lib"- )-- self.infeatures = in_features- self.outfeatures = out_features- self.bits = bits- self.group_size = group_size if group_size != -1 else self.infeatures- self.maxq = 2**self.bits - 1- self.qbias = 2 ** (self.bits - 1)- self.pack_num = 32 // self.bits- self.weight_dtype = weight_dtype- self.asym = not sym- if not self.infeatures % self.group_size == 0:- raise NotImplementedError("in_features must be divisible by group_size")- if "awq" in self.QUANT_TYPE:- self.register_buffer(- "qweight", torch.zeros((in_features, out_features // self.pack_num), dtype=torch.int32)- )- else:- self.register_buffer(- "qweight",- torch.zeros((self.infeatures // 32 * self.bits, self.outfeatures), dtype=torch.int32),- )- self.register_buffer(- "qzeros",- torch.zeros(- (- math.ceil(self.infeatures / self.group_size),- self.outfeatures // 32 * self.bits,- ),- dtype=torch.int32,- ),- )- self.register_buffer(- "scales",- torch.zeros(- (math.ceil(self.infeatures / self.group_size), self.outfeatures),- dtype=weight_dtype,- ),- )- if bias:- self.register_buffer("bias", torch.zeros((self.outfeatures), dtype=torch.float))- else:- self.bias = None-- def extra_repr(self) -> str:- return "in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(- self.infeatures,- self.outfeatures,- self.bias is not None,- self.bits,- self.group_size,- )-- def post_init(self):- if self.qweight.device.type not in ["cpu", "xpu"]:- raise NotImplementedError(- f"Device type {self.qweight.device.type} is not supported. Only CPU and XPU devices are supported."- )- if self.qweight.device.type != "cpu" and self.asym:- raise NotImplementedError("Asymmetric quantization is only supported on CPU devices")- if "awq" in self.QUANT_TYPE:- intweight, zeros = unpack_awq(- self.qweight, self.qzeros, self.bits- ) # weight: k x n zeros: k / group_size x n- intweight, zeros = reverse_awq_order(intweight, zeros, self.bits) # weight: k x n zeros: k / group_size x n- intweight = torch.bitwise_and(intweight, self.maxq)- zeros = torch.bitwise_and(zeros, self.maxq)-- intweight = intweight - self.qbias # from uint8_t to int8_t- zeros = zeros - self.qbias # from uint8_t to int8_t- else:- # intweight: k x n, zeros: k / group_size x n- intweight, zeros = unpack_to_8bit_signed(self.qweight, self.qzeros, self.bits, self.ZP_BIAS, self.asym)- if zeros is None:- zeros = torch.empty(0, dtype=torch.int8)- else:- # change it to int8 with offset 128- zeros = (zeros.to(torch.int32) - self.qbias).to(torch.int8)- if self.asym:- intweight = (intweight.to(torch.int32) - self.qbias).to(torch.int8)- else:- intweight -= self.qbias- intweight = intweight.to(torch.int8)-- logger.debug(- f"ARK repack quantized weight: K:{intweight.shape[0]}, N:{intweight.shape[1]}, weight_dtype:{BITS_DTYPE_MAPPING[self.bits]}, scale_dtype:fp32, compute_dtype:fp32, group_size:{self.group_size}"- )-- if self.qweight.device.type == "xpu":- self.sdt = "fp16"- self.cdt = "int8"- self.torch_dt = torch.float16- else:- self.sdt = "fp32"- self.cdt = "auto"- self.torch_dt = torch.float32-- self.wdt = BITS_DTYPE_MAPPING[self.bits]- scales = self.scales.to(self.torch_dt).contiguous()-- self.qweight = ark.repack_quantized_weight(- intweight.contiguous(),- scales,- zeros.contiguous(),- self.group_size,- # compute_dtype- self.cdt,- # weight_dtype- self.wdt,- # scale_dtype- self.sdt,- self.asym,- )-- # free mem- self.qzeros = torch.empty(0)- self.scales = torch.empty(0)- if self.bias is not None:- self.bias = self.bias.to(self.torch_dt)- else:- self.bias = torch.empty(0)-- def forward(self, x: torch.Tensor):- raw_input_dtype = x.dtype- x = x.to(self.torch_dt)- out_shape = x.shape[:-1] + (self.outfeatures,)- x = x.view(-1, x.shape[-1])- self.bias = self.bias.to(self.torch_dt)- outputs = ark.woqgemm(- x, # convert xd to 2d,- self.qweight,- self.bias,- self.outfeatures,- self.infeatures,- self.group_size,- self.cdt, # compute_dtype- self.wdt, # weight_dtype- self.sdt, # scale_dtype- self.asym,- )- return outputs.to(raw_input_dtype).view(out_shape)---class QuantLinearGPTQ(QuantLinear):- QUANT_TYPE = "ark_gptq"- ZP_BIAS = 1---class QuantLinearAWQ(QuantLinear):- QUANT_TYPE = "ark_awq"--[email protected]_grad()-def unpack_to_8bit_signed(qweight, qzeros, bits, gptq_bias, asym):- wf = torch.tensor(list(range(0, 32, bits)), dtype=torch.int32, device=qweight.device).unsqueeze(0)- zeros = None- if asym:- zp_shape = list(qzeros.shape)- zp_shape[1] = zp_shape[1] * (32 // bits)-- zeros = torch.bitwise_right_shift(torch.unsqueeze(qzeros, 2).expand(-1, -1, 32 // bits), wf.unsqueeze(0)).to(- torch.int16 if bits == 8 else torch.int8- )- torch.bitwise_and(zeros, (2**bits) - 1, out=zeros)- if bits == 8:- zeros = zeros.to(torch.uint8)- zeros += gptq_bias- try:- zeros = zeros.reshape(zp_shape)- except:- # zeros and scales have different iteam numbers.- # remove 1 (due to 0 + 1 in line 252)- zeros = zeros[zeros != 1]- zeros = zeros.reshape(zp_shape)- weight = torch.bitwise_right_shift(torch.unsqueeze(qweight, 1).expand(-1, 32 // bits, -1), wf.unsqueeze(-1)).to(- torch.int16 if bits == 8 else torch.int8- )- weight.bitwise_and_((2**bits) - 1)- weight = weight.view(-1, weight.shape[-1])-- return weight, zeros---# Copied from qlinear_marlin.py[email protected]_grad()-def dequantize_weight(qweight, qzeros, scales, bits):- unpacked_qweight, unpacked_qzeros = unpack_to_8bit_signed(qweight, qzeros, bits)- group_size = unpacked_qweight.shape[0] // scales.shape[0]- scales = scales.repeat_interleave(group_size, dim=0)- if unpacked_qzeros is not None:- unpacked_qzeros = unpacked_qzeros.repeat_interleave(group_size, dim=0)- else:- unpacked_qzeros = torch.full_like(scales, 8 if bits == 4 else 128, dtype=torch.int32, device=qweight.device)- unpacked_qweight = (unpacked_qweight - unpacked_qzeros) * scales-- return unpacked_qweight, unpacked_qzeros---def ark_post_init(model):- for _, submodule in model.named_modules():- if isinstance(submodule, QuantLinear):- submodule.post_init()-- return model---__all__ = ["QuantLinear", "QuantLinearGPTQ", "QuantLinearAWQ"]diff --git a/auto_round_extension/ark/requirements.txt b/auto_round_extension/ark/requirements.txtnew file mode 100644index 000000000..0ce522987--- /dev/null+++ b/auto_round_extension/ark/requirements.txt@@ -0,0 +1,3 @@+torch>=2.10.0+dpcpp-cpp-rt~=2025.3.0+onednn~=2025.3.0; sys_platform=='linux'\ No newline at end of filediff --git a/auto_round_extension/ark/setup.py b/auto_round_extension/ark/setup.pynew file mode 100644index 000000000..f862c0e3a--- /dev/null+++ b/auto_round_extension/ark/setup.py@@ -0,0 +1,226 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++import ctypes+import os+import re+import subprocess+import sys+from pathlib import Path++from setuptools import Extension, find_packages, setup+from setuptools.command.build_ext import build_ext+from setuptools.command.build_py import build_py++build_mode = os.environ.get("BUILD_MODE", "dev").lower()+try:+ file_path = "./auto_round_kernel/version.py"+ with open(file_path) as version_file:+ (__version__,) = re.findall('__version__ = "(.*)"', version_file.read())+except Exception as error:+ assert False, f"Failed to read version from {file_path}: {error}"+++def get_build_version():+ if os.path.exists("PKG-INFO"):+ with open("PKG-INFO", encoding="utf-8") as f:+ for line in f:+ if line.startswith("Version:"):+ return line.split(":", 1)[1].strip()++ if build_mode == "release":+ return __version__+ try:+ result = subprocess.run(["git", "describe", "--tags"], capture_output=True, text=True, check=True)+ distance = result.stdout.strip().split("-")[-2]+ commit = result.stdout.strip().split("-")[-1]+ return f"{__version__}.dev{distance}+{commit}"+ except subprocess.CalledProcessError:+ return __version__+++def fetch_requirements(path):+ requirements = []+ with open(path, "r") as fd:+ requirements = [r.strip() for r in fd]+ return requirements+++def parse_major_minor(version_str):+ major, minor = version_str.split(".")[:2]+ return int(major), int(minor)+++def detect_oneapi_version():+ """+ Auto-detect the sourced oneAPI version using the icx compiler or environment variables.+ Returns a string like '2025.3' or None if detection fails.+ """+ try:+ result = subprocess.run(+ ["icx", "--version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True+ )+ match = re.search(r"Compiler\s+(\d{4}\.\d+)", result.stdout)+ if match:+ return match.group(1)+ except (FileNotFoundError, subprocess.CalledProcessError):+ pass+ cmplr_root = os.environ.get("CMPLR_ROOT", "")+ match = re.search(r"compiler[/\\](\d{4}\.\d+)", cmplr_root, re.IGNORECASE)+ if match:+ return match.group(1)++ return None+++oneapi_version = detect_oneapi_version()+print(f"oneapi_version detected: {oneapi_version}")++if not oneapi_version:+ raise RuntimeError(+ "Failed to auto-detect oneAPI version. "+ "Please ensure you have sourced the oneAPI environment (e.g., source /opt/intel/oneapi/setvars.sh) "+ "and that the 'icx' compiler is in your PATH."+ )++requirements = fetch_requirements("requirements.txt")+enable_sycl_tla = parse_major_minor(oneapi_version) >= (2025, 3)+++def get_system_memory_gb():+ if hasattr(os, "sysconf"):+ page_size_names = ("SC_PAGE_SIZE", "SC_PAGESIZE")+ page_size = None+ for name in page_size_names:+ if name in os.sysconf_names:+ page_size = os.sysconf(name)+ break+ if page_size is not None and "SC_PHYS_PAGES" in os.sysconf_names:+ phys_pages = os.sysconf("SC_PHYS_PAGES")+ if phys_pages > 0:+ return (page_size * phys_pages) / (1024**3)++ if sys.platform == "win32":++ class MEMORYSTATUSEX(ctypes.Structure):+ _fields_ = [+ ("dwLength", ctypes.c_ulong),+ ("dwMemoryLoad", ctypes.c_ulong),+ ("ullTotalPhys", ctypes.c_ulonglong),+ ("ullAvailPhys", ctypes.c_ulonglong),+ ("ullTotalPageFile", ctypes.c_ulonglong),+ ("ullAvailPageFile", ctypes.c_ulonglong),+ ("ullTotalVirtual", ctypes.c_ulonglong),+ ("ullAvailVirtual", ctypes.c_ulonglong),+ ("ullAvailExtendedVirtual", ctypes.c_ulonglong),+ ]++ memory_status = MEMORYSTATUSEX()+ memory_status.dwLength = ctypes.sizeof(MEMORYSTATUSEX)+ if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(memory_status)):+ return memory_status.ullTotalPhys / (1024**3)++ return 64+++def get_sycl_tla_job_count(cpu_job_count):+ memory_gb = get_system_memory_gb()+ memory_based_jobs = max(+ 1, int(memory_gb // 16)+ ) # about 5GB/job for SYCL TLA build, use at most 5/16 of total memory to avoid OOM+ return min(cpu_job_count, memory_based_jobs)+++ROOT = Path(__file__).resolve().parent+SRC_DIR = ROOT / "auto_round_kernel"+BUILD_DIR = ROOT / "build"+XBUILD_DIR = ROOT / "xbuild"+++class CMakeBuild(build_ext):+ def run(self):+ cmake_cmd = [+ "cmake",+ "-S",+ str(SRC_DIR),+ "-B",+ str(BUILD_DIR),+ "-DCMAKE_BUILD_TYPE=Release",+ ]+ if sys.platform == "win32":+ cmake_cmd.append("-GNinja")+ subprocess.check_call(cmake_cmd)++ n_job = os.cpu_count() or 2+ n_job = n_job // 2+ subprocess.check_call(["cmake", "--build", str(BUILD_DIR), "-j", str(n_job)])++ ext = "pyd" if sys.platform == "win32" else "so"++ so_files = list(BUILD_DIR.rglob(f"auto_round_kernel*.{ext}"))+ if not so_files:+ raise RuntimeError(f"Can't find auto_round_kernel*.{ext} in '{BUILD_DIR}'")++ target = Path(self.build_lib) / "auto_round_kernel"+ target.mkdir(parents=True, exist_ok=True)+ for so in so_files:+ self.copy_file(str(so), str(target / so.name))++ cmake_cmd = [+ "cmake",+ "-S",+ str(SRC_DIR),+ "-B",+ str(XBUILD_DIR),+ "-DCMAKE_BUILD_TYPE=Release",+ "-DCMAKE_CXX_COMPILER=icx",+ "-DARK_XPU=ON",+ f"-DARK_SYCL_TLA={'ON' if enable_sycl_tla else 'OFF'}",+ ]+ if sys.platform == "win32":+ cmake_cmd.append("-GNinja")+ xpu_n_job = get_sycl_tla_job_count(n_job) if enable_sycl_tla else n_job+ subprocess.check_call(cmake_cmd)+ subprocess.check_call(["cmake", "--build", str(XBUILD_DIR), "-j", str(xpu_n_job)])++ so_files = list(XBUILD_DIR.rglob(f"auto_round_kernel*.{ext}"))+ if not so_files:+ raise RuntimeError(f"Can't find auto_round_kernel*.{ext} in '{XBUILD_DIR}'")++ for so in so_files:+ self.copy_file(str(so), str(target / so.name))+++class BuildPyThenCMake(build_py):+ def run(self):+ self.run_command("build_ext")+ super().run()+++ext_modules = [Extension("auto_round_kernel.auto_round_kernel", sources=[])]++setup(+ name="auto-round-lib",+ version=get_build_version(),+ description="Auto Round Kernel binary package",+ author_email="[email protected]",+ long_description=open("README.md", "r", encoding="utf-8").read(),+ long_description_content_type="text/markdown",+ keywords="quantization,auto-around,LLM,kernel",+ license="Apache 2.0",+ url="https://github.com/intel/auto-round/auto_round_extension/ark",+ packages=["auto_round_kernel"],+ include_package_data=False,+ ext_modules=ext_modules,+ cmdclass={+ "build_ext": CMakeBuild,+ "build_py": BuildPyThenCMake,+ },+ install_requires=requirements,+ classifiers=[+ "Intended Audience :: Science/Research",+ "Programming Language :: Python :: 3",+ "Topic :: Scientific/Engineering :: Artificial Intelligence",+ "License :: OSI Approved :: Apache Software License",+ ],+)diff --git a/auto_round_extension/ark/test/test_bench_bmg.py b/auto_round_extension/ark/test/test_bench_bmg.pynew file mode 100644index 000000000..e4bddd382--- /dev/null+++ b/auto_round_extension/ark/test/test_bench_bmg.py@@ -0,0 +1,290 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++"""Benchmark our FP16 SDPA and INT8 sage_dynquant against reference sage_attention_page on BMG.++Configurations extracted from the reference performance table:+ batch=1, q_seq=4096, k_seq in {8192, 16384, 32768},+ q_head in {16, 32, 64, 80, 96, 128}, kv_head=8,+ head_dim in {64, 128}, block_size=512, dtype=float16, prefill (non-causal)+"""++import csv+import math+import sys+import time++import auto_round_kernel+import torch++ark = None+++def get_ark():+ global ark+ if ark is None:+ ark = auto_round_kernel.ARK()+ return ark+++def bench(fn, n_warmup=50, n_runs=100):+ for _ in range(n_warmup):+ fn()+ torch.xpu.synchronize()+ st = time.time()+ for _ in range(n_runs):+ fn()+ torch.xpu.synchronize()+ return (time.time() - st) / n_runs+++# Reference numbers from the other team's sage_attention_page benchmark (latency in us)+REF_LATENCY = {+ # (q_head, head_dim, k_seq): latency_us+ (96, 128, 8192): 15959.507,+ (96, 128, 16384): 37002.037,+ (96, 128, 32768): 81393.724,+ (96, 64, 8192): 12055.134,+ (96, 64, 16384): 28907.408,+ (96, 64, 32768): 63472.798,+ (80, 128, 8192): 13501.044,+ (80, 128, 16384): 34071.083,+ (80, 128, 32768): 68894.799,+ (80, 64, 8192): 10320.806,+ (80, 64, 16384): 24004.720,+ (80, 64, 32768): 52323.563,+ (128, 128, 8192): 20964.512,+ (128, 128, 16384): 49097.406,+ (128, 128, 32768): 105522.442,+ (128, 64, 8192): 15976.130,+ (128, 64, 16384): 38390.276,+ (128, 64, 32768): 85014.886,+ (64, 128, 8192): 10555.244,+ (64, 128, 16384): 24920.573,+ (64, 128, 32768): 53484.178,+ (64, 64, 8192): 8117.160,+ (64, 64, 16384): 19467.069,+ (64, 64, 32768): 42681.761,+ (32, 128, 8192): 5704.147,+ (32, 128, 16384): 12386.491,+ (32, 128, 32768): 27239.501,+ (32, 64, 8192): 4081.240,+ (32, 64, 16384): 9875.475,+ (32, 64, 32768): 22412.057,+ (16, 128, 8192): 2874.092,+ (16, 128, 16384): 6634.709,+ (16, 128, 32768): 14326.504,+ (16, 64, 8192): 2246.905,+ (16, 64, 16384): 5303.178,+ (16, 64, 32768): 11535.505,+}+++# Arc Pro B60: 160 XVEs × 512 FP16 OPs/XVE/clock × 2.4 GHz ≈ 196.6 TFLOPS+# Cross-validated with reference table: 78 TFLOPS / 40% MFU ≈ 195 TFLOPS+BMG_PEAK_TFLOPS_INT8 = 196.6+BMG_PEAK_TFLOPS_FP16 = BMG_PEAK_TFLOPS_INT8 / 2++# All configurations from the table+CONFIGS = [+ # (batch, q_seq, k_seq, q_head, kv_head, head_dim, block_size)+ (1, 4096, 8192, 96, 8, 128, 128),+ (1, 4096, 16384, 96, 8, 128, 128),+ (1, 4096, 32768, 96, 8, 128, 128),+ (1, 4096, 8192, 96, 8, 64, 128),+ (1, 4096, 16384, 96, 8, 64, 128),+ (1, 4096, 32768, 96, 8, 64, 128),+ (1, 4096, 8192, 80, 8, 128, 128),+ (1, 4096, 16384, 80, 8, 128, 128),+ (1, 4096, 32768, 80, 8, 128, 128),+ (1, 4096, 8192, 80, 8, 64, 128),+ (1, 4096, 16384, 80, 8, 64, 128),+ (1, 4096, 32768, 80, 8, 64, 128),+ (1, 4096, 8192, 128, 8, 128, 128),+ (1, 4096, 16384, 128, 8, 128, 128),+ (1, 4096, 32768, 128, 8, 128, 128),+ (1, 4096, 8192, 128, 8, 64, 128),+ (1, 4096, 16384, 128, 8, 64, 128),+ (1, 4096, 32768, 128, 8, 64, 128),+ (1, 4096, 8192, 64, 8, 128, 128),+ (1, 4096, 16384, 64, 8, 128, 128),+ (1, 4096, 32768, 64, 8, 128, 128),+ (1, 4096, 8192, 64, 8, 64, 128),+ (1, 4096, 16384, 64, 8, 64, 128),+ (1, 4096, 32768, 64, 8, 64, 128),+ (1, 4096, 8192, 32, 8, 128, 128),+ (1, 4096, 16384, 32, 8, 128, 128),+ (1, 4096, 32768, 32, 8, 128, 128),+ (1, 4096, 8192, 32, 8, 64, 128),+ (1, 4096, 16384, 32, 8, 64, 128),+ (1, 4096, 32768, 32, 8, 64, 128),+ (1, 4096, 8192, 16, 8, 128, 128),+ (1, 4096, 16384, 16, 8, 128, 128),+ (1, 4096, 32768, 16, 8, 128, 128),+ (1, 4096, 8192, 16, 8, 64, 128),+ (1, 4096, 16384, 16, 8, 64, 128),+ (1, 4096, 32768, 16, 8, 64, 128),+]+++def compute_flops(batch, q_seq, k_seq, q_head, kv_head, head_dim):+ """Compute total FLOPs for attention (Q*K + softmax + S*V)."""+ group = q_head // kv_head+ flops = group * kv_head * q_seq * head_dim * k_seq * 2 # Q*K^T+ flops += group * kv_head * q_seq * k_seq * 2 # softmax+ flops += group * kv_head * q_seq * head_dim * k_seq * 2 # S*V+ flops *= batch++ return flops+++def compute_mem_bytes(batch, q_seq, k_seq, q_head, kv_head, head_dim, elem_size=2):+ """Compute read+write bytes (Q + K + V + O)."""+ read_bytes = q_head * q_seq * head_dim * elem_size # Q+ read_bytes += kv_head * k_seq * head_dim * elem_size # K+ read_bytes += kv_head * k_seq * head_dim * elem_size # V+ write_bytes = q_head * q_seq * head_dim * elem_size # O+ read_bytes *= batch+ write_bytes *= batch+ return read_bytes, write_bytes+++def run_one(batch, q_seq, k_seq, q_head, kv_head, head_dim, block_size, dt=torch.float16, is_causal=False, dev="xpu"):+ """Run both FP16 SDPA and INT8 sage_dynquant for one configuration."""+ scale = 1.0 / math.sqrt(head_dim)+ q = torch.randn(batch, q_head, q_seq, head_dim, dtype=dt, device=dev)+ k = torch.randn(batch, kv_head, k_seq, head_dim, dtype=dt, device=dev)+ v = torch.randn(batch, kv_head, k_seq, head_dim, dtype=dt, device=dev)++ # --- FP16 SDPA (our kernel) ---+ dur_fp16 = bench(lambda: get_ark().sdpa(q, k, v, scale=scale, is_causal=is_causal))+ fp16_us = dur_fp16 * 1e6++ # --- INT8 sage_dynquant (our kernel) ---+ dur_i8 = bench(lambda: get_ark().sagev1(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=block_size))+ i8_us = dur_i8 * 1e6++ # Correctness: INT8 vs PyTorch FP16+ out_i8 = get_ark().sagev1(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=block_size)+ ref = torch.nn.functional.scaled_dot_product_attention(+ q, k, v, scale=scale, is_causal=is_causal, enable_gqa=(q_head // kv_head) > 1+ )+ dff = (ref - out_i8).abs()++ # Metrics+ flops = compute_flops(batch, q_seq, k_seq, q_head, kv_head, head_dim)+ ratio = (+ 1 if not is_causal else ((k_seq - q_seq) + q_seq / 2) / k_seq+ ) # Approximate ratio of effective k_seq for causal attention+ flops = flops * ratio # Adjust FLOPs for causal case where effective k_seq is smaller+ fp16_tflops = flops / dur_fp16 / 1e12 if dur_fp16 > 0 else 0+ i8_tflops = flops / dur_i8 / 1e12 if dur_i8 > 0 else 0+ fp16_mfu = fp16_tflops / BMG_PEAK_TFLOPS_FP16+ i8_mfu = i8_tflops / BMG_PEAK_TFLOPS_INT8++ # Reference latency from the other team+ ref_us = REF_LATENCY.get((q_head, head_dim, k_seq), None)++ return {+ "q_head_num": q_head,+ "kv_head_num": kv_head,+ "head_dim": head_dim,+ "batch_size": batch,+ "q_seq_len": q_seq,+ "k_seq_len": k_seq,+ "block_size": block_size,+ "ref_us": ref_us,+ "fp16_us": fp16_us,+ "fp16_tflops": fp16_tflops,+ "fp16_mfu": fp16_mfu,+ "fp16_vs_ref": fp16_us / ref_us if ref_us else None,+ "i8_us": i8_us,+ "i8_tflops": i8_tflops,+ "i8_mfu": i8_mfu,+ "i8_vs_ref": i8_us / ref_us if ref_us else None,+ "i8_vs_fp16": i8_us / fp16_us if fp16_us > 0 else None,+ "diff_max": dff.max().item(),+ "diff_mean": dff.mean().item(),+ }+++def main():+ dev = "xpu"+ dt = torch.float16+ is_causal = True++ print("=" * 160)+ print("BMG Benchmark: Our FP16 SDPA vs Our INT8 sage_dynquant vs Reference sage_attention_page")+ print(f"dtype={dt}, causal={is_causal}, device={dev}")+ print("=" * 160)++ header = (+ f"{'Hq':>4s} {'Hkv':>3s} {'D':>4s} {'Sq':>5s} {'Skv':>6s} "+ f"{'Ref(us)':>10s} "+ f"{'FP16(us)':>10s} {'TFLOPS':>7s} {'MFU':>5s} {'vs Ref':>7s} "+ f"{'INT8(us)':>10s} {'TFLOPS':>7s} {'MFU':>5s} {'vs Ref':>7s} {'vs FP16':>7s} "+ f"{'DiffMax':>9s} {'DiffMean':>9s}"+ )+ print(header)+ print("-" * 160)++ results = []+ for batch, q_seq, k_seq, q_head, kv_head, head_dim, block_size in CONFIGS:+ r = run_one(batch, q_seq, k_seq, q_head, kv_head, head_dim, block_size, dt=dt, is_causal=is_causal, dev=dev)+ results.append(r)++ ref_str = f"{r['ref_us']:.0f}" if r["ref_us"] else "N/A"+ fp16_vs_ref = f"{r['fp16_vs_ref']:.2f}x" if r["fp16_vs_ref"] else "N/A"+ i8_vs_ref = f"{r['i8_vs_ref']:.2f}x" if r["i8_vs_ref"] else "N/A"+ i8_vs_fp16 = f"{r['i8_vs_fp16']:.2f}x" if r["i8_vs_fp16"] else "N/A"++ print(+ f"{r['q_head_num']:4d} {r['kv_head_num']:3d} {r['head_dim']:4d} {r['q_seq_len']:5d} {r['k_seq_len']:6d} "+ f"{ref_str:>10s} "+ f"{r['fp16_us']:10.0f} {r['fp16_tflops']:7.1f} {r['fp16_mfu']:4.0%} {fp16_vs_ref:>7s} "+ f"{r['i8_us']:10.0f} {r['i8_tflops']:7.1f} {r['i8_mfu']:4.0%} {i8_vs_ref:>7s} {i8_vs_fp16:>7s} "+ f"{r['diff_max']:9.4f} {r['diff_mean']:9.6f}"+ )++ # Write CSV+ csv_path = f"bench_bmg_comparison_{'causal' if is_causal else 'non_causal'}.csv"+ fieldnames = list(results[0].keys())+ with open(csv_path, "w", newline="") as f:+ writer = csv.DictWriter(f, fieldnames=fieldnames)+ writer.writeheader()+ writer.writerows(results)+ print(f"\nResults saved to {csv_path}")++ # Summary+ print("\n" + "=" * 100)+ print("Summary")+ print("=" * 100)++ fp16_ratios = [r["fp16_vs_ref"] for r in results if r["fp16_vs_ref"]]+ i8_ratios = [r["i8_vs_ref"] for r in results if r["i8_vs_ref"]]+ i8_fp16_ratios = [r["i8_vs_fp16"] for r in results if r["i8_vs_fp16"]]++ if fp16_ratios:+ faster_fp16 = sum(1 for x in fp16_ratios if x < 1.0)+ print("\n Our FP16 SDPA vs Reference sage_attention_page:")+ print(f" Faster configs: {faster_fp16}/{len(fp16_ratios)}")+ print(f" Avg ratio: {sum(fp16_ratios)/len(fp16_ratios):.3f}x")+ print(f" Best: {min(fp16_ratios):.3f}x Worst: {max(fp16_ratios):.3f}x")++ if i8_ratios:+ faster_i8 = sum(1 for x in i8_ratios if x < 1.0)+ print("\n Our INT8 sage_dynquant vs Reference sage_attention_page:")+ print(f" Faster configs: {faster_i8}/{len(i8_ratios)}")+ print(f" Avg ratio: {sum(i8_ratios)/len(i8_ratios):.3f}x")+ print(f" Best: {min(i8_ratios):.3f}x Worst: {max(i8_ratios):.3f}x")++ if i8_fp16_ratios:+ faster_i8_fp16 = sum(1 for x in i8_fp16_ratios if x < 1.0)+ print("\n Our INT8 vs Our FP16:")+ print(f" INT8 faster configs: {faster_i8_fp16}/{len(i8_fp16_ratios)}")+ print(f" Avg ratio: {sum(i8_fp16_ratios)/len(i8_fp16_ratios):.3f}x")+ print(f" Best: {min(i8_fp16_ratios):.3f}x Worst: {max(i8_fp16_ratios):.3f}x")+++if __name__ == "__main__":+ main()diff --git a/auto_round_extension/ark/test/test_flash_attn.py b/auto_round_extension/ark/test/test_flash_attn.pynew file mode 100644index 000000000..ade89ee22--- /dev/null+++ b/auto_round_extension/ark/test/test_flash_attn.py@@ -0,0 +1,140 @@+#!/usr/bin/env python+# -*- coding: utf-8 -*-+#+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""+Unit tests for Flash Attention kernel.++This tests the ARK Flash Attention kernel at the unit test level+"""++import math++import auto_round_kernel+import pytest+import torch++ark = auto_round_kernel.ARK()+++def is_xpu_available():+ return hasattr(torch, "xpu") and torch.xpu.is_available()+++def has_flash_attn():+ """Check if Flash Attention kernel is available."""+ if ark.xpu_lib is None:+ return False+ return hasattr(ark.xpu_lib, "sdpa")+++def reference_attention(Q, K, V, scale, is_causal=True):+ """Reference scaled dot-product attention implementation."""+ # Q: [batch, num_heads, seq_q, head_dim]+ # K: [batch, num_heads, seq_kv, head_dim]+ # V: [batch, num_heads, seq_kv, head_dim]++ # Compute attention scores: [batch, num_heads, seq_q, seq_kv]+ ref = torch.nn.functional.scaled_dot_product_attention(+ Q,+ K,+ V,+ scale=scale,+ attn_mask=None,+ is_causal=is_causal,+ enable_gqa=True if K.shape[1] != Q.shape[1] else False,+ )+ return ref++[email protected](not is_xpu_available(), reason="XPU not available")[email protected](not has_flash_attn(), reason="Flash Attention kernel not built (need ARK_SYCL_TLA=ON)")+class TestSDPA:+ """Unit tests for Flash Attention Prefill kernel."""++ def sdpa_basic(self, batch, num_heads, num_headskv, seq_len, seq_lenkv, head_dim, dtype, is_causal):++ Q = torch.randn(batch, num_heads, seq_len, head_dim, dtype=dtype, device="xpu") - 0.5+ K = torch.randn(batch, num_headskv, seq_lenkv, head_dim, dtype=dtype, device="xpu") - 0.5+ V = torch.randn(batch, num_headskv, seq_lenkv, head_dim, dtype=dtype, device="xpu") - 0.5++ scale = 1.0 / math.sqrt(head_dim)++ output = ark.sdpa(Q, K, V, scale=scale, is_causal=is_causal)+ ref_output = reference_attention(Q, K, V, scale, is_causal=is_causal)+ torch.testing.assert_close(output.to(ref_output.dtype), ref_output, rtol=1e-2, atol=1e-2)++ @pytest.mark.parametrize("num_heads_q, num_heads_kv", [(32, 8), (64, 8), (64, 1)])+ def test_prefill_gqa(self, num_heads_q, num_heads_kv):+ """Test prefill with Grouped Query Attention (GQA)."""+ batch = 1+ seq_len = 512+ head_dim = 128+ dtype = torch.float16+ self.sdpa_basic(batch, num_heads_q, num_heads_kv, seq_len, seq_len, head_dim, dtype=dtype, is_causal=True)++ @pytest.mark.parametrize("seq_len", [33, 100, 1000])+ def test_prefill_various_seq_lengths(self, seq_len):+ """Test prefill with various sequence lengths."""+ batch = 1+ num_heads = 8+ head_dim = 64+ dtype = torch.float16+ self.sdpa_basic(batch, num_heads, num_heads, seq_len, seq_len, head_dim, dtype=dtype, is_causal=True)++ @pytest.mark.parametrize("head_dim", [64, 128, 96, 192])+ def test_prefill_various_head_dims(self, head_dim):+ """Test prefill with various head dimensions."""+ batch = 1+ num_heads = 8+ seq_len = 64+ dtype = torch.float16+ self.sdpa_basic(batch, num_heads, num_heads, seq_len, seq_len, head_dim, dtype=dtype, is_causal=True)++ def test_prefill_non_causal(self):+ """Test prefill without causal mask."""+ batch = 1+ num_heads = 8+ seq_len = 64+ head_dim = 64+ dtype = torch.float16+ self.sdpa_basic(batch, num_heads, num_heads, seq_len, seq_len, head_dim, dtype=dtype, is_causal=False)++ @pytest.mark.parametrize("head_dim", [64, 128, 96, 192])+ def test_decode_various_head_dims(self, head_dim):+ """Test decode with various head dimensions."""+ batch = 1+ num_heads = 32+ seq_len = 512+ dtype = torch.float16+ self.sdpa_basic(batch, num_heads, num_heads, 1, seq_len, head_dim, dtype=dtype, is_causal=False)++ @pytest.mark.parametrize("batch", [64, 128, 96, 192])+ def test_decode_various_batches(self, batch):+ """Test decode with various batch sizes."""+ num_heads = 32+ seq_len = 512+ head_dim = 128+ dtype = torch.float16+ self.sdpa_basic(batch, num_heads, num_heads, 1, seq_len, head_dim, dtype=dtype, is_causal=False)+++if __name__ == "__main__":+ import pathlib++ test_file = pathlib.Path(__file__).resolve()+ ark_root = test_file.parent.parent+ pytest.main([str(test_file), "-v", "--confcutdir", str(test_file.parent)])diff --git a/auto_round_extension/ark/test/test_matmul.py b/auto_round_extension/ark/test/test_matmul.pynew file mode 100644index 000000000..6debd7685--- /dev/null+++ b/auto_round_extension/ark/test/test_matmul.py@@ -0,0 +1,225 @@+#!/usr/bin/env python+# -*- coding: utf-8 -*-+#+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import os+import time++import auto_round_kernel+import pytest+import torch++ark = auto_round_kernel.ARK()+++def main_op(m, k, n, dt, batch_size, runs, has_bias, record_property, device):+ print(f"\n m={m}, k={k}, n={n}, dt={dt}, batch={batch_size}")+ torch.manual_seed(0)+ cdt = dt+ if device == "cpu":+ cpu_cores = os.cpu_count()+ print(f"cpu_cores: {cpu_cores}", f"torch threads: {torch.get_num_threads()}")+ cdt = torch.float32+ try:+ activation = torch.rand(m, k, dtype=dt, device=device) - 0.5+ wei = torch.rand(n, k, dtype=dt, device=device) - 0.5+ bias = torch.rand(1, n, dtype=cdt, device=device) if has_bias else torch.Tensor()++ act_set = activation.unsqueeze(0).repeat(batch_size, 1, 1)+ wei_set = wei.unsqueeze(0).repeat(batch_size, 1, 1)++ except RuntimeError as e:+ if record_property:+ pytest.fail(f"OOM during basic init: {e}")+ else:+ print(f"OOM during basic init: {e}")+ return++ tar_dst = ark.matmul(activation, wei, bias)+ ref_dst = torch.matmul(activation, wei.T).to(cdt)+ if has_bias:+ ref_dst = ref_dst + bias+ diff = abs(tar_dst - ref_dst)+ # print(f" abs(ref_dst).max(): {abs(ref_dst).max()}, abs(ref_dst).mean(): {abs(ref_dst).mean()}, abs(tar_dst).max(): {abs(tar_dst).max()}, abs(tar_dst).mean(): {abs(tar_dst).mean()}")+ print(f" Max Diff: {diff.max().item():.6f}, Mean Diff: {diff.mean().item():.6f}")+ if dt == torch.float32:+ atol = 0.001+ rtol = 0.03+ if dt == torch.float16:+ atol = 0.1+ rtol = 0.06+ if dt == torch.bfloat16:+ atol = 0.6+ rtol = 0.1+ assert torch.allclose(tar_dst, ref_dst, atol=atol, rtol=rtol), "Verification Failed!"+ print(" -> Verification PASS")++ try:+ act_set = activation.unsqueeze(0).repeat(batch_size, 1, 1)+ wei_set = wei.unsqueeze(0).repeat(batch_size, 1, 1)+ dst_set = tar_dst.unsqueeze(0).repeat(batch_size, 1, 1)+ except RuntimeError as e:+ if record_property:+ pytest.fail(f"OOM during basic init: {e}")+ else:+ print(f"OOM during basic init: {e}")+ return++ for i in range(runs):+ idx = i % batch_size+ tar_dst = ark.matmul(act_set[idx], wei_set[idx], bias)+ if device == "xpu":+ torch.xpu.synchronize()++ st = time.perf_counter()+ for i in range(runs):+ idx = i % batch_size+ tar_dst = ark.matmul(act_set[idx], wei_set[idx], bias)+ if device == "xpu":+ torch.xpu.synchronize()+ dur = (time.perf_counter() - st) / runs++ # GFLOPS+ ops = m * n * k * 2+ gflops = ops / dur / 1e9++ # Bandwidth+ elem_size = activation.element_size()+ total_bytes = (m * k + k * n + m * n) * elem_size+ bandwidth = total_bytes / dur / 1e9++ if record_property:+ record_property("Time_ms", round(dur * 1000, 4))+ record_property("GFLOPS", round(gflops, 2))+ record_property("Bandwidth_GBs", round(bandwidth, 2))++ print(f"[Performance] Time: {dur*1000:.4f} ms")+ print(f" GFLOPS: {gflops:.2f}")+ print(f" Bandwidth: {bandwidth:.2f} GB/s")+++def woqgemm(m, k, n, dt, batch_size, runs, record_property, device):+ print(f"\n m={m}, k={k}, n={n}, dt={dt}, batch={batch_size}")+ torch.manual_seed(0)++ A = torch.rand(batch_size, m, k, dtype=dt, device=device) - 0.5+ bias = torch.rand(1, n, dtype=dt, device=device) + 2+ B = torch.randint(-128, 127, (batch_size, n, k), dtype=torch.int8, device=device)+ scaleB = torch.rand(n, 1, dtype=dt, device=device) / 100+ C = ark.woqgemm_s8(A[0], B[0], scaleB, bias)++ maxA = abs(A[0]).max()+ scaleA = maxA / 127+ ratio = (1 / scaleA).to(dt)+ qA = torch.round(A[0] * ratio).to(torch.int8)+ DA = qA.to(dt) * scaleA+ DB = B[0].to(dt) * scaleB+ ref = torch.matmul(DA, DB.T)+ ref = ref + bias+ dff = abs(C - ref)+ print(f" Max Diff: {dff.max().item():.6f}, Mean Diff: {dff.mean().item():.6f}")+ # QDQ tolerance+ assert torch.allclose(C, ref, atol=1, rtol=0.1), "Verification Failed!"+ print(" -> Verification PASS")+ for i in range(runs):+ idx = i % batch_size+ tar_dst = ark.woqgemm_s8(A[idx], B[idx], scaleB, bias)+ if device == "xpu":+ torch.xpu.synchronize()++ st = time.perf_counter()+ for i in range(runs):+ idx = i % batch_size+ tar_dst = ark.woqgemm_s8(A[idx], B[idx], scaleB, bias)+ if device == "xpu":+ torch.xpu.synchronize()++ dur = (time.perf_counter() - st) / runs++ # GFLOPS+ ops = m * n * k * 2+ gflops = ops / dur / 1e9++ # Bandwidth+ elem_size = A.element_size()+ total_bytes = (m * k + m * n) * elem_size + k * n+ bandwidth = total_bytes / dur / 1e9++ if record_property:+ record_property("Time_ms", round(dur * 1000, 4))+ record_property("GFLOPS", round(gflops, 2))+ record_property("Bandwidth_GBs", round(bandwidth, 2))++ print(f"[Performance] Time: {dur*1000:.4f} ms")+ print(f" GFLOPS: {gflops:.2f}")+ print(f" Bandwidth: {bandwidth:.2f} GB/s")++[email protected]("m", [4096])[email protected]("k, n", [(4096, 4096)])[email protected]("dt", [torch.float16, torch.float32])[email protected]("batch_size", [1])[email protected]("runs", [1])+def test_woqgemm_xpu(m, k, n, dt, batch_size, runs, record_property):+ if not torch.xpu.is_available():+ pytest.skip("No XPU Device")+ with torch.no_grad():+ woqgemm(m, k, n, dt, batch_size, runs, record_property, "xpu")+ torch.xpu.empty_cache()++[email protected]("m", [1, 8, 16, 32, 64, 128, 256, 1024])[email protected]("k, n", [(4096, 4096)])[email protected]("dt", [torch.float32, torch.float16, torch.bfloat16])[email protected]("batch_size", [1])[email protected]("runs", [1])+def test_xpu(m, k, n, dt, batch_size, runs, record_property):+ if not torch.xpu.is_available():+ pytest.skip("No XPU Device")+ with torch.no_grad():+ main_op(m, k, n, dt, batch_size, runs, True, record_property, "xpu")+ torch.xpu.empty_cache()++[email protected]("m", [1, 8, 16, 32, 64, 128, 256, 1024])[email protected]("k, n", [(4096, 4096)])[email protected]("dt", [torch.float32])[email protected]("batch_size", [1])[email protected]("runs", [1])+def test_cpu(m, k, n, dt, batch_size, runs, record_property):+ main_op(m, k, n, dt, batch_size, runs, True, record_property, "cpu")+++# LOCAL_TEST: python test_matmul.py+# CI pytest: pytest -v test_matmul.py -k 'test_xpu'+# pytest -s -v test_matmul.py -k 'test_xpu' --csv results_matmul_all.csv --csv-columns file,function,parameters_as_columns,properties_as_columns+if __name__ == "__main__":++ def mock_record(key, value):+ pass++ # woqgemm(512, 4096, 4096, torch.float16, batch_size=2, runs=10, record_property=mock_record, device='cpu')+ # woqgemm(512, 4096, 4096, torch.float32, batch_size=2, runs=10, record_property=mock_record, device='cpu')++ test_cpu(1024, 4096, 4096, torch.float32, batch_size=2, runs=10, record_property=mock_record)+ # test_cpu(1024, 4096, 4096, torch.float16, batch_size=2, runs=10, record_property=mock_record)+ # test_cpu(1024, 4096, 4096, torch.bfloat16, batch_size=2, runs=10, record_property=mock_record)++ woqgemm(4096, 4096, 4096, torch.float16, batch_size=2, runs=100, record_property=mock_record, device="xpu")+ woqgemm(4096, 4096, 4096, torch.float32, batch_size=2, runs=100, record_property=mock_record, device="xpu")+ test_xpu(4096, 4096, 8192, torch.float32, batch_size=2, runs=100, record_property=mock_record)+ test_xpu(4096, 4096, 8192, torch.float16, batch_size=2, runs=100, record_property=mock_record)+ test_xpu(4096, 4096, 8192, torch.bfloat16, batch_size=2, runs=100, record_property=mock_record)diff --git a/auto_round_extension/ark/test/test_moe.py b/auto_round_extension/ark/test/test_moe.pynew file mode 100644index 000000000..1134f7b4d--- /dev/null+++ b/auto_round_extension/ark/test/test_moe.py@@ -0,0 +1,170 @@+#!/usr/bin/env python+# -*- coding: utf-8 -*-+#+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""+Unit tests for MOE GEMM kernel.++This tests the ARK MOE GEMM kernel at the unit test level+"""++import auto_round_kernel+import pytest+import torch++ark = auto_round_kernel.ARK()+++def is_xpu_available():+ return hasattr(torch, "xpu") and torch.xpu.is_available()+++def has_moe_gemm():+ """Check if MOE GEMM kernel is available."""+ if ark.xpu_lib is None:+ return False+ return hasattr(ark.xpu_lib, "moe_gemm")++[email protected](not is_xpu_available(), reason="XPU not available")[email protected](not has_moe_gemm(), reason="MOE GEMM kernel not built (need ARK_SYCL_TLA=ON)")+class TestMoEGemm:+ """Unit tests for MOE GEMM kernel."""++ def test_moe_gemm_fp16_basic(self):+ """Test basic MOE GEMM with FP16."""+ num_experts = 8+ total_tokens = 64+ N = 256 # output features+ K = 128 # input features++ # Create test tensors+ # Note: weights layout is [num_experts, K, N] (Row major)+ activations = torch.randn(total_tokens, K, dtype=torch.float16, device="xpu")+ weights = torch.randn(num_experts, K, N, dtype=torch.float16, device="xpu")++ # Distribute tokens evenly among experts+ tokens_per_expert = [total_tokens // num_experts] * num_experts+ num_tokens_per_expert = torch.tensor(tokens_per_expert, dtype=torch.int32, device="xpu")++ # Run MOE GEMM+ output = ark.moe_gemm(activations, weights, num_tokens_per_expert)++ # Verify output shape+ assert output.shape == (total_tokens, N), f"Expected shape {(total_tokens, N)}, got {output.shape}"+ assert output.dtype == torch.float16, f"Expected dtype float16, got {output.dtype}"++ # Compute reference output using PyTorch+ # GEMM: [num_tokens, K] @ [K, N] -> [num_tokens, N]+ ref_output = torch.zeros(total_tokens, N, dtype=torch.float16, device="xpu")+ token_offset = 0+ for expert_idx in range(num_experts):+ num_tokens = tokens_per_expert[expert_idx]+ if num_tokens > 0:+ expert_activations = activations[token_offset : token_offset + num_tokens]+ expert_weight = weights[expert_idx] # [K, N]+ # GEMM: [num_tokens, K] @ [K, N] -> [num_tokens, N]+ ref_output[token_offset : token_offset + num_tokens] = expert_activations @ expert_weight+ token_offset += num_tokens++ # Compare results (allow some tolerance for FP16)+ torch.testing.assert_close(output, ref_output, rtol=1e-2, atol=1e-2)+ print("FP16 MOE GEMM test passed!")++ def test_moe_gemm_uneven_distribution(self):+ """Test MOE GEMM with uneven token distribution."""+ num_experts = 4+ total_tokens = 100+ N = 256+ K = 128++ activations = torch.randn(total_tokens, K, dtype=torch.float16, device="xpu")+ weights = torch.randn(num_experts, K, N, dtype=torch.float16, device="xpu")++ # Uneven distribution: 10, 30, 25, 35 tokens+ tokens_per_expert = [10, 30, 25, 35]+ assert sum(tokens_per_expert) == total_tokens+ num_tokens_per_expert = torch.tensor(tokens_per_expert, dtype=torch.int32, device="xpu")++ output = ark.moe_gemm(activations, weights, num_tokens_per_expert)++ assert output.shape == (total_tokens, N)++ # Reference computation+ ref_output = torch.zeros(total_tokens, N, dtype=torch.float16, device="xpu")+ token_offset = 0+ for expert_idx in range(num_experts):+ num_tokens = tokens_per_expert[expert_idx]+ if num_tokens > 0:+ expert_activations = activations[token_offset : token_offset + num_tokens]+ expert_weight = weights[expert_idx] # [K, N]+ ref_output[token_offset : token_offset + num_tokens] = expert_activations @ expert_weight+ token_offset += num_tokens++ torch.testing.assert_close(output, ref_output, rtol=1e-2, atol=1e-2)+ print("Uneven distribution MOE GEMM test passed!")++ def test_moe_gemm_many_experts(self):+ """Test MOE GEMM with many experts (like DeepSeek models)."""+ num_experts = 64 # DeepSeek uses 64 experts+ total_tokens = 128+ N = 256+ K = 128++ activations = torch.randn(total_tokens, K, dtype=torch.float16, device="xpu")+ weights = torch.randn(num_experts, K, N, dtype=torch.float16, device="xpu")++ # Only activate some experts (sparse activation pattern)+ tokens_per_expert = [0] * num_experts+ # Activate 8 experts with 16 tokens each+ active_experts = [0, 8, 16, 24, 32, 40, 48, 56]+ for i, exp_idx in enumerate(active_experts):+ tokens_per_expert[exp_idx] = 16++ num_tokens_per_expert = torch.tensor(tokens_per_expert, dtype=torch.int32, device="xpu")++ output = ark.moe_gemm(activations, weights, num_tokens_per_expert)++ assert output.shape == (total_tokens, N)+ print(f"Many experts ({num_experts}) MOE GEMM test passed!")++ @pytest.mark.parametrize(+ "N,K",+ [+ (256, 128),+ (512, 256),+ (1024, 512),+ (2048, 1024),+ ],+ )+ def test_moe_gemm_various_sizes(self, N, K):+ """Test MOE GEMM with various matrix sizes."""+ num_experts = 8+ total_tokens = 64++ activations = torch.randn(total_tokens, K, dtype=torch.float16, device="xpu")+ weights = torch.randn(num_experts, K, N, dtype=torch.float16, device="xpu")+ tokens_per_expert = [8] * num_experts+ num_tokens_per_expert = torch.tensor(tokens_per_expert, dtype=torch.int32, device="xpu")++ output = ark.moe_gemm(activations, weights, num_tokens_per_expert)++ assert output.shape == (total_tokens, N)+ print(f"MOE GEMM test passed for N={N}, K={K}")+++if __name__ == "__main__":+ pytest.main([__file__, "-v"])diff --git a/auto_round_extension/ark/test/test_packq.py b/auto_round_extension/ark/test/test_packq.pynew file mode 100644index 000000000..c77d6449c--- /dev/null+++ b/auto_round_extension/ark/test/test_packq.py@@ -0,0 +1,134 @@+#!/usr/bin/env python+# -*- coding: utf-8 -*-+#+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import os++import auto_round_kernel+import pytest+import torch+from ut_utils import compare2, decode_fp8_to_float, gen_weis8, get_scale_torch_dt, get_torch_dt, sample_valid_fp8++ark = auto_round_kernel.ARK()+++def main_op(k, n, weight_type, scale_type, asym, blocksize, device, compute_type):+ print(+ f"\n k={k}, n={n}, blocksize={blocksize}, compute_type={compute_type}, "+ f"weight_type={weight_type}, scale_type={scale_type}, asym={asym}, device={device}"+ )++ torch.manual_seed(0)+ if device == "cpu":+ cpu_cores = os.cpu_count()+ print(f"cpu_cores per numa node: {cpu_cores}")+ # ark.set_threads(cpu_cores)+ if device == "xpu" and compute_type is None:+ compute_type = scale_type+ sdt = get_torch_dt(compute_type)+ if device == "cpu":+ sdt = torch.float32+ scale_dt = get_scale_torch_dt(scale_type)+ if scale_type == "fp8_e8m0":+ # int-valued exponents; actual scale factor is 2**exp+ scale = torch.randint(-8, 8, (k // blocksize, n), dtype=torch.int32, device=device).to(scale_dt)+ scale_re = torch.pow(torch.tensor(2.0, device=device, dtype=torch.float32), scale.to(torch.float32))+ scale_re = scale_re.repeat_interleave(repeats=blocksize, dim=0).to(sdt)+ else:+ scale = torch.randn(k // blocksize, n, dtype=scale_dt, device=device) / 100 + 0.01+ scale_re = scale.repeat_interleave(repeats=blocksize, dim=0).to(sdt)+ odt = sdt+ if scale_dt == "fp8_e8m0":+ odt = torch.float32+ if weight_type.startswith("fp8_"):+ if device == "cpu":+ pytest.skip("FP8 packq test is for XPU")+ if asym:+ pytest.skip("FP8 packq does not support asym in this test")+ raw_qweight = sample_valid_fp8((k, n), weight_type, device)+ decoded = decode_fp8_to_float(raw_qweight.cpu(), weight_type).to(odt).to(device)+ zp = torch.Tensor()+ ref_dst = decoded * scale_re.to(odt)+ qweight_for_pack = raw_qweight+ else:+ if scale_type == "fp8_e8m0":+ pytest.skip("fp8_e8m0 scale is only supported for fp8 weights")+ raw_s8_wei = gen_weis8(weight_type, device, k, n)+ if asym:+ zp = gen_weis8(weight_type, device, k // blocksize, n)+ zp_re = zp.repeat_interleave(repeats=blocksize, dim=0)+ ref_dst = (raw_s8_wei.to(sdt) - zp_re.to(sdt)) * scale_re+ else:+ zp = torch.Tensor()+ ref_dst = raw_s8_wei.to(sdt) * scale_re+ qweight_for_pack = raw_s8_wei++ packw = ark.repack_quantized_weight(+ qweight_for_pack, scale, zp, blocksize, compute_type, weight_type, scale_type, asym+ )+ revert_wei = ark.unpack_weight(packw, odt, n, k, blocksize, compute_type, weight_type, scale_type, asym)+ ref_dst = ref_dst.T+ compare2(revert_wei, ref_dst)+ atol = 0.001+ assert torch.allclose(revert_wei, ref_dst, atol=atol)++[email protected]("k, n", [(1024, 1024), (512, 2048)])[email protected]("blocksize", [16, 32, 128])[email protected]("weight_type", ["int2", "int4", "int8", "fp8_e4m3", "fp8_e5m2"])+ "scale_type, compute_type", [("fp16", "fp16"), ("fp32", "fp32"), ("fp8_e8m0", "fp16"), ("fp8_e8m0", "fp32")]+)[email protected]("asym", [False, True])+def test_xpu(k, n, weight_type, scale_type, asym, blocksize, compute_type):+ if not torch.xpu.is_available():+ pytest.skip("No XPU Device")+ with torch.no_grad():+ main_op(k, n, weight_type, scale_type, asym, blocksize, "xpu", compute_type)+ torch.xpu.empty_cache()++[email protected]("k, n", [(1024, 1024), (512, 2048)])[email protected]("blocksize", [32, 128])[email protected]("weight_type", ["int2", "int4", "int8"])[email protected]("scale_type", ["fp32"])[email protected]("compute_type", ["auto", "int8", "bf16", "fp32"])[email protected]("asym", [True, False])+def test_cpu(k, n, weight_type, scale_type, asym, blocksize, compute_type):+ main_op(k, n, weight_type, scale_type, asym, blocksize, "cpu", compute_type)+++# LOCAL_TEST: python test_packq.py+# CI pytest: pytest -v test_packq.py -k 'test_'+if __name__ == "__main__":+ # main_op(1024, 1024, 'int2', 'fp32', True, 32, 'cpu', 'fp32')+ # main_op(1024, 1024, 'int2', 'fp32', False, 32, 'cpu', 'fp32')+ # main_op(1024, 1024, 'int4', 'fp32', False, 32, 'cpu', 'fp32')+ # main_op(1024, 1024, 'int4', 'fp32', False, 32, 'cpu', 'bf16')+ # main_op(1024, 1024, 'int4', 'fp32', False, 32, 'cpu', 'int8')+ # main_op(1024, 1024, 'int8', 'fp32', False, 128, 'cpu', 'int8')++ # main_op(512, 2048, "int2", "fp32", False, 128, "cpu", "fp32")+ main_op(1024, 1024, "int2", "fp32", True, 32, "xpu", None)+ main_op(1024, 1024, "int4", "fp32", True, 32, "xpu", None)+ # main_op(1024, 1024, "int8", "fp32", False, 32, "xpu", None)+ # main_op(1024, 1024, "int4", "fp16", False, 32, "xpu", None)++ # main_op(1024, 1024, "fp8_e4m3", "fp16", False, 32, "xpu", None)+ # main_op(1024, 1024, "fp8_e4m3", "fp32", False, 32, "xpu", None)+ # main_op(1024, 1024, "fp8_e4m3", "fp8_e8m0", False, 32, "xpu", None)+ # main_op(1024, 1024, "fp8_e5m2", "fp16", False, 32, "xpu", None)+ # main_op(1024, 1024, "fp8_e5m2", "fp32", False, 32, "xpu", None)diff --git a/auto_round_extension/ark/test/test_sage_dynquant.py b/auto_round_extension/ark/test/test_sage_dynquant.pynew file mode 100644index 000000000..54e3e9832--- /dev/null+++ b/auto_round_extension/ark/test/test_sage_dynquant.py@@ -0,0 +1,95 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++"""Benchmark for DynQuant SAGE with block-wise INT8 quantization.++Tests sage_dynquant with different quant_block_size values (per-token, 32, 64, 128, 256).+Compares accuracy and latency against SAGE v1 and FP16 SDPA references.+"""++import math+import time++import auto_round_kernel+import torch++ark = None+++def get_ark():+ global ark+ if ark is None:+ ark = auto_round_kernel.ARK()+ return ark+++def bench(fn, n_warmup=50, n_runs=100):+ """Benchmark helper: returns average time in ms."""+ for _ in range(n_warmup):+ fn()+ torch.xpu.synchronize()+ st = time.time()+ for _ in range(n_runs):+ fn()+ torch.xpu.synchronize()+ return (time.time() - st) / n_runs * 1e3+++def run_benchmark(batch=2, seq=17776, h_q=30, h_kv=30, H=128, H_v=128, dt=torch.float16, is_causal=False, dev="xpu"):+ # Align seq to largest block_size for divisibility+ block_sizes = [1, 32, 64, 128, 256]+ max_bs = max(block_sizes)+ seq = seq // max_bs * max_bs+ q = torch.randn(batch, h_q, seq, H, dtype=dt, device=dev)+ k = torch.randn(batch, h_kv, seq, H, dtype=dt, device=dev)+ v = torch.randn(batch, h_kv, seq, H_v, dtype=dt, device=dev)+ scale = 1 / math.sqrt(H)++ print("=== SageDynQuant Block-wise Benchmark ===")+ print(f"Batch:{batch} Seq:{seq} HeadQ:{h_q} HeadKV:{h_kv} HDim:{H} Causal:{is_causal}")+ print()++ # --- FP16 reference ---+ ref = torch.nn.functional.scaled_dot_product_attention(+ q, k, v, scale=scale, is_causal=is_causal, enable_gqa=(h_q // h_kv) > 1+ )++ # --- Test different block sizes ---+ results = {}++ for bs in block_sizes:+ out = get_ark().sage_dynquant(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=bs)+ dff = (ref - out).abs()+ t = bench(lambda bs=bs: get_ark().sage_dynquant(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=bs))+ results[bs] = (t, dff.max().item(), dff.mean().item())+ print(+ f" SageDynQuan block_size={bs:3d}: {t:8.1f} ms diff max={dff.max().item():.6f} mean={dff.mean().item():.6f}"+ )++ # --- SAGE v1 reference for each block_size (pre-quantized INT8) ---+ sage_results = {}+ for bs in block_sizes:+ q_i8 = torch.randint(-128, 127, (batch, h_q, seq, H), dtype=torch.int8, device=dev)+ k_i8 = torch.randint(-128, 127, (batch, h_kv, seq, H), dtype=torch.int8, device=dev)+ qs = torch.randn(batch, h_q, seq // bs, 1, dtype=torch.float32, device=dev) / 100 + 0.001+ ks = torch.randn(batch, h_kv, seq // bs, 1, dtype=torch.float32, device=dev) / 100 + 0.001+ sage_t = bench(+ lambda bs=bs, q_i8=q_i8, k_i8=k_i8, qs=qs, ks=ks: get_ark().sage(+ q_i8, k_i8, v, scale_block_size=bs, qscale=qs, kscale=ks, scale=scale, is_causal=is_causal+ )+ )+ sage_results[bs] = sage_t+ print(f" SAGE v1 bs={bs:3d}: {sage_t:8.1f} ms")++ print()+ print("--- Summary (dynquant vs SAGE v1 at same block_size) ---")+ for bs in block_sizes:+ t, mx, mn = results[bs]+ sage_t = sage_results[bs]+ overhead = (t / sage_t - 1) * 100+ print(f" block_size={bs:3d}: dynquant {t:8.1f} ms vs sage_v1 {sage_t:8.1f} ms (+{overhead:.0f}%)")+++if __name__ == "__main__":+ run_benchmark()+ run_benchmark(is_causal=True)diff --git a/auto_round_extension/ark/test/test_sdpa.py b/auto_round_extension/ark/test/test_sdpa.pynew file mode 100644index 000000000..4aaabdc14--- /dev/null+++ b/auto_round_extension/ark/test/test_sdpa.py@@ -0,0 +1,758 @@+# # Copyright (C) 2026 Intel Corporation+# # SPDX-License-Identifier: Apache-2.0++import math+import time++import auto_round_kernel+import pandas as pd+import torch++ark = None+++ARK_BENCHMARK_COLUMNS = [+ "#",+ "API Version",+ "Decode/Prefill",+ "data type",+ "Batch",+ "NumHeads_q",+ "NumHeads_kv",+ "Seq Length QO",+ "Seq Length KV",+ "Head Size QK",+ "Head Size VO",+ "Causal Mask",+ "Variable SeqLen",+ "CRI GB/s",+ "CRI TFlop/s",+ "CRI Time (ms)",+ "CRI flops/clk",+ "CRI % of peak",+ "Comments",+]+++def dtype_to_name(dtype):+ dtype_names = {+ torch.float16: "float16",+ torch.bfloat16: "bfloat16",+ torch.float32: "float32",+ torch.float64: "float64",+ }+ return dtype_names.get(dtype, str(dtype).replace("torch.", ""))+++def resolve_torch_dtype(dtype):+ if isinstance(dtype, torch.dtype):+ return dtype+ dtype_aliases = {+ "fp16": torch.float16,+ "float16": torch.float16,+ "half": torch.float16,+ "bf16": torch.bfloat16,+ "bfloat16": torch.bfloat16,+ "fp32": torch.float32,+ "float32": torch.float32,+ "float": torch.float32,+ "fp64": torch.float64,+ "float64": torch.float64,+ "double": torch.float64,+ }+ normalized = str(dtype).strip().lower()+ if normalized not in dtype_aliases:+ raise ValueError(f"Unsupported dtype: {dtype}")+ return dtype_aliases[normalized]+++def save_ark_benchmark_results(df, output_file="benchmark_results.xlsx"):+ for column in ARK_BENCHMARK_COLUMNS:+ if column not in df.columns:+ df[column] = None+ df = df[ARK_BENCHMARK_COLUMNS]+ try:+ df.to_excel(output_file, index=False, engine="openpyxl")+ except Exception as exc:+ print(f"Error saving Excel file {output_file}: {exc}")+++def build_attn_bias(seq, seq_kv, dtype, device, is_causal):+ attn_bias = torch.zeros(seq, seq_kv, dtype=dtype, device=device)+ if is_causal:+ diagonal = seq_kv - seq+ causal_mask = torch.ones(seq, seq_kv, dtype=torch.bool, device=device).tril(diagonal=diagonal)+ attn_bias.masked_fill_(causal_mask.logical_not(), float("-inf"))+ return attn_bias+++def benchmark_ark_sdpa_case(+ batch,+ seq,+ seq_kv,+ h_q,+ h_kv,+ head_size_qk,+ head_size_vo=None,+ dt=torch.float16,+ dev="xpu",+ is_causal=False,+ warmup_runs=10,+ benchmark_runs=100,+):+ if head_size_vo is None:+ head_size_vo = head_size_qk++ group = h_q // h_kv+ q = torch.rand(batch, h_q, seq, head_size_qk, dtype=dt, device=dev)+ k = torch.rand(batch, h_kv, seq_kv, head_size_qk, dtype=dt, device=dev)+ v = torch.rand(batch, h_kv, seq_kv, head_size_vo, dtype=dt, device=dev)+ attn_bias = build_attn_bias(seq, seq_kv, dt, q.device, is_causal)+ scale = 1 / math.sqrt(head_size_qk)+ ref = torch.nn.functional.scaled_dot_product_attention(q, k, v, scale=scale, enable_gqa=group > 1, is_causal=True)+ ret = get_ark().sdpa(q, k, v, scale=scale, is_causal=is_causal)+ dff = abs(ref - ret)+ print_top_diffs(dff, ref, ret, topk=4, threshold=1)+ for _ in range(warmup_runs):+ get_ark().sdpa(q, k, v, scale=scale, is_causal=is_causal)+ if dev == "xpu":+ torch.xpu.synchronize()++ st = time.time()+ for _ in range(benchmark_runs):+ get_ark().sdpa(q, k, v, scale=scale, is_causal=is_causal)+ if dev == "xpu":+ torch.xpu.synchronize()+ et = time.time()+ torch.xpu.empty_cache()+ dur = (et - st) / benchmark_runs+ ops = group * h_kv * seq * head_size_qk * seq_kv * 2+ # ops += group * h_kv * seq * seq_kv * 2+ ops += group * h_kv * seq * head_size_vo * seq_kv * 2+ ops *= batch++ mem = batch * h_q * seq * head_size_qk * dt.itemsize+ mem += batch * h_kv * seq_kv * head_size_qk * dt.itemsize+ mem += batch * h_kv * seq_kv * head_size_vo * dt.itemsize+ mem += seq * seq_kv * dt.itemsize+ return {+ "CRI GB/s": mem / dur / 1e9,+ "CRI TFlop/s": ops / dur / 1e12,+ "CRI Time (ms)": dur * 1e3,+ }+++TEST_CASES = [+ # #--- bfloat16 Cases ---+ ("prefill", "float16", 128, 1, 96, 8, 4096, 4096, True, False, "F16 Case 10 Prefill"),+ # ("prefill", "float16", 128, 1, 96, 8, 4096, 8192, True, False, "F16 Case 10 Prefill"),+ ("prefill", "float16", 128, 1, 96, 8, 8192, 8192, True, False, "F16 Case 10 Prefill"),+ ("prefill", "float16", 128, 1, 96, 8, 16384, 16384, True, False, "F16 Case 10 Prefill"),+ # ("prefill", "float16", 128, 1, 96, 8, 32768, 32768, True, False, "F16 Case 10 Prefill"),+ # mode, dtype, hdim, batch, nh_q, nh_kv, seq_qo, seq_kv, is_causal, is_varlen, comment+ # ("decode", "bfloat16", 1, 32, 8, 1, 4096, False, False, "BF16 Case 10"),+ # # --- MXFP8 Cases ---+ # ("prefill", "mx_float_e4m3", 1, 4, 4, 512, 512, False, False, "MXFP8 e4m3 Case 1 Prefill"),+ #+ # ("prefill", "mx_float_e5m2", 1, 4, 4, 512, 512, False, False, "MXFP8 e5m2 Case 5 Prefill"),+ # # --- FP8 Cases ---+ # ("prefill", "float_e4m3", 1, 4, 4, 512, 512, False, False, "FP8 e4m3 Case 1 Prefill"),+]+++def run_ark_sdpa_to_excel(+ test_cases,+ output_file="benchmark_results.xlsx",+ api_version="ARK.sdpa",+ dev="xpu",+ warmup_runs=10,+ benchmark_runs=100,+):+ results = []+ print(f"Starting ARK SDPA benchmark run... ({len(test_cases)} cases)")++ for idx, case in enumerate(test_cases, start=1):+ (+ mode,+ dtype,+ head_dim,+ batch,+ num_heads_q,+ num_heads_kv,+ seq_len_qo,+ seq_len_kv,+ is_causal,+ is_varlen,+ comment,+ ) = case+ torch_dtype = resolve_torch_dtype(dtype)+ row_data = {+ "#": idx,+ "API Version": api_version,+ "Decode/Prefill": mode.capitalize(),+ "data type": dtype_to_name(torch_dtype),+ "Batch": batch,+ "NumHeads_q": num_heads_q,+ "NumHeads_kv": num_heads_kv,+ "Seq Length QO": seq_len_qo,+ "Seq Length KV": seq_len_kv,+ "Head Size QK": head_dim,+ "Head Size VO": head_dim,+ "Causal Mask": "Yes" if is_causal else "No",+ "Variable SeqLen": "Yes" if is_varlen else "No",+ "CRI flops/clk": None,+ "CRI % of peak": None,+ "Comments": comment,+ }++ print(f"[{idx}/{len(test_cases)}] Running {comment}...")+ try:+ metrics = benchmark_ark_sdpa_case(+ batch=batch,+ seq=seq_len_qo,+ seq_kv=seq_len_kv,+ h_q=num_heads_q,+ h_kv=num_heads_kv,+ head_size_qk=head_dim,+ head_size_vo=head_dim,+ dt=torch_dtype,+ dev=dev,+ is_causal=is_causal,+ warmup_runs=warmup_runs,+ benchmark_runs=benchmark_runs,+ )+ row_data.update(metrics)+ except Exception as exc:+ print(f"Error running benchmark for case {idx} ({comment}): {exc}")+ row_data.update(+ {+ "CRI GB/s": None,+ "CRI TFlop/s": None,+ "CRI Time (ms)": None,+ "Comments": f"{comment} (Run Failed: {exc})",+ }+ )++ results.append(row_data)+ result_df = pd.DataFrame(results)+ print(result_df[["NumHeads_q", "NumHeads_kv", "Seq Length QO", "Seq Length KV", "CRI TFlop/s"]])+ save_ark_benchmark_results(result_df, output_file=output_file)++ print(f"Success! Results saved to {output_file}")+ return pd.DataFrame(results)+++def is_xpu_available():+ return hasattr(torch, "xpu") and torch.xpu.is_available()+++def get_ark():+ """Lazily initialize and return the ARK instance."""+ global ark+ if ark is None:+ if not is_xpu_available():+ raise RuntimeError("XPU is not available; cannot initialize auto_round_kernel.ARK()")+ ark = auto_round_kernel.ARK()+ return ark+++def has_sdpa():+ """Check if Flash Attention kernel is available."""+ try:+ ark_instance = get_ark()+ except Exception:+ return False+ if ark_instance.xpu_lib is None:+ return False+ return hasattr(ark_instance.xpu_lib, "sdpa")+++def build_mask(seq, seq_kv, dt=torch.float32, dev="xpu", const_count=0, const_value=0.0):+ mask = torch.rand(seq, seq_kv, dtype=dt, device=dev)+ if const_count <= 0:+ return mask+ flat_mask = mask.reshape(-1)+ const_count = min(const_count, flat_mask.numel())+ indices = torch.randperm(flat_mask.numel(), device=flat_mask.device)[:const_count]+ flat_mask[indices] = const_value+ return mask+++def print_top_diffs(diff, ref, out, topk=10, threshold=0):+ flat_diff = diff.reshape(-1)+ topk = min(topk, flat_diff.numel())+ top_values, top_indices = torch.topk(flat_diff, k=topk)+ flat_ref = ref.reshape(-1)+ flat_out = out.reshape(-1)++ print(f"diff max={diff.max()} mean={diff.mean()}")+ print(f"Top {topk} diff entries:")+ if diff.max() > threshold:+ for rank, (value, flat_index) in enumerate(zip(top_values, top_indices), start=1):+ coord = tuple(int(index.item()) for index in torch.unravel_index(flat_index, diff.shape))+ ref_value = flat_ref[flat_index].item()+ out_value = flat_out[flat_index].item()+ print(f"#{rank}: index={coord}, diff={value.item()}, ref={ref_value}, out={out_value}")+++def sftm(seq, h, block=-1, dt=torch.float32, dev="xpu"):+ input = torch.rand(seq, h, dtype=dt, device=dev)+ ref = torch.softmax(input, dim=-1)+ print(ref)+ if block != -1:+ block_sum = []+ out = torch.zeros_like(ref)+ for i in range(0, h, block):+ inp = input[:, i : i + block]+ maxinp = inp.max(dim=-1, keepdim=True).values+ inexp = torch.exp(inp - maxinp)+ sum = torch.sum(inexp, dim=-1, keepdim=True)+ out[:, i : i + block] = inexp / sum+ block_sum.append(sum)+ block_sums = torch.concat(block_sum, dim=-1)+ all_sum = torch.sum(block_sums, dim=-1, keepdim=True)+ scale = block_sums / all_sum+ for i in range(len(block_sum)):+ out[:, i * block : (i + 1) * block] = out[:, i * block : (i + 1) * block] * scale[:, i : i + 1]+ print(out)+ diff = abs(ref - out)+ print_top_diffs(diff, ref, out)+++def bench_sdpa(seq, seq_kv, h_q, h_kv, H, H_v, dt=torch.float32, dev="xpu", mask_const_count=0, mask_const_value=0.0):+ group = h_q // h_kv+ q = torch.rand(group, h_kv, seq, H, dtype=dt, device=dev)+ k = torch.rand(1, h_kv, H, seq_kv, dtype=dt, device=dev)+ v = torch.rand(1, h_kv, seq_kv, H_v, dtype=dt, device=dev)+ mask = build_mask(seq, seq_kv, dt=dt, dev=dev, const_count=mask_const_count, const_value=mask_const_value)+ scale = 1 / math.sqrt(H)+ n_runs = 100+ for i in range(n_runs):+ score = torch.matmul(q, k) * scale+ score = score + mask+ score = torch.softmax(score, dim=-1)+ out = torch.matmul(score, v) # group, h_kv, seq, H_v+ if dev == "xpu":+ torch.xpu.synchronize()+ st = time.time()+ for i in range(n_runs):+ score = torch.matmul(q, k) * scale+ score = score + mask+ score = torch.softmax(score, dim=-1)+ out = torch.matmul(score, v) # group, h_kv, seq, H_v+ if dev == "xpu":+ torch.xpu.synchronize()+ et = time.time()+ dur = (et - st) / n_runs+ OPS = group * h_kv * seq * H * seq_kv * 2 # q*k+ OPS += group * h_kv * seq * seq_kv * 2 # sfmx+ OPS += group * h_kv * seq * H_v * seq_kv * 2 # s*v+ MEM = group * h_kv * seq * H * dt.itemsize # q+ MEM += h_kv * seq_kv * H * dt.itemsize # q+ MEM += h_kv * seq_kv * H_v * dt.itemsize # q+ print(f"Seq_Q:{seq}, Seq_KV:{seq_kv}, HeadNum_Q:{h_q}, HeadNum_KV:{h_kv}, HeadDim_QK:{H}, HeadDim_V:{H_v}")+ print(f"FLOPS:{OPS/dur/1e9:.2f} G, MEM:{MEM/dur/1e9:.2f} GB/s")+ # out = out.reshape(h_q, seq, H_v)+ # out = out.transpose(0, 1)+ # print(out)+++def bench_torch(seq, seq_kv, h_q, h_kv, H, H_v, dt=torch.float32, dev="xpu", mask_const_count=0, mask_const_value=0.0):+ group = h_q // h_kv+ q = torch.rand(1, h_q, seq, H, dtype=dt, device=dev)+ k = torch.rand(1, h_kv, seq_kv, H, dtype=dt, device=dev)+ v = torch.rand(1, h_kv, seq_kv, H_v, dtype=dt, device=dev)+ mask = build_mask(seq, seq_kv, dt=dt, dev=dev, const_count=mask_const_count, const_value=mask_const_value)+ scale = 1 / math.sqrt(H)+ n_runs = 100+ for i in range(n_runs):+ ref = torch.nn.functional.scaled_dot_product_attention(q, k, v, mask, scale=scale)+ if dev == "xpu":+ torch.xpu.synchronize()+ st = time.time()+ for i in range(n_runs):+ ref = torch.nn.functional.scaled_dot_product_attention(q, k, v, mask, scale=scale)+ if dev == "xpu":+ torch.xpu.synchronize()+ et = time.time()+ dur = (et - st) / n_runs+ OPS = group * h_kv * seq * H * seq_kv * 2 # q*k+ OPS += group * h_kv * seq * seq_kv * 2 # sfmx+ OPS += group * h_kv * seq * H_v * seq_kv * 2 # s*v+ MEM = group * h_kv * seq * H * dt.itemsize # q+ MEM += h_kv * seq_kv * H * dt.itemsize # q+ MEM += h_kv * seq_kv * H_v * dt.itemsize # q+ print(f"Seq_Q:{seq}, Seq_KV:{seq_kv}, HeadNum_Q:{h_q}, HeadNum_KV:{h_kv}, HeadDim_QK:{H}, HeadDim_V:{H_v}")+ print(f"FLOPS:{OPS/dur/1e9:.2f} G, MEM:{MEM/dur/1e9:.2f} GB/s")+ # out = out.reshape(h_q, seq, H_v)+ # out = out.transpose(0, 1)+ # print(out)+++def bench_ark(+ batch,+ seq,+ seq_kv,+ h_q,+ h_kv,+ H,+ H_v,+ dt=torch.float32,+ dev="xpu",+ mask_const_count=0,+ mask_const_value=0.0,+ has_mask=False,+ is_causal=False,+):+ group = h_q // h_kv+ q = torch.rand(batch, h_q, seq, H, dtype=dt, device=dev)+ k = torch.rand(batch, h_kv, seq_kv, H, dtype=dt, device=dev)+ v = torch.rand(batch, h_kv, seq_kv, H_v, dtype=dt, device=dev)+ mask = build_mask(seq, seq_kv, dt=dt, dev=dev, const_count=mask_const_count, const_value=mask_const_value)+ scale = 1 / math.sqrt(H)+ n_runs = 20 if seq_kv > 8192 else 100+ for i in range(n_runs):+ ref = get_ark().sdpa(q, k, v, mask if has_mask else None, softmax_scale=scale, is_causal=is_causal)+ if dev == "xpu":+ torch.xpu.synchronize()+ st = time.time()+ for i in range(n_runs):+ ref = get_ark().sdpa(q, k, v, mask if has_mask else None, softmax_scale=scale, is_causal=is_causal)+ if dev == "xpu":+ torch.xpu.synchronize()+ et = time.time()+ dur = (et - st) / n_runs+ OPS = group * h_kv * seq * H * seq_kv * 2 # q*k+ OPS += group * h_kv * seq * seq_kv * 2 # sfmx+ OPS += group * h_kv * seq * H_v * seq_kv * 2 # s*v+ OPS *= batch+ MEM = group * h_kv * seq * H * dt.itemsize # q+ MEM += h_kv * seq_kv * H * dt.itemsize # q+ MEM += h_kv * seq_kv * H_v * dt.itemsize # q+ MEM *= batch+ print(+ f"Batch:{batch} Seq_Q:{seq}, Seq_KV:{seq_kv}, HeadNum_Q:{h_q}, HeadNum_KV:{h_kv}, HeadDim_QK:{H}, HeadDim_V:{H_v} Causal:{is_causal}"+ )+ print(f"Time:{dur*1e3:.3f} FLOPS:{OPS/dur/1e9:.2f} G, MEM:{MEM/dur/1e9:.2f} GB/s")+ # out = out.reshape(h_q, seq, H_v)+ # out = out.transpose(0, 1)+ # print(out)+++def compare_sdpa(seq, seq_kv, h_q, h_kv, H, H_v, dt=torch.float32, dev="xpu", mask_const_count=0, mask_const_value=0.0):+ group = h_q // h_kv+ q = torch.rand(1, h_q, seq, H, dtype=dt, device=dev)+ k = torch.rand(1, h_kv, seq_kv, H, dtype=dt, device=dev)+ v = torch.rand(1, h_kv, seq_kv, H_v, dtype=dt, device=dev)+ mask = build_mask(seq, seq_kv, dt=dt, dev=dev, const_count=mask_const_count, const_value=mask_const_value)+ mask[:, :] = -100+ scale = 1 / math.sqrt(H)+ ref = torch.nn.functional.scaled_dot_product_attention(q, k, v, mask, scale=scale)++ q = q.view(group, h_kv, seq, H)+ k = k.transpose(2, 3).contiguous()+ score = torch.matmul(q, k) * scale+ score = score + mask+ score = torch.softmax(score, dim=-1)+ out = torch.matmul(score, v) # group, h_kv, seq, H_v+ diff = abs(ref - out)+ print_top_diffs(diff, ref, out)+++def compare_sdpa_ark(+ batch,+ seq,+ seq_kv,+ h_q,+ h_kv,+ H,+ H_v,+ dt=torch.float32,+ dev="xpu",+ qk_scale_factor=4.0,+ v_scale_factor=100.0,+):+ group = h_q // h_kv+ torch.manual_seed(1234)+ q = torch.randn(batch, h_q, seq, H, dtype=dt, device=dev) * qk_scale_factor+ k = torch.randn(batch, h_kv, seq_kv, H, dtype=dt, device=dev) * qk_scale_factor+ v = torch.randn(batch, h_kv, seq_kv, H_v, dtype=dt, device=dev) * v_scale_factor+ decode = seq == 1+ attn_bias = torch.zeros(seq, seq_kv, dtype=q.dtype, device=q.device)+ temp_mask = torch.ones(seq, seq_kv, dtype=torch.bool, device=q.device).tril(diagonal=seq_kv - seq)+ attn_bias.masked_fill_(temp_mask.logical_not(), float("-inf"))+ scale = 1 / math.sqrt(H)+ ref = torch.nn.functional.scaled_dot_product_attention(+ q, k, v, attn_mask=attn_bias, scale=scale, enable_gqa=group > 1, is_causal=False+ )+ if not decode:+ out = get_ark().sdpa(q, k, v, attn_bias, softmax_scale=scale, is_causal=False)+ diff = abs(ref - out)+ print_top_diffs(diff, ref, out, topk=4, threshold=1)++ out2 = get_ark().sdpa(q, k, v, softmax_scale=scale, is_causal=True)+ diff = abs(out2 - out)+ print_top_diffs(diff, out2, out, topk=4)+ else:+ out = get_ark().sdpa(q, k, v, softmax_scale=scale, is_causal=True)+ diff = abs(ref - out)+ print_top_diffs(diff, ref, out, topk=4, threshold=1)+++def compare_sdpa_sage(batch, seq, seq_kv, h_q, h_kv, H, H_v, dt=torch.float16, dev="xpu"):+ group = h_q // h_kv+ q = torch.randint(-4, 4, (batch, h_q, seq, H), dtype=torch.int8, device=dev)+ k = torch.randint(-4, 4, (batch, h_kv, seq_kv, H), dtype=torch.int8, device=dev)+ v = torch.rand(batch, h_kv, seq_kv, H_v, dtype=dt, device=dev) * 100+ scale = 1 / math.sqrt(H)+ out = get_ark().sdpa(q, k, v, softmax_scale=scale, is_causal=False)++ n_runs = 100+ for i in range(n_runs):+ ref = get_ark().sdpa(q, k, v, softmax_scale=scale, is_causal=False)+ if dev == "xpu":+ torch.xpu.synchronize()+ st = time.time()+ for i in range(n_runs):+ ref = get_ark().sdpa(q, k, v, softmax_scale=scale, is_causal=False)+ if dev == "xpu":+ torch.xpu.synchronize()+ et = time.time()+ dur = (et - st) / n_runs++ q = q.view(batch, group, h_kv, seq, H).contiguous().to(torch.float32)+ k = k.view(batch, 1, h_kv, seq_kv, H).transpose(3, 4).contiguous().to(torch.float32)+ score = torch.matmul(q, k)+ score = score * scale+ score = torch.softmax(score, dim=-1)++ ref = torch.matmul(score.to(dt), v.view(batch, 1, h_kv, seq_kv, H_v)).view(+ batch, h_q, seq, H+ ) # group, h_kv, seq, H_v+ dff = abs(ref - out)+ print_top_diffs(dff, ref, out, topk=4, threshold=1)++ OPS = group * h_kv * seq * H * seq_kv * 2 # q*k+ OPS += group * h_kv * seq * seq_kv * 2 # sfmx+ OPS += group * h_kv * seq * H_v * seq_kv * 2 # s*v+ OPS *= batch+ MEM = group * h_kv * seq * H * dt.itemsize # q+ MEM += h_kv * seq_kv * H * dt.itemsize # q+ MEM += h_kv * seq_kv * H_v * dt.itemsize # q+ MEM *= batch+ print(+ f"Batch:{batch} Seq_Q:{seq}, Seq_KV:{seq_kv}, HeadNum_Q:{h_q}, HeadNum_KV:{h_kv}, HeadDim_QK:{H}, HeadDim_V:{H_v}"+ )+ print(f"Time:{dur*1e3:.3f} FLOPS:{OPS/dur/1e9:.2f} G, MEM:{MEM/dur/1e9:.2f} GB/s")+++def compare_sdpa_sage_scale(batch, seq, seq_kv, h_q, h_kv, H, H_v, dt=torch.float16, is_causal=False, dev="xpu"):+ group = h_q // h_kv+ block_size = 64+ seq = seq // block_size * block_size+ seq_kv = seq_kv // block_size * block_size+ q = torch.randint(-128, 127, (batch, h_q, seq, H), dtype=torch.int8, device=dev)+ k = torch.randint(-128, 127, (batch, h_kv, seq_kv, H), dtype=torch.int8, device=dev)+ if block_size:+ q_scale = torch.randn(batch, h_q, seq // block_size, 1, dtype=torch.float32, device=dev) / 100 + 0.001+ k_scale = torch.randn(batch, h_kv, seq_kv // block_size, 1, dtype=torch.float32, device=dev) / 100 + 0.001+ else:+ q_scale = None+ k_scale = None+ v = torch.rand(batch, h_kv, seq_kv, H_v, dtype=dt, device=dev) * 100+ scale = 1 / math.sqrt(H)+ # q_scale[...]=1+ # k_scale[...]=1+ attn_bias = torch.zeros(seq, seq_kv, dtype=v.dtype, device=q.device)+ temp_mask = torch.ones(seq, seq_kv, dtype=torch.bool, device=q.device).tril(diagonal=seq_kv - seq)+ attn_bias.masked_fill_(temp_mask.logical_not(), float("-inf"))+ scale = 1 / math.sqrt(H)+ # attn_bias = None++ out = get_ark().sage(+ q, k, v, quant_block_size=block_size, qscale=q_scale, kscale=k_scale, scale=scale, is_causal=is_causal+ )++ n_runs = 10 if seq > 8192 else 100+ for i in range(n_runs):+ ref = get_ark().sage(+ q, k, v, quant_block_size=block_size, qscale=q_scale, kscale=k_scale, scale=scale, is_causal=is_causal+ )+ if dev == "xpu":+ torch.xpu.synchronize()+ st = time.time()+ for i in range(n_runs):+ ref = get_ark().sage(+ q, k, v, quant_block_size=block_size, qscale=q_scale, kscale=k_scale, scale=scale, is_causal=is_causal+ )+ if dev == "xpu":+ torch.xpu.synchronize()+ et = time.time()+ dur = (et - st) / n_runs+ if block_size:+ q_fp = (q.to(torch.float32) * q_scale.repeat_interleave(block_size, dim=2)).to(dt)+ k_fp = (k.to(torch.float32) * k_scale.repeat_interleave(block_size, dim=2)).to(dt)+ else:+ q_fp = q.to(dt)+ k_fp = k.to(dt)+ ref = torch.nn.functional.scaled_dot_product_attention(+ q_fp, k_fp, v, scale=scale, is_causal=is_causal, enable_gqa=group > 1+ )+ # out = get_ark().sdpa(q_fp, k_fp, v, softmax_scale=scale, is_causal=is_causal)+ # print(ref)+ dff = abs(ref - out)+ print_top_diffs(dff, ref, out, topk=4, threshold=1)++ OPS = group * h_kv * seq * H * seq_kv * 2 # q*k+ OPS += group * h_kv * seq * seq_kv * 2 # sfmx+ OPS += group * h_kv * seq * H_v * seq_kv * 2 # s*v+ OPS *= batch+ MEM = group * h_kv * seq * H * dt.itemsize # q+ MEM += h_kv * seq_kv * H * dt.itemsize # q+ MEM += h_kv * seq_kv * H_v * dt.itemsize # q+ MEM *= batch+ print(+ f"Batch:{batch} Seq_Q:{seq}, Seq_KV:{seq_kv}, HeadNum_Q:{h_q}, HeadNum_KV:{h_kv}, HeadDim_QK:{H}, HeadDim_V:{H_v}"+ )+ print(f"Time:{dur*1e3:.3f} FLOPS:{OPS/dur/1e9:.2f} G, MEM:{MEM/dur/1e9:.2f} GB/s")+++def compare_sdpa_sage_dynquant(+ batch, seq, seq_kv, h_q, h_kv, H, H_v, dt=torch.float16, is_causal=False, dev="xpu", quant_block_size=64+):+ """Test dynamic quantization SAGE attention.++ Q, K, V are FP16 inputs. The kernel dynamically quantizes Q, K to INT8+ """++ group = h_q // h_kv+ q = torch.randn(batch, h_q, seq, H, dtype=dt, device=dev)+ k = torch.randn(batch, h_kv, seq_kv, H, dtype=dt, device=dev)+ v = torch.randn(batch, h_kv, seq_kv, H_v, dtype=dt, device=dev) * 100+ scale = 1 / math.sqrt(H)+ if seq_kv > 4096 and is_causal:+ if seq_kv == seq:+ ref = torch.nn.functional.scaled_dot_product_attention(+ q, k, v, scale=scale, enable_gqa=group > 1, is_causal=is_causal+ )+ else:+ ref = torch.zeros_like(q)+ else:+ attn_bias = torch.zeros(seq, seq_kv, dtype=q.dtype, device=q.device)+ temp_mask = torch.ones(seq, seq_kv, dtype=torch.bool, device=q.device).tril(diagonal=seq_kv - seq)+ attn_bias.masked_fill_(temp_mask.logical_not(), float("-inf"))+ scale = 1 / math.sqrt(H)+ if is_causal:+ ref = torch.nn.functional.scaled_dot_product_attention(+ q, k, v, scale=scale, enable_gqa=group > 1, is_causal=False, attn_mask=attn_bias+ )+ else:+ ref = torch.nn.functional.scaled_dot_product_attention(+ q, k, v, scale=scale, enable_gqa=group > 1, is_causal=is_causal+ )++ # Dynamic quantization SAGE kernel+ out = get_ark().sagev1(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=quant_block_size)++ dff = abs(ref - out)+ print(f"=== Dynamic Quantization SAGE Test (block_size={quant_block_size}) ===")+ print(+ f"Batch:{batch} Seq_Q:{seq}, Seq_KV:{seq_kv}, HeadNum_Q:{h_q}, HeadNum_KV:{h_kv}, HeadDim_QK:{H}, HeadDim_V:{H_v}, Causal:{is_causal}"+ )+ print_top_diffs(dff, ref, out, topk=4, threshold=1)++ # Benchmark+ n_runs = 10 if seq > 8192 else 100+ for i in range(n_runs):+ _ = get_ark().sagev1(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=quant_block_size)+ if dev == "xpu":+ torch.xpu.synchronize()+ st = time.time()+ for i in range(n_runs):+ _ = get_ark().sagev1(q, k, v, scale=scale, is_causal=is_causal, quant_block_size=quant_block_size)+ if dev == "xpu":+ torch.xpu.synchronize()+ et = time.time()+ dur = (et - st) / n_runs++ OPS = group * h_kv * seq * H * seq_kv * 2 # q*k+ OPS += group * h_kv * seq * seq_kv * 2 # sfmx+ OPS += group * h_kv * seq * H_v * seq_kv * 2 # s*v+ OPS *= batch+ MEM = group * h_kv * seq * H * dt.itemsize # q+ MEM += h_kv * seq_kv * H * dt.itemsize # k+ MEM += h_kv * seq_kv * H_v * dt.itemsize # v+ MEM *= batch+ print(f"Time:{dur*1e3:.3f} FLOPS:{OPS/dur/1e9:.2f} G, MEM:{MEM/dur/1e9:.2f} GB/s")+ return out, ref+++if __name__ == "__main__":+ # run_ark_sdpa_to_excel(TEST_CASES)+ # sftm(1, 1024, 512)+ # sftm(100, 1024, 512)+ # compare_sdpa(1024, 1024, 32, 32, 128, 128, dt=torch.float16)+ # compare_sdpa(1, 1024, 32, 32, 128, 128, dt=torch.float16)+ # compare_sdpa(+ # 512,+ # 512,+ # 32,+ # 32,+ # 128,+ # 128,+ # dt=torch.float16+ # )+ # compare_sdpa_ark(+ # 1,+ # 1,+ # 4096,+ # 32,+ # 32,+ # 128,+ # 128,+ # dt=torch.float16+ # )+ # compare_sdpa_ark(+ # 1,+ # 4096,+ # 8192,+ # 32,+ # 32,+ # 128,+ # 128,+ # dt=torch.float16+ # )+ # compare_sdpa_ark(+ # 1, 512, 512, 32, 32, 128, 128,+ # dt=torch.float16+ # )+ # compare_sdpa_ark(2, 1776, 1776, 30, 30, 128, 128, dt=torch.float16)+ # bench_ark(2, 17776, 17776, 30, 30, 64, 64, dt=torch.float16, is_causal=False)+ # bench_ark(1, 4096, 8192*1, 96, 8, 64, 64, dt=torch.float16)+ # bench_ark(1, 4096, 8192*1, 96, 8, 64, 64, dt=torch.float16, is_causal=True)+ # compare_sdpa_sage_scale(1, 8192, 8192, 96, 8, 128, 128, dt=torch.float16)+ # compare_sdpa_sage_scale(1, 8192, 8192, 96, 8, 128, 128, dt=torch.float16, is_causal=True)+ compare_sdpa_sage_dynquant(1, 8192, 8192, 96, 8, 128, 128, dt=torch.float16)+ compare_sdpa_sage_dynquant(1, 8192, 8192, 96, 8, 128, 128, dt=torch.float16, is_causal=True)+ # compare_sdpa_sage_scale(2, 4096, 4096, 96, 8, 128, 128, dt=torch.float16)+ # compare_sdpa_sage_scale(1, 4096, 4096, 96, 8, 128, 128, dt=torch.float16, is_causal=True)+ # compare_sdpa_sage(16, 512, 512, 32, 32, 128, 128, dt=torch.float16)+ # compare_sdpa_ark(16, 512, 512, 32, 32, 128, 128, dt=torch.float16)+ # compare_sdpa_ark(16, 256, 512, 32, 8, 128, 128, dt=torch.float16)+ # compare_sdpa_ark(16, 256, 512, 32, 8, 128, 128, dt=torch.bfloat16)+ # compare_sdpa_ark(16, 1, 512, 32, 8, 128, 128, dt=torch.float16)+ # compare_sdpa_ark(16, 1, 512, 32, 8, 128, 128, dt=torch.bfloat16)+ # # bench_sdpa(1024, 1024, 32, 32, 128, 128, dt=torch.float16)+ # # bench_torch(1024, 1024, 32, 32, 128, 128, dt=torch.float16)+ # bench_ark(2, 17776, 17776, 30, 30, 128, 128, dt=torch.float16, is_causal=False)+ # bench_ark(2, 17776, 17776, 30, 30, 64, 64, dt=torch.float16, is_causal=False)+ # --- Dynamic Quantization SAGE Tests ---+ # compare_sdpa_sage_dynquant(1, 4096, 4096, 32, 32, 128, 128, dt=torch.float16)+ # compare_sdpa_sage_dynquant(2, 4096, 4096, 96, 8, 128, 128, dt=torch.float16)+ # compare_sdpa_sage_dynquant(1, 4096, 4096, 96, 8, 128, 128, dt=torch.float16, is_causal=True)+ # compare_sdpa_sage_dynquant(2, 4096, 4096, 30, 30, 64, 64, dt=torch.float16)+ # bench_ark(1, 4096, 4096, 96, 8, 128, 128, dt=torch.float16, is_causal=True)+ # bench_ark(32, 512, 512, 32, 32, 128, 128, dt=torch.float16)+ # bench_ark(32, 512, 512, 16, 16, 128, 128, dt=torch.bfloat16)+ # bench_ark(32, 1, 512, 32, 8, 128, 128, dt=torch.float16)+ # bench_ark(32, 1, 512, 16, 16, 128, 128, dt=torch.bfloat16)diff --git a/auto_round_extension/ark/test/test_weightonly.py b/auto_round_extension/ark/test/test_weightonly.pynew file mode 100644index 000000000..d26e843fd--- /dev/null+++ b/auto_round_extension/ark/test/test_weightonly.py@@ -0,0 +1,293 @@+#!/usr/bin/env python+# -*- coding: utf-8 -*-+#+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import os+import time++import auto_round_kernel+import pytest+import torch+from ut_utils import decode_fp8_to_float, gen_weis8, get_torch_dt, sample_valid_fp8, sample_valid_fp8_e8m0_xpu_safe++ark = auto_round_kernel.ARK()+++def main_op(m, n, k, blocksize, compute_type, weight_type, scale_type, asym, device, is_ci=False):+ print(+ f"\n m={m}, n={n}, k={k}, blocksize={blocksize}, compute_type={compute_type}, "+ f"weight_type={weight_type}, scale_type={scale_type}, asym={asym}, device={device}"+ )+ torch.manual_seed(0)+ if weight_type == "int8" and compute_type == "int8" and asym:+ pytest.skip("NO A8W8C8")+ stype = get_torch_dt(scale_type)+ if compute_type == "int8":+ dtype = stype+ ctype = torch.int8+ else:+ dtype = get_torch_dt(compute_type)+ ctype = dtype+ if device == "cpu":+ cpu_cores = os.cpu_count()+ print(f"cpu_cores per numa node: {cpu_cores}")+ # ark.set_threads(cpu_cores)+ dtype = torch.float32+ if weight_type.startswith("fp8_"):+ if asym:+ pytest.skip("FP8 weight-only does not support asym in this test")+ if scale_type not in {"fp16", "fp32", "fp8_e8m0"}:+ pytest.skip("FP8 test supports scale_type fp16/fp32/fp8_e8m0")+ if compute_type not in {"fp16", "fp32"}:+ pytest.skip("FP8 test supports compute_type fp16/fp32")++ if device == "xpu" and scale_type == "fp8_e8m0":+ # For fp16 compute, keep magnitudes small to avoid overflow to inf/nan.+ fp8_exp_range = None+ if compute_type == "fp16":+ fp8_exp_range = (0, 12) if weight_type == "fp8_e4m3" else (10, 21)+ raw_qweight = sample_valid_fp8_e8m0_xpu_safe((k, n), weight_type, device, exp_range=fp8_exp_range)+ else:+ raw_qweight = sample_valid_fp8((k, n), weight_type, device)+ decoded = decode_fp8_to_float(raw_qweight.cpu(), weight_type).to(dtype).to(device)+ if scale_type == "fp8_e8m0":+ # Narrow exponent range for fp16 to avoid inf/nan in matmul and kernel output.+ if compute_type == "fp16":+ scale = torch.randint(-4, 4, (k // blocksize, n), dtype=torch.int32, device=device).to(torch.float32)+ else:+ scale = torch.randint(-8, 8, (k // blocksize, n), dtype=torch.int32, device=device).to(torch.float32)+ scale_re = torch.pow(torch.tensor(2.0, device=device, dtype=torch.float32), scale)+ scale_re = scale_re.repeat_interleave(repeats=blocksize, dim=0)+ ref_dst = decoded * scale_re.to(dtype)+ else:+ scale = torch.randn(k // blocksize, n, dtype=stype, device=device) / 100 + 0.01+ scale_re = scale.repeat_interleave(repeats=blocksize, dim=0)+ ref_dst = decoded * scale_re.to(dtype)+ bias = torch.randn(1, n, dtype=dtype, device=device)+ zp = torch.Tensor()+ qweight_for_pack = raw_qweight+ else:+ if scale_type == "fp8_e8m0":+ pytest.skip("fp8_e8m0 scale is only supported for fp8 weights")+ raw_s8_wei = gen_weis8(weight_type, device, k, n)+ scale = torch.rand(k // blocksize, n, dtype=stype, device=device) / 300 + 0.002+ bias = torch.randn(1, n, dtype=dtype, device=device)++ scale_re = scale.repeat_interleave(repeats=blocksize, dim=0)+ if asym:+ zp = gen_weis8(weight_type, device, k // blocksize, n)+ zp_re = zp.repeat_interleave(repeats=blocksize, dim=0)+ ref_dst = (raw_s8_wei.to(dtype) - zp_re.to(dtype)) * scale_re+ else:+ zp = torch.Tensor()+ ref_dst = raw_s8_wei.to(dtype) * scale_re+ qweight_for_pack = raw_s8_wei++ packw = ark.repack_quantized_weight(+ qweight_for_pack, scale, zp, blocksize, compute_type, weight_type, scale_type, asym+ )++ revert_wei_t = ark.unpack_weight(packw, dtype, n, k, blocksize, compute_type, weight_type, scale_type, asym)+ revert_wei = revert_wei_t.t()+ assert torch.allclose(revert_wei, ref_dst)++ tar_activation = torch.randn(m, k, dtype=dtype, device=device) - 0.5+ if device == "xpu" and weight_type.startswith("fp8_") and scale_type == "fp8_e8m0" and compute_type == "fp16":+ tar_activation = tar_activation * 0.125+ ref_c = torch.matmul(tar_activation, revert_wei)+ ref_c = ref_c + bias+ runs = 400+ if m == 1:+ runs = 4000+ if is_ci:+ runs = 1+ if device == "xpu" and weight_type.startswith("fp8_") and scale_type == "fp8_e8m0":+ # Avoid large batch repeats on CI for fp8_e8m0.+ for i in range(runs):+ tar_dst = ark.woqgemm(+ tar_activation, packw, bias, n, k, blocksize, compute_type, weight_type, scale_type, asym+ )++ else:+ batch = 8+ if m == 1:+ batch = 64+ if is_ci:+ batch = 1+ packw_set = torch.unsqueeze(packw, 0).repeat(batch, 1)+ tar_activation = torch.unsqueeze(tar_activation, 0).repeat(batch, 1, 1)+ for i in range(runs):+ tar_dst = ark.woqgemm(+ tar_activation[0], packw_set[0], bias, n, k, blocksize, compute_type, weight_type, scale_type, asym+ )++ if device == "xpu":+ torch.xpu.synchronize()+ diff = abs(ref_c - tar_dst)+ print(+ f" Max Diff: {diff.max().item():.6f}, Mean Diff: {diff.mean().item():.6f}, ref mean:{ref_c.mean():.6f}, OUT mean:{tar_dst.mean():.6f}"+ )+ rtol = 0.1+ atol = 2.0+ if weight_type.startswith("fp8_"):+ if weight_type == "fp8_e5m2":+ rtol = 0.2+ atol = 24.0+ if compute_type == "fp16":+ rtol = 0.25+ atol = 24.0+ # Use equal_nan for fp16+fp8_e8m0 only to tolerate rare backend NaN divergence+ if weight_type.startswith("fp8_") and scale_type == "fp8_e8m0" and compute_type == "fp16":+ assert torch.allclose(tar_dst, ref_c, rtol=rtol, atol=atol, equal_nan=True)+ else:+ assert torch.allclose(tar_dst, ref_c, rtol=rtol, atol=atol)+ st = time.perf_counter()+ if device == "xpu" and weight_type.startswith("fp8_") and scale_type == "fp8_e8m0":+ for i in range(runs):+ tar_dst = ark.woqgemm(+ tar_activation, packw, bias, n, k, blocksize, compute_type, weight_type, scale_type, asym+ )+ if (i + 1) % 100 == 0:+ torch.xpu.empty_cache()+ else:+ for i in range(runs):+ idx = i % batch+ tar_dst = ark.woqgemm(+ tar_activation[idx], packw_set[idx], bias, n, k, blocksize, compute_type, weight_type, scale_type, asym+ )+ if device == "xpu":+ torch.xpu.synchronize()+ dur = time.perf_counter() - st+ dur /= runs+ ops = m * n * k * 2+ memsize = tar_activation.element_size() * m * k + tar_dst.element_size() * m * n+ blks = k // blocksize+ if weight_type == "int4":+ memsize += n * k // 2+ if weight_type == "int2":+ memsize += n * k // 4+ if weight_type in {"int8", "fp8_e4m3", "fp8_e5m2"}:+ memsize += n * k+ memsize += blks * n * tar_activation.element_size()++ gflops = ops / dur / 1e9+ bandwidth = memsize / dur / 1e9+ print(f"[Performance] Time: {dur*1000:.4f} ms")+ print(f" GFLOPS: {gflops:.2f}")+ print(f" Bandwidth: {bandwidth:.2f} GB/s")+ # record_property("GFLOPS", round(gflops, 2))+ # record_property("Bandwidth_GBs", round(bandwidth, 2))+ # record_property("Latency_ms", round(dur * 1000, 4))++[email protected]("m", [1, 8, 16, 1024])[email protected]("n, k", [(1024, 768)])[email protected]("blocksize", [32, 128])[email protected]("weight_type", ["int2", "int4", "int8"])[email protected]("scale_type", ["fp32"])[email protected]("compute_type", ["int8", "bf16", "fp32"])[email protected]("asym", [True, False])+def test_cpu(m, n, k, blocksize, compute_type, weight_type, scale_type, asym):+ main_op(m, n, k, blocksize, compute_type, weight_type, scale_type, asym, "cpu", True)++[email protected]("m", [1, 8, 16, 32, 64, 128, 256, 1024])[email protected]("n, k", [(1024, 768)])[email protected]("blocksize", [32, 128])[email protected]("weight_type", ["int2", "int4", "int8"])[email protected]("scale_type, compute_type", [("fp16", "fp16"), ("fp32", "fp32"), ("fp16", "int8")])[email protected]("asym", [True, False])+def test_xpu(m, n, k, blocksize, compute_type, weight_type, scale_type, asym):+ if not torch.xpu.is_available():+ pytest.skip("No XPU Device")+ with torch.no_grad():+ main_op(m, n, k, blocksize, compute_type, weight_type, scale_type, asym, "xpu", True)+ torch.xpu.empty_cache()++[email protected]("m", [1, 1024])[email protected]("n, k", [(1024, 768)])[email protected]("blocksize", [32, 128])[email protected]("weight_type", ["fp8_e4m3", "fp8_e5m2"])+ "scale_type, compute_type", [("fp16", "fp16"), ("fp32", "fp32"), ("fp8_e8m0", "fp16"), ("fp8_e8m0", "fp32")]+)[email protected]("asym", [False])+def test_xpu_fp8(m, n, k, blocksize, compute_type, weight_type, scale_type, asym):+ if not torch.xpu.is_available():+ pytest.skip("No XPU Device")+ with torch.no_grad():+ main_op(m, n, k, blocksize, compute_type, weight_type, scale_type, asym, "xpu", True)+ torch.xpu.empty_cache()+++# LOCAL_TEST: python test_weightonly.py+# CI pytest: pytest -v test_weightonly.py -k 'test_'+# pytest -v test_weightonly.py -k 'test_' --csv results_woq.csv --csv-columns file,function,parameters_as_columns,properties_as_columns+if __name__ == "__main__":+ # main_op(1, 1024, 768, 32, "int8", "int2", "fp32", False, "cpu")+ # main_op(1, 4096, 4096, 128, "fp32", "int4", "fp32", False, "cpu")+ # main_op(1, 4096, 4096, 128, "int8", "int4", "fp32", False, "cpu")+ main_op(1024, 4096, 4096, 32, "fp32", "int8", "fp32", False, "cpu")+ main_op(1024, 4096, 4096, 32, "fp32", "int8", "fp32", False, "cpu")+ main_op(1024, 4096, 4096, 128, "int8", "int8", "fp32", False, "cpu")++ # main_op(1, 4096, 4096, 32, "fp32", "int4", "fp32", False, "xpu")+ # main_op(1, 4096, 4096, 32, "fp16", "int4", "fp16", False, "xpu")+ # main_op(1, 4096, 4096, 128, "fp16", "int4", "fp16", False, "xpu")+ # main_op(1, 4096, 4096, 32, "fp32", "int8", "fp32", False, "xpu")+ # main_op(1, 4096, 4096, 32, "fp16", "int8", "fp16", False, "xpu")+ # main_op(1, 4096, 4096, 128, "fp16", "int8", "fp16", False, "xpu")+ # main_op(1, 4096, 4096, 32, "fp32", "fp8_e4m3", "fp32", False, "xpu")+ # main_op(1, 4096, 4096, 32, "fp16", "fp8_e4m3", "fp16", False, "xpu")+ # main_op(1, 4096, 4096, 32, "fp32", "fp8_e5m2", "fp32", False, "xpu")+ # main_op(1, 4096, 4096, 32, "fp16", "fp8_e5m2", "fp16", False, "xpu")+ # main_op(1, 4096, 4096, 32, "fp32", "fp8_e4m3", "fp8_e8m0", False, "xpu")+ # main_op(1, 4096, 4096, 32, "fp16", "fp8_e4m3", "fp8_e8m0", False, "xpu")+ # main_op(1, 4096, 4096, 32, "fp16", "fp8_e5m2", "fp8_e8m0", False, "xpu")++ # main_op(4096, 4096, 4096, 32, "fp16", "int4", "fp16", False, "xpu")+ # main_op(4096, 4096, 4096, 4096, "int8", "int4", "fp32", False, "cpu")+ # main_op(4096, 4096, 4096, 32, "auto", "int4", "fp32", False, "cpu")+ # main_op(1, 1024, 768, 32, "int8", "int2", "fp32", False, "cpu")+ main_op(1024, 768, 768, 64, "int8", "int2", "fp32", False, "xpu")+ main_op(1024, 768, 768, 64, "int8", "int4", "fp32", False, "xpu")+ main_op(1024, 768, 3072, 64, "int8", "int4", "fp32", False, "xpu")+ main_op(1024, 3072, 768, 64, "int8", "int4", "fp32", False, "xpu")+ # main_op(4096, 4096, 4096, 4096, "int8", "int4", "fp16", False, "xpu")+ # main_op(4096, 4096, 4096, 32, "int8", "int4", "fp16", False, "xpu")+ # main_op(4096, 4096, 4096, 32, "fp16", "int4", "fp16", False, "xpu")+ # main_op(4096, 4096, 4096, 32, "fp16", "int8", "fp16", False, "xpu")+ # main_op(4096, 4096, 4096, 32, "fp32", "fp8_e4m3", "fp8_e8m0", False, "xpu")+ # main_op(4096, 4096, 4096, 32, "fp16", "fp8_e5m2", "fp8_e8m0", False, "xpu")++ # main_op(2048, 4096, 4096, 32, "fp32", "int4", "fp32", False, True, "xpu")+ # main_op(4096, 4096, 4096, 32, "fp16", "int4", "fp16", False, True, "xpu")+ # main_op(4096, 4096, 4096, 128, "fp16", "int4", "fp16", False, True, "xpu")++ # main_op(1024, 1024, 1024, 32, "fp32", "int8", "fp32", False, True, "xpu")+ # main_op(1024, 1024, 1024, 32, "fp16", "int8", "fp16", False, True, "xpu")+ # main_op(1024, 1024, 1024, 128, "fp16", "int8", "fp16", False, True, "xpu")++ # print("XPU FP8 TEST")++ # main_op(1024, 4096, 4096, 32, "fp32", "fp8_e4m3", "fp32", False, "xpu")+ # main_op(1024, 4096, 4096, 32, "fp16", "fp8_e4m3", "fp16", False, "xpu")++ # main_op(1024, 4096, 4096, 32, "fp32", "fp8_e5m2", "fp32", False, "xpu")+ # main_op(1024, 4096, 4096, 32, "fp16", "fp8_e5m2", "fp16", False, "xpu")+ # main_op(4096, 4096, 4096, 32, "fp32", "fp8_e5m2", "fp32", False, "xpu")+ # main_op(4096, 4096, 4096, 32, "fp16", "fp8_e5m2", "fp16", False, "xpu")diff --git a/auto_round_extension/ark/test/ut_utils.py b/auto_round_extension/ark/test/ut_utils.pynew file mode 100644index 000000000..c4e9e648e--- /dev/null+++ b/auto_round_extension/ark/test/ut_utils.py@@ -0,0 +1,165 @@+#!/usr/bin/env python+# -*- coding: utf-8 -*-+#+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import inspect+import time+from functools import wraps++import auto_round_kernel+import pytest+import torch+++def is_xpu_available():+ return torch.xpu.is_available()+++def capture_args(f):+ @wraps(f)+ def wrapper(*args, **kwargs):+ sig = inspect.signature(f)+ bound_args = sig.bind(*args, **kwargs)+ bound_args.apply_defaults()+ arg_strs = []+ for name, value in bound_args.arguments.items():+ arg_strs.append(f"{name}={value}")+ result = ", ".join(arg_strs)+ print(result)+ return f(*args, **kwargs)++ return wrapper+++def get_torch_dt(dtstr):+ if dtstr == "fp16":+ return torch.float16+ if dtstr == "bf16":+ return torch.bfloat16+ return torch.float32+++def get_scale_torch_dt(scale_type: str):+ if scale_type in {"fp16", "bf16"}:+ return get_torch_dt(scale_type)+ # fp8_e8m0 scales are passed as float32 exponents (int-valued)+ return torch.float32+++def compare2(a, b):+ diff = abs(a - b)+ print(diff.max(), diff.mean(), a.max(), a.mean())+++def gen_weis8(weight_type, device, k, n):+ if weight_type == "int4":+ raw_s8_wei = torch.randint(-8, 7, (k, n), dtype=torch.int8, device=device)+ if weight_type == "int2":+ raw_s8_wei = torch.randint(-2, 1, (k, n), dtype=torch.int8, device=device)+ if weight_type == "int8":+ raw_s8_wei = torch.randint(-128, 127, (k, n), dtype=torch.int8, device=device)+ return raw_s8_wei+++def decode_fp8_to_float(raw_bytes: torch.Tensor, fmt: str) -> torch.Tensor:+ """Decode FP8 E4M3/E5M2 payloads into float32 tensors"""+ x = raw_bytes.to(torch.uint8).to(torch.int32)+ sign = (x >> 7) & 0x1+ if fmt == "fp8_e4m3":+ exp = (x >> 3) & 0xF+ mant = x & 0x7+ exp_f = exp.to(torch.float32)+ mant_f = mant.to(torch.float32)+ val = torch.zeros_like(exp_f, dtype=torch.float32)+ exp_max = 0xF+ bias = 7+ sub_mask = (exp == 0) & (mant != 0)+ nan_mask = (exp == exp_max) & (mant == 7)+ norm_mask = (exp != 0) & (exp != exp_max)+ max_mask = (exp == exp_max) & (mant != 7)+ val = torch.where(sub_mask, mant_f * 0.001953125, val)+ norm_val = torch.pow(2.0, exp_f - float(bias)) * (1.0 + mant_f * 0.125)+ val = torch.where(norm_mask, norm_val, val)+ max_base = 2.0 ** (exp_max - bias)+ max_val = max_base * (1.0 + mant_f * 0.125)+ val = torch.where(max_mask, max_val, val)+ val = torch.where(nan_mask, torch.tensor(float("nan"), dtype=torch.float32), val)+ else:+ exp = (x >> 2) & 0x1F+ mant = x & 0x3+ exp_f = exp.to(torch.float32)+ mant_f = mant.to(torch.float32)+ val = torch.zeros_like(exp_f, dtype=torch.float32)+ exp_max = 0x1F+ bias = 15+ sub_mask = (exp == 0) & (mant != 0)+ inf_mask = (exp == exp_max) & (mant == 0)+ nan_mask = (exp == exp_max) & (mant != 0)+ norm_mask = (exp != 0) & (exp != exp_max)+ val = torch.where(sub_mask, mant_f * 0.0000152587890625, val)+ norm_val = torch.pow(2.0, exp_f - float(bias)) * (1.0 + mant_f * 0.25)+ val = torch.where(norm_mask, norm_val, val)+ val = torch.where(inf_mask, torch.tensor(float("inf"), dtype=torch.float32), val)+ val = torch.where(nan_mask, torch.tensor(float("nan"), dtype=torch.float32), val)+ sign_mask = torch.where(sign.bool(), -torch.ones_like(val), torch.ones_like(val))+ return val * sign_mask+++def sample_valid_fp8(shape, fmt: str, device):+ """Generate FP8 payload bytes that avoid NaN encodings."""+ if isinstance(device, str):+ device = torch.device(device)+ sign = torch.randint(0, 2, shape, dtype=torch.int32, device=device)+ if fmt == "fp8_e4m3":+ exp = torch.randint(0, 15, shape, dtype=torch.int32, device=device)+ mant = torch.randint(0, 8, shape, dtype=torch.int32, device=device)+ nan_mask = (exp == 15) & (mant == 7)+ mant = torch.where(nan_mask, torch.randint(0, 7, shape, dtype=torch.int32, device=device), mant)+ payload = (sign << 7) | (exp << 3) | mant+ else:+ exp = torch.randint(1, 31, shape, dtype=torch.int32, device=device)+ mant = torch.randint(0, 4, shape, dtype=torch.int32, device=device)+ nan_mask = (exp == 31) & (mant != 0)+ mant = torch.where(nan_mask, torch.zeros_like(mant), mant)+ payload = (sign << 7) | (exp << 2) | mant+ payload = torch.where(payload >= 128, payload - 256, payload)+ return payload.to(torch.int8)+++def sample_valid_fp8_e8m0_xpu_safe(shape, fmt: str, device, *, exp_range=None):+ """XPU-safe FP8 payload generator for fp8_e8m0 CI.++ CI B580 is sensitive to large temporary allocations on XPU when generating+ int32 random tensors; generate on CPU with uint8 intermediates and copy.+ """+ if isinstance(device, str):+ device = torch.device(device)+ cpu = torch.device("cpu")+ sign = torch.randint(0, 2, shape, dtype=torch.uint8, device=cpu)+ if fmt == "fp8_e4m3":+ low, high = exp_range if exp_range is not None else (0, 15)+ exp = torch.randint(low, high, shape, dtype=torch.uint8, device=cpu)+ mant = torch.randint(0, 8, shape, dtype=torch.uint8, device=cpu)+ nan_mask = (exp == 15) & (mant == 7)+ if nan_mask.any():+ mant[nan_mask] = torch.randint(0, 7, (int(nan_mask.sum().item()),), dtype=torch.uint8, device=cpu)+ payload_u8 = (sign << 7) | (exp << 3) | mant+ else:+ low, high = exp_range if exp_range is not None else (1, 31)+ exp = torch.randint(low, high, shape, dtype=torch.uint8, device=cpu) # avoid exp==31+ mant = torch.randint(0, 4, shape, dtype=torch.uint8, device=cpu)+ payload_u8 = (sign << 7) | (exp << 2) | mant+ return payload_u8.view(torch.int8).to(device)diff --git a/pyproject.toml b/pyproject.tomlindex c275a8b98..9fd3b8337 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -3,7 +3,7 @@ requires = ["setuptools>=64", "wheel"]build-backend = "setuptools.build_meta"[tool.codespell]-skip = 'pyproject.toml,.azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt'+skip = 'pyproject.toml,.azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt,auto_round_extension/ark/*'ignore-words = ".azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt"[tool.isort]@@ -19,7 +19,8 @@ line-length = 120[tool.typos.files]extend-exclude = [".azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt",- ".azure-pipelines/scripts/cuda_unit_test/runpod_manager.py"+ ".azure-pipelines/scripts/cuda_unit_test/runpod_manager.py",+ "auto_round_extension/ark"][tool.typos.default.extend-words]ue = "ue"diff --git a/setup.cfg b/setup.cfgindex f70fc33e8..bbe94f2ba 100644--- a/setup.cfg+++ b/setup.cfg@@ -17,5 +17,3 @@ console_scripts =auto_round_best = auto_round.__main__:run_bestauto-round-light = auto_round.__main__:run_lightauto_round_light = auto_round.__main__:run_light- auto-round-lib-install = auto_round_extension.ark.install_kernel:main-