diff --git a/.gitignore b/.gitignore index 3ba1f9e..ab8b43c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ build +build-*/ install .vscode .vtune diff --git a/CMakeLists.txt b/CMakeLists.txt index fbb707f..a51dc1f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,7 @@ option(VISP_INSTALL_MODELS "Download and install default models" ${VISP_PACKAGE} option(VISP_FMT_LIB "Use external fmt library instead of standard C++ " OFF) option(VISP_STATIC_GGML "Force static build of ggml" OFF) option(VISP_ASAN "Enable AddressSanitizer" OFF) +option(GGML_GTX "Build the GTX NPU ggml backend (depend/ggml-gtx)" OFF) include(GNUInstallDirs) set(VISP_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR} CACHE PATH "Location of header files") @@ -102,6 +103,13 @@ else() set(VISP_NO_WARNINGS -w) endif() +# 설치본은 자기 위치 기준으로 라이브러리를 찾는다. ggml 을 포함한 모든 공유 라이브러리가 +# 같은 디렉터리에 설치되므로 $ORIGIN 하나면 되고, 이 값이 없으면 /lib 가 로더 +# 검색 경로가 아닌 환경에서 실행이 `cannot open shared object file` 로 실패한다. +if(NOT APPLE AND NOT WIN32) + set(CMAKE_INSTALL_RPATH "\$ORIGIN") +endif() + # Dependencies add_subdirectory(depend/stb) @@ -143,6 +151,11 @@ endif() # 갈린다(`char name[GGML_MAX_NAME]` 이 구조체 레이아웃이다). add_subdirectory(depend/llama/ggml) +# GTX NPU 백엔드는 depend/llama 밖(depend/ggml-gtx)에 둔다. depend/llama 가 upstream 태그를 +# 그대로 가리켜야 클론한 사람이 서브모듈을 받을 수 있다(g2c DECISIONS.md 「GGML_MAX_NAME」). +if(GGML_GTX) + add_subdirectory(depend/ggml-gtx) +endif() set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_SAVED}) # Vision.cpp libraries and executables @@ -166,6 +179,10 @@ install(TARGETS visioncpp RUNTIME DESTINATION ${VISP_BIN_INSTALL_DIR} LIBRARY DESTINATION ${VISP_LIB_INSTALL_DIR} ARCHIVE DESTINATION ${VISP_LIB_INSTALL_DIR}) + +# libvisioncpp 는 ggml 라이브러리와 같은 디렉터리에 설치된다. 그 자리를 자기 기준으로 찾게 +# 해야 /lib 가 로더 검색 경로에 없는 환경에서도 열린다. +set_target_properties(visioncpp PROPERTIES INSTALL_RPATH "\$ORIGIN") install(DIRECTORY include/ DESTINATION ${VISP_INCLUDE_INSTALL_DIR}) if(PROJECT_IS_TOP_LEVEL) install(FILES README.md LICENSE DESTINATION .) @@ -176,9 +193,10 @@ endif() install(TARGETS vision-cli RUNTIME DESTINATION ${VISP_BIN_INSTALL_DIR}) -if(VISP_CI OR VISP_DEV) - set_target_properties(vision-cli PROPERTIES INSTALL_RPATH "\$ORIGIN/../${VISP_LIB_INSTALL_DIR}") -endif() +# 설치본은 자기 위치 기준으로 라이브러리를 찾는다. 이 값이 없으면 CMake 가 RPATH 를 비워 +# 설치하고, /lib 가 로더 검색 경로가 아닌 환경에서 실행이 +# `libvisioncpp.so.0: cannot open shared object file` 로 실패한다. +set_target_properties(vision-cli PROPERTIES INSTALL_RPATH "\$ORIGIN/../${VISP_LIB_INSTALL_DIR}") install(DIRECTORY bindings/python DESTINATION . PATTERN "__pycache__" EXCLUDE) diff --git a/README.md b/README.md index eacbe6f..5476299 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,8 @@ New here? [**Getting started**](docs/getting-started.md) walks through a first r | [**Depth-Anything**](#depth-anything-v2) | Depth estimation | CPU, Vulkan | | [**MI-GAN**](#mi-gan) | Inpainting | CPU, Vulkan | | [**ESRGAN**](#real-esrgan) | Super-resolution | CPU, Vulkan | -| [**YOLOv9t**](#yolov9t) | Object detection | CPU | | [**MMDetection** models](docs/mmdet-detectors.md) | Object detection, segmentation, tracking | CPU | -| [**Compiled PyTorch models**](docs/mmdet-detectors.md#models-whose-head-survives-tracing) | Any traceable `nn.Module` — ultralytics YOLO, torchvision · needs the compiler, [GTX_Compiler](https://github.com/Sudo42b/GTX_Compiler), which carries this repository as a submodule | CPU | +| [**Compiled PyTorch models**](docs/mmdet-detectors.md#models-whose-head-survives-tracing) | Any traceable `nn.Module`, e.g. torchvision · needs the compiler, [GTX_Compiler](https://github.com/Sudo42b/GTX_Compiler), which carries this repository as a submodule | CPU | | [_Implement a model [**Guide**]_](docs/model-implementation-guide.md) | | | **Backbones:** SWIN (v1), DINO (v2), TinyViT @@ -136,17 +135,6 @@ vision-cli migan -m MIGAN-512-places2-F16.gguf -i image.png mask.png -o output.p vision-cli esrgan -m RealESRGAN-x4plus_anime-6B-F16.gguf -i input.png -o output.png ``` -#### YOLOv9t - -YOLOv9t object detection (anchor-free, DFL decode). The arch is self-contained -(`ELAN1` / `RepNCSPELAN4` / `SPPELAN` blocks + detection head with `make_anchors` -/ `dist2bbox` / DFL) and builds into `libvisioncpp`. - -[Paper (arXiv)](https://arxiv.org/abs/2402.13616) | [Repository (GitHub)](https://github.com/WongKinYiu/yolov9) | License: GPL-3.0 - -Used via the library API (`visp::yolov9t::yolov9t_forward`) — no dedicated -`vision-cli` subcommand yet. Weights are GGUF (`general.architecture = "yolov9t"`). - Models need to be converted to GGUF before they can be used. This will also rearrange or precompute tensors for more optimal inference. diff --git a/depend/ggml-gtx/CMakeLists.txt b/depend/ggml-gtx/CMakeLists.txt new file mode 100644 index 0000000..969c2bd --- /dev/null +++ b/depend/ggml-gtx/CMakeLists.txt @@ -0,0 +1,43 @@ +# GTX NPU backend for ggml. +# +# It lives here rather than inside depend/llama so that depend/llama can point at an +# upstream tag without local patches. ggml's own registry does not know this backend, +# so visp registers it at startup (load_ggml_backends in src/visp/ml.cpp). + +message(STATUS "Using GTX NPU backend") + +# ggml_add_backend_library 는 GGML_VERSION(_MAJOR) 로 so 버전을 붙인다. 그 변수는 ggml 폴더 +# 안에서만 정의되므로 ggml 타깃에 붙은 값을 가져온다. +get_target_property(GGML_VERSION ggml VERSION) +get_target_property(GGML_VERSION_MAJOR ggml SOVERSION) + +ggml_add_backend_library(ggml-gtx + src/ggml-gtx.cpp + src/pcie_irq_host.cpp + ) + +target_include_directories(ggml-gtx + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CMAKE_SOURCE_DIR}/depend/llama/ggml/src) # ggml-impl.h, ggml-backend-impl.h + +# Consumers of ggml (visioncpp, vision-cli) see this and register the backend. +target_compile_definitions(ggml-gtx PUBLIC GGML_USE_GTX) + +# GTX backend requires Windows (for PCIe driver) or Linux (for /dev/gtx*) +if (WIN32) + # Windows: SetupAPI for device enumeration, could also need custom driver lib + target_link_libraries(ggml-gtx PRIVATE setupapi) + target_compile_definitions(ggml-gtx PRIVATE GGML_GTX_WINDOWS) +elseif (UNIX AND NOT APPLE) + # Linux: mmap for BAR access + target_compile_definitions(ggml-gtx PRIVATE GGML_GTX_LINUX) +else() + message(WARNING "GTX backend may not work on this platform: ${CMAKE_SYSTEM_NAME}") +endif() + +# Shared library export definitions +if (BUILD_SHARED_LIBS) + target_compile_definitions(ggml-gtx PRIVATE GGML_GTX_BUILD) + target_compile_definitions(ggml-gtx PUBLIC GGML_GTX_SHARED) +endif() diff --git a/depend/ggml-gtx/include/ggml-gtx-baremetal.h b/depend/ggml-gtx/include/ggml-gtx-baremetal.h new file mode 100644 index 0000000..ff72774 --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx-baremetal.h @@ -0,0 +1,487 @@ +//================================================================== +// GGML Baremetal - Minimal GGML for RISC-V Baremetal Environment +// +// This is a stripped-down version of GGML designed to run on +// baremetal RISC-V systems without OS support. +// +// Key differences from standard GGML: +// - No dynamic memory allocation (malloc/free) +// - No file I/O +// - No threading +// - Static memory pools +// - Hardware-specific optimizations via GTX intrinsics +//================================================================== + +#ifndef GGML_BAREMETAL_H +#define GGML_BAREMETAL_H + +#include +#include +#include + +#include "ggml.h" + +#ifdef __cplusplus +extern "C" { +#endif + +//================================================================== +// Configuration +//================================================================== + +#define GGML_GTX_MAX_DIMS 4 +#define GGML_GTX_MAX_SRC 10 +#define GGML_GTX_MAX_NAME 32 +#define GGML_GTX_MAX_OP_PARAMS 64 +#define GGML_GTX_MAX_NODES 4096 +#define GGML_GTX_MAX_TENSORS 8192 + +// Memory alignment for tensors +#define GGML_GTX_MEM_ALIGN 64 + +// Default memory pool sizes (can be overridden) +#ifndef GGML_GTX_POOL_SIZE +#define GGML_GTX_POOL_SIZE (64 * 1024 * 1024) // 64 MB +#endif + +#ifndef GGML_GTX_SCRATCH_SIZE +#define GGML_GTX_SCRATCH_SIZE (10 * 1024 * 1024) // n1s1 L2 capacity +#endif + +//================================================================== +// Utility Macros +//================================================================== + +#define GGML_GTX_UNUSED(x) ((void)(x)) +#define GGML_GTX_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1)) +#define GGML_GTX_MIN(a, b) ((a) < (b) ? (a) : (b)) +#define GGML_GTX_MAX(a, b) ((a) > (b) ? (a) : (b)) + +// Assertion macro - can be customized for baremetal +#ifndef GGML_GTX_ASSERT + #ifdef GGML_GTX_DEBUG + #define GGML_GTX_ASSERT(x) do { if (!(x)) { ggml_gtx_abort(__FILE__, __LINE__, #x); } } while(0) + #else + #define GGML_GTX_ASSERT(x) ((void)0) + #endif +#endif + +//================================================================== +// Data Types +//================================================================== + +typedef enum { + GGML_GTX_TYPE_F32 = 0, + GGML_GTX_TYPE_F16 = 1, + GGML_GTX_TYPE_Q4_0 = 2, + GGML_GTX_TYPE_Q4_1 = 3, + GGML_GTX_TYPE_Q5_0 = 6, + GGML_GTX_TYPE_Q5_1 = 7, + GGML_GTX_TYPE_Q8_0 = 8, + GGML_GTX_TYPE_Q8_1 = 9, + GGML_GTX_TYPE_I8 = 24, + GGML_GTX_TYPE_I16 = 25, + GGML_GTX_TYPE_I32 = 26, + GGML_GTX_TYPE_I64 = 27, + GGML_GTX_TYPE_F64 = 28, + GGML_GTX_TYPE_BF16 = 30, + GGML_GTX_TYPE_COUNT, +} ggml_gtx_type; + +//================================================================== +// Operations +//================================================================== + +// 장치로 보내는 연산 번호다. **숫자를 고정해 둔다** — ggml enum(GGML_OP_*) 에서 파생하지 않는다. +// depend/llama 는 upstream 을 따라가고 upstream 은 새 연산을 enum 중간에 끼워 넣기 때문이다. +// 값은 2026-10-26 에 ISS 에서 검증한 판의 ggml 번호를 그대로 굳힌 것이고, 오른쪽 주석이 짝이 되는 +// ggml 연산이다. ggml 연산 → 이 번호 변환은 ggml-gtx.cpp 의 gtx_wire_op_from_ggml_op() 가 이름으로 한다. +// 이미 있는 값을 바꾸지 마라. 새 연산은 맨 뒤에 붙인다. +typedef enum { + GGML_GTX_OP_NONE = 0, // GGML_OP_NONE + GGML_GTX_OP_DUP = 1, // GGML_OP_DUP + GGML_GTX_OP_ADD = 2, // GGML_OP_ADD + GGML_GTX_OP_ADD_ID = 3, // GGML_OP_ADD_ID + GGML_GTX_OP_ADD1 = 4, // GGML_OP_ADD1 + GGML_GTX_OP_ACC = 5, // GGML_OP_ACC + GGML_GTX_OP_SUB = 6, // GGML_OP_SUB + GGML_GTX_OP_MUL = 7, // GGML_OP_MUL + GGML_GTX_OP_DIV = 8, // GGML_OP_DIV + GGML_GTX_OP_SQR = 9, // GGML_OP_SQR + GGML_GTX_OP_SQRT = 10, // GGML_OP_SQRT + GGML_GTX_OP_LOG = 11, // GGML_OP_LOG + GGML_GTX_OP_SIN = 12, // GGML_OP_SIN + GGML_GTX_OP_COS = 13, // GGML_OP_COS + GGML_GTX_OP_SUM = 14, // GGML_OP_SUM + GGML_GTX_OP_SUM_ROWS = 15, // GGML_OP_SUM_ROWS + GGML_GTX_OP_CUMSUM = 16, // GGML_OP_CUMSUM + GGML_GTX_OP_MEAN = 17, // GGML_OP_MEAN + GGML_GTX_OP_REPEAT = 20, // GGML_OP_REPEAT + GGML_GTX_OP_CONCAT = 22, // GGML_OP_CONCAT + GGML_GTX_OP_NORM = 24, // GGML_OP_NORM + GGML_GTX_OP_RMS_NORM = 25, // GGML_OP_RMS_NORM + GGML_GTX_OP_GROUP_NORM = 27, // GGML_OP_GROUP_NORM + GGML_GTX_OP_L2_NORM = 28, // GGML_OP_L2_NORM + GGML_GTX_OP_MUL_MAT = 29, // GGML_OP_MUL_MAT + GGML_GTX_OP_MUL_MAT_ID = 30, // GGML_OP_MUL_MAT_ID + GGML_GTX_OP_OUT_PROD = 31, // GGML_OP_OUT_PROD + GGML_GTX_OP_SCALE = 32, // GGML_OP_SCALE + GGML_GTX_OP_SET = 33, // GGML_OP_SET + GGML_GTX_OP_CPY = 34, // GGML_OP_CPY + GGML_GTX_OP_CONT = 35, // GGML_OP_CONT + GGML_GTX_OP_RESHAPE = 36, // GGML_OP_RESHAPE + GGML_GTX_OP_VIEW = 37, // GGML_OP_VIEW + GGML_GTX_OP_PERMUTE = 38, // GGML_OP_PERMUTE + GGML_GTX_OP_TRANSPOSE = 39, // GGML_OP_TRANSPOSE + GGML_GTX_OP_GET_ROWS = 40, // GGML_OP_GET_ROWS + GGML_GTX_OP_SET_ROWS = 42, // GGML_OP_SET_ROWS + GGML_GTX_OP_DIAG = 43, // GGML_OP_DIAG + GGML_GTX_OP_DIAG_MASK_INF = 44, // GGML_OP_DIAG_MASK_INF + GGML_GTX_OP_DIAG_MASK_ZERO = 45, // GGML_OP_DIAG_MASK_ZERO + GGML_GTX_OP_SOFTMAX = 46, // GGML_OP_SOFT_MAX + GGML_GTX_OP_ROPE = 48, // GGML_OP_ROPE + GGML_GTX_OP_CLAMP = 50, // GGML_OP_CLAMP + GGML_GTX_OP_CONV_TRANSPOSE_1D = 51, // GGML_OP_CONV_TRANSPOSE_1D + GGML_GTX_OP_IM2COL = 52, // GGML_OP_IM2COL + GGML_GTX_OP_IM2COL_3D = 54, // GGML_OP_IM2COL_3D + GGML_GTX_OP_CONV_2D = 55, // GGML_OP_CONV_2D + GGML_GTX_OP_CONV_3D = 56, // GGML_OP_CONV_3D + GGML_GTX_OP_CONV_2D_DW = 57, // GGML_OP_CONV_2D_DW + GGML_GTX_OP_CONV_TRANSPOSE_2D = 58, // GGML_OP_CONV_TRANSPOSE_2D + GGML_GTX_OP_POOL_1D = 59, // GGML_OP_POOL_1D + GGML_GTX_OP_POOL_2D = 60, // GGML_OP_POOL_2D + GGML_GTX_OP_UPSCALE = 62, // GGML_OP_UPSCALE + GGML_GTX_OP_PAD = 63, // GGML_OP_PAD + GGML_GTX_OP_PAD_REFLECT_1D = 64, // GGML_OP_PAD_REFLECT_1D + GGML_GTX_OP_ROLL = 65, // GGML_OP_ROLL + GGML_GTX_OP_ARANGE = 66, // GGML_OP_ARANGE + GGML_GTX_OP_TIMESTEP_EMBEDDING = 67, // GGML_OP_TIMESTEP_EMBEDDING + GGML_GTX_OP_ARGSORT = 68, // GGML_OP_ARGSORT + GGML_GTX_OP_TOP_K = 69, // GGML_OP_TOP_K + GGML_GTX_OP_LEAKY_RELU = 70, // GGML_OP_LEAKY_RELU + GGML_GTX_OP_TRI = 71, // GGML_OP_TRI + GGML_GTX_OP_FILL = 72, // GGML_OP_FILL + GGML_GTX_OP_FLASH_ATTN_EXT = 73, // GGML_OP_FLASH_ATTN_EXT + GGML_GTX_OP_SSM_CONV = 75, // GGML_OP_SSM_CONV + GGML_GTX_OP_SSM_SCAN = 76, // GGML_OP_SSM_SCAN + GGML_GTX_OP_WIN_PART = 77, // GGML_OP_WIN_PART + GGML_GTX_OP_WIN_UNPART = 78, // GGML_OP_WIN_UNPART + GGML_GTX_OP_GET_REL_POS = 79, // GGML_OP_GET_REL_POS + GGML_GTX_OP_ADD_REL_POS = 80, // GGML_OP_ADD_REL_POS + GGML_GTX_OP_GATED_LINEAR_ATTN = 82, // GGML_OP_GATED_LINEAR_ATTN + GGML_GTX_OP_SOLVE_TRI = 84, // GGML_OP_SOLVE_TRI + + // GTX extensions / ggml unary specializations + GGML_GTX_OP_NEG = 97, // GGML_OP_COUNT when frozen + GGML_GTX_OP_ABS, + GGML_GTX_OP_EXP, + GGML_GTX_OP_MAX, + GGML_GTX_OP_MIN, + GGML_GTX_OP_LAYER_NORM, + GGML_GTX_OP_RELU, + GGML_GTX_OP_GELU, + GGML_GTX_OP_SILU, + GGML_GTX_OP_TANH, + GGML_GTX_OP_SIGMOID, + GGML_GTX_OP_ELU, + GGML_GTX_OP_HARDSWISH, + GGML_GTX_OP_HARDSIGMOID, + GGML_GTX_OP_SOFTPLUS, + GGML_GTX_OP_FLASH_ATTN, + GGML_GTX_OP_SGN, + GGML_GTX_OP_STEP, + GGML_GTX_OP_CEIL, + GGML_GTX_OP_FLOOR, + GGML_GTX_OP_TRUNC, + GGML_GTX_OP_ROUND, + GGML_GTX_OP_GELU_QUICK, + GGML_GTX_OP_GLU, + GGML_GTX_OP_SWIGLU, + GGML_GTX_OP_GEGLU, + GGML_GTX_OP_REGLU, + GGML_GTX_OP_GELU_ERF, + GGML_GTX_OP_XIELU, + GGML_GTX_OP_SWIGLU_OAI, + GGML_GTX_OP_GEGLU_ERF, + GGML_GTX_OP_GEGLU_QUICK, + GGML_GTX_OP_EXPM1, + GGML_GTX_OP_ARGMAX_OP, + GGML_GTX_OP_COUNT_EQUAL, + GGML_GTX_OP_RWKV_WKV6, + GGML_GTX_OP_RWKV_WKV7, + + GGML_GTX_OP_COUNT, + GGML_GTX_OP_CONV1D, + GGML_GTX_OP_CONV2D, + GGML_GTX_OP_MAXPOOL1D, + GGML_GTX_OP_MAXPOOL2D, + GGML_GTX_OP_AVG_POOL_1D, + GGML_GTX_OP_AVG_POOL_2D, + GGML_GTX_OP_ARGMAX, + GGML_GTX_OP_ARGMIN, + GGML_GTX_OP_FLATTEN, + GGML_GTX_OP_BATCH_NORM, + GGML_GTX_OP_SELF_ATTENTION, + GGML_GTX_OP_MULTI_HEAD_SELF_ATTENTION, + + + +} ggml_gtx_op; + +//================================================================== +// Status Codes +//================================================================== + +typedef enum { + GGML_GTX_STATUS_SUCCESS = 0, + GGML_GTX_STATUS_ALLOC_FAILED, + GGML_GTX_STATUS_INVALID_PARAM, + GGML_GTX_STATUS_OP_NOT_SUPPORTED, + GGML_GTX_STATUS_COMPUTE_FAILED, +} ggml_gtx_status; + +//================================================================== +// Tensor Structure +//================================================================== + +typedef struct ggml_gtx_tensor { + ggml_gtx_type type; + ggml_gtx_op op; + + int64_t ne[GGML_GTX_MAX_DIMS]; // number of elements + size_t nb[GGML_GTX_MAX_DIMS]; // stride in bytes + + void * data; + + char name[GGML_GTX_MAX_NAME]; + + int32_t op_params[GGML_GTX_MAX_OP_PARAMS / sizeof(int32_t)]; + + struct ggml_gtx_tensor * src[GGML_GTX_MAX_SRC]; + struct ggml_gtx_tensor * view_src; + size_t view_offs; + + // Flags + bool is_param; + bool requires_grad; +} ggml_gtx_tensor; + +//================================================================== +// Computation Graph +//================================================================== + +typedef struct { + int n_nodes; + int n_leafs; + + ggml_gtx_tensor * nodes[GGML_GTX_MAX_NODES]; + ggml_gtx_tensor * leafs[GGML_GTX_MAX_NODES]; + ggml_gtx_tensor * grads[GGML_GTX_MAX_NODES]; +} ggml_gtx_cgraph; + +//================================================================== +// Context (Static Memory Pool Based) +//================================================================== + +typedef struct { + void * mem_buffer; // Memory pool start + size_t mem_size; // Total pool size + size_t mem_used; // Currently used + + int n_tensors; // Number of allocated tensors + ggml_gtx_tensor tensors[GGML_GTX_MAX_TENSORS]; + + // Scratch buffer for intermediate computations + void * scratch_buffer; + size_t scratch_size; + size_t scratch_used; +} ggml_gtx_context; + +//================================================================== +// Memory Pool Configuration +//================================================================== + +typedef struct { + void * mem_buffer; // Pre-allocated memory buffer + size_t mem_size; // Size of buffer + void * scratch_buffer; // Pre-allocated scratch buffer + size_t scratch_size; // Size of scratch buffer +} ggml_gtx_init_params; + +//================================================================== +// Core API +//================================================================== + +// Initialization +ggml_gtx_context * ggml_gtx_init(ggml_gtx_init_params params); +void ggml_gtx_reset(ggml_gtx_context * ctx); // Reset without freeing +size_t ggml_gtx_used_mem(const ggml_gtx_context * ctx); + +// Type utilities +size_t ggml_gtx_type_size(ggml_gtx_type type); +const char * ggml_gtx_type_name(ggml_gtx_type type); +const char * ggml_gtx_op_name(ggml_gtx_op op); +bool ggml_gtx_is_quantized(ggml_gtx_type type); + +// Tensor creation (from static pool) +ggml_gtx_tensor * ggml_gtx_new_tensor(ggml_gtx_context * ctx, ggml_gtx_type type, int n_dims, const int64_t * ne); +ggml_gtx_tensor * ggml_gtx_new_tensor_1d(ggml_gtx_context * ctx, ggml_gtx_type type, int64_t ne0); +ggml_gtx_tensor * ggml_gtx_new_tensor_2d(ggml_gtx_context * ctx, ggml_gtx_type type, int64_t ne0, int64_t ne1); +ggml_gtx_tensor * ggml_gtx_new_tensor_3d(ggml_gtx_context * ctx, ggml_gtx_type type, int64_t ne0, int64_t ne1, int64_t ne2); +ggml_gtx_tensor * ggml_gtx_new_tensor_4d(ggml_gtx_context * ctx, ggml_gtx_type type, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3); + +// Tensor utilities +int64_t ggml_gtx_nelements(const ggml_gtx_tensor * tensor); +size_t ggml_gtx_nbytes(const ggml_gtx_tensor * tensor); +bool ggml_gtx_is_contiguous(const ggml_gtx_tensor * tensor); +void ggml_gtx_set_name(ggml_gtx_tensor * tensor, const char * name); + +// Data access +void ggml_gtx_set_f32(ggml_gtx_tensor * tensor, float value); +float ggml_gtx_get_f32_1d(const ggml_gtx_tensor * tensor, int64_t i); +void ggml_gtx_set_f32_1d(ggml_gtx_tensor * tensor, int64_t i, float value); + +//================================================================== +// Tensor Operations +//================================================================== + +// View operations (no data copy) +ggml_gtx_tensor * ggml_gtx_view_tensor(ggml_gtx_context * ctx, ggml_gtx_tensor * a); +ggml_gtx_tensor * ggml_gtx_view_1d(ggml_gtx_context * ctx, ggml_gtx_tensor * a, int64_t ne0, size_t offset); +ggml_gtx_tensor * ggml_gtx_view_2d(ggml_gtx_context * ctx, ggml_gtx_tensor * a, int64_t ne0, int64_t ne1, size_t nb1, size_t offset); +ggml_gtx_tensor * ggml_gtx_reshape(ggml_gtx_context * ctx, ggml_gtx_tensor * a, ggml_gtx_tensor * b); +ggml_gtx_tensor * ggml_gtx_reshape_2d(ggml_gtx_context * ctx, ggml_gtx_tensor * a, int64_t ne0, int64_t ne1); +ggml_gtx_tensor * ggml_gtx_reshape_3d(ggml_gtx_context * ctx, ggml_gtx_tensor * a, int64_t ne0, int64_t ne1, int64_t ne2); +ggml_gtx_tensor * ggml_gtx_transpose(ggml_gtx_context * ctx, ggml_gtx_tensor * a); +ggml_gtx_tensor * ggml_gtx_permute(ggml_gtx_context * ctx, ggml_gtx_tensor * a, int axis0, int axis1, int axis2, int axis3); +ggml_gtx_tensor * ggml_gtx_cont(ggml_gtx_context * ctx, ggml_gtx_tensor * a); + +// Arithmetic operations +ggml_gtx_tensor * ggml_gtx_add(ggml_gtx_context * ctx, ggml_gtx_tensor * a, ggml_gtx_tensor * b); +ggml_gtx_tensor * ggml_gtx_sub(ggml_gtx_context * ctx, ggml_gtx_tensor * a, ggml_gtx_tensor * b); +ggml_gtx_tensor * ggml_gtx_mul(ggml_gtx_context * ctx, ggml_gtx_tensor * a, ggml_gtx_tensor * b); +ggml_gtx_tensor * ggml_gtx_div(ggml_gtx_context * ctx, ggml_gtx_tensor * a, ggml_gtx_tensor * b); +ggml_gtx_tensor * ggml_gtx_neg(ggml_gtx_context * ctx, ggml_gtx_tensor * a); +ggml_gtx_tensor * ggml_gtx_abs(ggml_gtx_context * ctx, ggml_gtx_tensor * a); +ggml_gtx_tensor * ggml_gtx_sqrt(ggml_gtx_context * ctx, ggml_gtx_tensor * a); +ggml_gtx_tensor * ggml_gtx_exp(ggml_gtx_context * ctx, ggml_gtx_tensor * a); +ggml_gtx_tensor * ggml_gtx_log(ggml_gtx_context * ctx, ggml_gtx_tensor * a); +ggml_gtx_tensor * ggml_gtx_scale(ggml_gtx_context * ctx, ggml_gtx_tensor * a, float s); + +// Reduction operations +ggml_gtx_tensor * ggml_gtx_sum(ggml_gtx_context * ctx, ggml_gtx_tensor * a); +ggml_gtx_tensor * ggml_gtx_sum_rows(ggml_gtx_context * ctx, ggml_gtx_tensor * a); +ggml_gtx_tensor * ggml_gtx_mean(ggml_gtx_context * ctx, ggml_gtx_tensor * a); + +// Matrix operations +ggml_gtx_tensor * ggml_gtx_mul_mat(ggml_gtx_context * ctx, ggml_gtx_tensor * a, ggml_gtx_tensor * b); + +// Normalization +ggml_gtx_tensor * ggml_gtx_norm(ggml_gtx_context * ctx, ggml_gtx_tensor * a, float eps); +ggml_gtx_tensor * ggml_gtx_rms_norm(ggml_gtx_context * ctx, ggml_gtx_tensor * a, float eps); + +// Activation functions +ggml_gtx_tensor * ggml_gtx_relu(ggml_gtx_context * ctx, ggml_gtx_tensor * a); +ggml_gtx_tensor * ggml_gtx_gelu(ggml_gtx_context * ctx, ggml_gtx_tensor * a); +ggml_gtx_tensor * ggml_gtx_silu(ggml_gtx_context * ctx, ggml_gtx_tensor * a); +ggml_gtx_tensor * ggml_gtx_tanh(ggml_gtx_context * ctx, ggml_gtx_tensor * a); +ggml_gtx_tensor * ggml_gtx_softmax(ggml_gtx_context * ctx, ggml_gtx_tensor * a); + +// Position encoding +ggml_gtx_tensor * ggml_gtx_rope(ggml_gtx_context * ctx, ggml_gtx_tensor * a, ggml_gtx_tensor * b, + int n_dims, int mode, int n_ctx_orig, + float freq_base, float freq_scale, + float ext_factor, float attn_factor, + float beta_fast, float beta_slow); + +// Attention +ggml_gtx_tensor * ggml_gtx_flash_attn(ggml_gtx_context * ctx, + ggml_gtx_tensor * q, + ggml_gtx_tensor * k, + ggml_gtx_tensor * v, + bool masked); + +// Misc +ggml_gtx_tensor * ggml_gtx_get_rows(ggml_gtx_context * ctx, ggml_gtx_tensor * a, ggml_gtx_tensor * b); +ggml_gtx_tensor * ggml_gtx_concat(ggml_gtx_context * ctx, ggml_gtx_tensor * a, ggml_gtx_tensor * b, int dim); +ggml_gtx_tensor * ggml_gtx_clamp(ggml_gtx_context * ctx, ggml_gtx_tensor * a, float min, float max); + + +// Convolution +ggml_gtx_tensor * ggml_gtx_conv_1d(ggml_gtx_context * ctx, + ggml_gtx_tensor * x, + ggml_gtx_tensor * w, + int64_t stride, + int64_t padding); + +ggml_gtx_tensor * ggml_gtx_conv_2d(ggml_gtx_context * ctx, + ggml_gtx_tensor * x, + ggml_gtx_tensor * w, + int64_t stride, + int64_t padding); + +// Max Pooling 1D +ggml_gtx_tensor * ggml_gtx_max_pool_1d(ggml_gtx_context * ctx, + ggml_gtx_tensor * x, + int64_t pool_size, + int64_t stride, + int64_t padding); +// Max Pooling 2D +ggml_gtx_tensor * ggml_gtx_max_pool_2d(ggml_gtx_context * ctx, + ggml_gtx_tensor * x,int64_t pool_size_h, + int64_t pool_size_w, int64_t stride_h, int64_t stride_w, + int64_t padding_h, int64_t padding_w); +// Average Pooling 1D +ggml_gtx_tensor * ggml_gtx_avg_pool_1d(ggml_gtx_context * ctx, + ggml_gtx_tensor * x, + int64_t pool_size, + int64_t stride, + int64_t padding); +// Average Pooling 2D +ggml_gtx_tensor * ggml_gtx_avg_pool_2d(ggml_gtx_context * ctx, + ggml_gtx_tensor * x,int64_t pool_size_h, + int64_t pool_size_w, int64_t stride_h, int64_t stride_w, + int64_t padding_h, int64_t padding_w); + +// Leaky ReLU +ggml_gtx_tensor * ggml_gtx_leaky_relu(ggml_gtx_context * ctx, + ggml_gtx_tensor * a, + float alpha); + +// argmax +ggml_gtx_tensor * ggml_gtx_argmax(ggml_gtx_context * ctx, ggml_gtx_tensor * a); +// argmin +ggml_gtx_tensor * ggml_gtx_argmin(ggml_gtx_context * ctx, ggml_gtx_tensor * a); +// flatten +ggml_gtx_tensor * ggml_gtx_flatten(ggml_gtx_context * ctx, ggml_gtx_tensor * a); +// upscale +ggml_gtx_tensor * ggml_gtx_upscale(ggml_gtx_context * ctx, ggml_gtx_tensor * a, int scale_factor); +// batchnorm +ggml_gtx_tensor * ggml_gtx_batch_norm(ggml_gtx_context * ctx, ggml_gtx_tensor * a, ggml_gtx_tensor * mean, ggml_gtx_tensor * var, float eps); +// self-attention +ggml_gtx_tensor * ggml_gtx_self_attention(ggml_gtx_context * ctx, ggml_gtx_tensor * q, ggml_gtx_tensor * k, ggml_gtx_tensor * v); +// multi-head self-attention +ggml_gtx_tensor * ggml_gtx_multi_head_self_attention(ggml_gtx_context * ctx, ggml_gtx_tensor * q, ggml_gtx_tensor * k, ggml_gtx_tensor * v, int num_heads); + + + +//================================================================== +// Computation Graph API +//================================================================== + +void ggml_gtx_graph_init(ggml_gtx_cgraph * graph); +void ggml_gtx_build_forward(ggml_gtx_cgraph * graph, ggml_gtx_tensor * tensor); +ggml_gtx_status ggml_gtx_graph_compute(ggml_gtx_context * ctx, ggml_gtx_cgraph * graph); + +//================================================================== +// Debug / Utility +//================================================================== + +void ggml_gtx_print_tensor(const ggml_gtx_tensor * tensor); +void ggml_gtx_abort(const char * file, int line, const char * msg); + +#ifdef __cplusplus +} +#endif + +#endif // GGML_BAREMETAL_H diff --git a/depend/ggml-gtx/include/ggml-gtx-compute.h b/depend/ggml-gtx/include/ggml-gtx-compute.h new file mode 100644 index 0000000..458066d --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx-compute.h @@ -0,0 +1,1883 @@ +//================================================================== +// GGML Baremetal - GTX NPU Compute API +// +// Native GTX NPU computation interface following the NEST/SPU +// parallel execution model from reference_usage.c +// +// Execution Model: +// __split() +// __start_plan(nest_id) +// __start_shared() // DDR <-> L2 DMA (TMU) +// __credit_ld() // Signal to threads +// __load() / __mcast_g2s() // DDR -> L2 +// __credit_chk() // Wait for threads +// __store() // L2 -> DDR +// __credit_st() // Signal completion +// __end_shared() +// __start_thread(spu_id) // L2 <-> L1 + Compute (SPU) +// __credit_chk() // Wait for shared +// __load() // L2 -> L1 +// // compute (__mm, __add_vv, etc.) +// __store() // L1 -> L2 +// __credit_ld/st() // Signal completion +// __end_thread(spu_id) +// __end_plan(nest_id) +// __join() +// +// Author: Based on reference_usage.c patterns +//================================================================== + +#ifndef GGML_GTX_COMPUTE_H +#define GGML_GTX_COMPUTE_H + +#include "ggml-gtx-baremetal.h" +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +//================================================================== +// Hardware Configuration +//================================================================== + +#ifndef GGML_GTX_NEST_NUM +#define GGML_GTX_NEST_NUM 1 +#endif + +#ifndef GGML_GTX_SPU_NUM_PER_NEST +#define GGML_GTX_SPU_NUM_PER_NEST 1 +#endif + +#ifndef GGML_GTX_DTYPE_SIZE +#define GGML_GTX_DTYPE_SIZE 2 // FP16 +#endif + +#ifndef GGML_GTX_L1_BANK_SIZE +#define GGML_GTX_L1_BANK_SIZE (128 * 1024 * 2 + 2 * 64 * 1024) // (128KB)*2 + 64KB*2=384 KB +#endif + +//================================================================== +// Memory Addresses (from reference_usage.c) +//================================================================== + +// DDR Base Addresses +#define GGML_GTX_DDR_INPUT 0x50000000ULL +#define GGML_GTX_DDR_A 0x50000000ULL +#define GGML_GTX_DDR_B 0x51000000ULL +#define GGML_GTX_DDR_C 0x52000000ULL +#define GGML_GTX_DDR_RESULT 0x60000000ULL + +#ifndef GGML_GTX_FIXED_DDR_BASE +#define GGML_GTX_FIXED_DDR_BASE 0x370000000ULL +#endif +#define GGML_GTX_FIXED_ARENA_ADDR (GGML_GTX_FIXED_DDR_BASE + 0x01100000ULL) +#define GGML_GTX_FIXED_ARENA_SIZE 0x7EA00000ULL +#define GGML_GTX_DDR_TEMP_0 (GGML_GTX_FIXED_ARENA_ADDR + GGML_GTX_FIXED_ARENA_SIZE) +#define GGML_GTX_DDR_TEMP_1 (GGML_GTX_DDR_TEMP_0 + 0x00200000ULL) + +// L2 SPM Base Addresses +#define GGML_GTX_L2_INPUT 0x000000 +#define GGML_GTX_L2_A 0x000000 +#define GGML_GTX_L2_B 0x050000 +#define GGML_GTX_L2_C 0x080000 +#define GGML_GTX_L2_RESULT 0x0A0000 +#define GGML_GTX_L2_BANK_OFFSET 0x100000 + +// L2 SPM for Transpose operations +#define GGML_GTX_L2_A_T 0x000000 +#define GGML_GTX_L2_B_T 0x500000 +#define GGML_GTX_L2_RESULT_T 0xA00000 + +// L1 SPM Bank Address Offsets +// These are the canonical address definitions. +// ggml-gtx-config.h also defines these (as both macros and enum values), +// so we guard with #ifndef to avoid redefinition. +#ifndef GGML_GTX_L1_BANK_A_OFFSET +#define GGML_GTX_L1_BANK_A_OFFSET 0x00000 +#endif +#ifndef GGML_GTX_L1_BANK_B_OFFSET +#define GGML_GTX_L1_BANK_B_OFFSET 0x20000 +#endif +#ifndef GGML_GTX_L1_BANK_C_OFFSET +#define GGML_GTX_L1_BANK_C_OFFSET 0x30000 +#endif +#ifndef GGML_GTX_L1_BANK_R_OFFSET +#define GGML_GTX_L1_BANK_R_OFFSET 0x50000 +#endif + +//================================================================== +// Utility Macros +//================================================================== + +#define GGML_GTX_CEIL_DIV(a, b) (((a) + (b) - 1) / (b)) +#define GGML_GTX_LAST_TILE(x, tile) (((x) % (tile)) == 0 ? (tile) : ((x) % (tile))) +#define GGML_GTX_EFF_MASK_64(n) ((n) == 64 ? UINT64_MAX : ((1ULL << (n)) - 1)) +#define GGML_GTX_EFF_MASK_16(n) ((n) == 16 ? UINT16_MAX : ((1ULL << (n)) - 1)) + +//================================================================== +// Tiling Context +//================================================================== + +// Per-thread tiling information +typedef struct { + uint32_t cal_num; // Number of calculations for this thread + uint32_t iteration; // Number of full iterations + uint32_t remain; // Remaining calculations after iterations + uint64_t offset; // Byte offset in DDR +} ggml_gtx_thread_tile_t; + +// Per-nest tiling information +typedef struct { + uint32_t cal_num; // Number of calculations for this nest + uint8_t eff_thread_int; // Effective number of threads + uint16_t eff_thread_hex; // Bitmask of effective threads + ggml_gtx_thread_tile_t threads[GGML_GTX_SPU_NUM_PER_NEST]; +} ggml_gtx_nest_tile_t; + +// Complete tiling plan for an operation +typedef struct { + // Input dimensions + uint16_t i_row; + uint16_t i_col; + + // Tiling parameters + uint32_t unit_per_thread; // Bytes per calculation unit + uint32_t max_cal_per_thread; // Max calculations that fit in L1 + + // Effective hardware + uint8_t eff_nest_int; // Number of nests to use + uint64_t eff_nest_hex; // Bitmask of effective nests + + // Per-nest information + ggml_gtx_nest_tile_t nests[GGML_GTX_NEST_NUM]; +} ggml_gtx_tile_plan_t; + +// Matrix multiplication tiling (more complex) +typedef struct { + // Matrix dimensions: C[A_ROW, B_ROW] = A[A_ROW, H_COL] x B^T[B_ROW, H_COL] + uint16_t a_row; + uint16_t h_col; + uint16_t b_row; + uint16_t h_col_origin; // Original stride for A + uint16_t res_col_origin; // Original stride for result + + // Tile sizes + uint16_t tile_a_row; + uint16_t tile_h_col; + uint16_t tile_b_row; + + // Tile counts + uint16_t num_tile_row_a; + uint16_t num_tile_col_h; + uint16_t num_tile_row_b; + + // Last tile sizes + uint16_t last_row_a; + uint16_t last_col_h; + uint16_t last_row_b; + + // Effective hardware + uint8_t eff_nest_int; + uint64_t eff_nest_hex; + uint8_t eff_thread_int[GGML_GTX_NEST_NUM]; + uint16_t eff_thread_hex[GGML_GTX_NEST_NUM]; + + // Per-nest/thread tile distribution + uint16_t num_row_a_per_spread[GGML_GTX_NEST_NUM]; + uint16_t num_row_b_per_nest[GGML_GTX_NEST_NUM]; + uint16_t num_row_a_per_thread[GGML_GTX_NEST_NUM][GGML_GTX_SPU_NUM_PER_NEST]; +} ggml_gtx_matmul_tile_t; + +//================================================================== +// Compute Context +//================================================================== + +typedef struct { + // Hardware configuration + uint8_t nest_num; + uint8_t spu_num_per_nest; + + // Statistics + uint64_t total_ops; + uint64_t dma_bytes_in; + uint64_t dma_bytes_out; +} ggml_gtx_compute_ctx_t; + +//================================================================== +// Tiling API +//================================================================== + +// Initialize tiling plan for element-wise operations (add, mul, div, gelu, etc.) +void ggml_gtx_tile_plan_init( + ggml_gtx_tile_plan_t * plan, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// Initialize tiling plan for matrix multiplication +void ggml_gtx_matmul_tile_init( + ggml_gtx_matmul_tile_t * plan, + uint16_t a_row, + uint16_t h_col, + uint16_t b_row, + uint16_t h_col_origin, + uint16_t res_col_origin, + uint16_t tile_a_row, + uint16_t tile_h_col, + uint16_t tile_b_row, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +//================================================================== +// Element-wise Operations (Tensor x Tensor) +//================================================================== + +// Add two tensors: R = A + B +void ggml_gtx_compute_add_tt( + uint64_t ddr_a, + uint64_t ddr_b, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// Subtract two tensors: R = A - B +void ggml_gtx_compute_sub_tt( + uint64_t ddr_a, + uint64_t ddr_b, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// Multiply two tensors: R = A * B +void ggml_gtx_compute_mul_tt( + uint64_t ddr_a, + uint64_t ddr_b, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// Divide two tensors: R = A / B +void ggml_gtx_compute_div_tt( + uint64_t ddr_a, + uint64_t ddr_b, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +//================================================================== +// Element-wise Operations (Tensor x Scalar) +//================================================================== + +// Divide tensor by scalar: R = A / scalar +void ggml_gtx_compute_div_ts( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint16_t scalar, // FP16 format + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// Multiply tensor by scalar: R = A * scalar +void ggml_gtx_compute_mul_ts( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint16_t scalar, // FP16 format + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// Add scalar to tensor: R = A + scalar +void ggml_gtx_compute_add_ts( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint16_t scalar, // FP16 format + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// Scale tensor by scalar and add bias: R = A * scale + bias +void ggml_gtx_compute_scale( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint16_t scale, // FP16 format + uint16_t bias, // FP16 format + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +//================================================================== +// Element-wise Operations (Tensor x Vector - broadcast) +//================================================================== + +// Add vector to each row: R[i] = A[i] + B +void ggml_gtx_compute_add_tv( + uint64_t ddr_a, + uint64_t ddr_b, // 1D vector of size i_col + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// Multiply vector with each row: R[i] = A[i] * B +void ggml_gtx_compute_mul_tv( + uint64_t ddr_a, + uint64_t ddr_b, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +//================================================================== +// Activation Functions +//================================================================== + +// GELU activation: R = gelu(A) +void ggml_gtx_compute_gelu( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// ReLU activation: R = relu(A) +void ggml_gtx_compute_relu( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// SiLU activation: R = silu(A) = A * sigmoid(A) +void ggml_gtx_compute_silu( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// Tanh activation: R = tanh(A) +void ggml_gtx_compute_tanh( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// Sigmoid activation: R = sigmoid(A) +void ggml_gtx_compute_sigmoid( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// SQR: R = A * A +void ggml_gtx_compute_sqr( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// SQRT: R = sqrt(A) +void ggml_gtx_compute_sqrt( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// NEG: R = -A +void ggml_gtx_compute_neg( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// ABS: R = abs(A) +void ggml_gtx_compute_abs( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// EXP: R = exp(A) +void ggml_gtx_compute_exp( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// LOG: R = log(A) +void ggml_gtx_compute_log( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// CLAMP: R = clamp(A, min, max) +void ggml_gtx_compute_clamp( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t min_val, + uint16_t max_val, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// SGN: R = sign(A) — returns -1, 0, or 1 +void ggml_gtx_compute_sgn( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// STEP: R = step(A) — returns 0 if A < 0, 1 otherwise (Heaviside) +void ggml_gtx_compute_step( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// CEIL: R = ceil(A) +void ggml_gtx_compute_ceil( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// FLOOR: R = floor(A) +void ggml_gtx_compute_floor( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// TRUNC: R = trunc(A) — truncate towards zero +void ggml_gtx_compute_trunc( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// ROUND: R = round(A) — round to nearest even +void ggml_gtx_compute_round( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// SUM: R = sum(A) +void ggml_gtx_compute_sum( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// SUM_ROWS: R = sum_rows(A) +void ggml_gtx_compute_sum_rows( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// MEAN: R = mean(A) +void ggml_gtx_compute_mean( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// MAX: Per-row max reduction using __max_vs (FP16 GTX ISA) +void ggml_gtx_compute_max( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// MIN: Per-row min reduction using __min_vs (FP16 GTX ISA) +void ggml_gtx_compute_min( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// ARANGE: Generate sequence [start, start+step, start+2*step, ...] +// No input tensor — output only. All params as FP16 bit patterns. +void ggml_gtx_compute_arange( + uint64_t ddr_result, + uint16_t n_elements, + uint16_t start_fp16, // FP16 bit pattern + uint16_t step_fp16, // FP16 bit pattern + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +//================================================================== +// Normalization +//================================================================== + +// Layer normalization: R = (A - mean) / sqrt(var + epsilon) +void ggml_gtx_compute_layernorm( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint16_t epsilon, // FP16 format + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// Layer normalization with affine: R = scale * ((A - mean) / sqrt(var + eps)) + shift +void ggml_gtx_compute_layernorm_affine( + uint64_t ddr_input, + uint64_t ddr_scale, + uint64_t ddr_shift, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint16_t epsilon, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// RMS normalization: R = A / sqrt(mean(A^2) + epsilon) +void ggml_gtx_compute_rmsnorm( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint16_t epsilon, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// RMS normalization over compact L2-resident rows. +void ggml_gtx_compute_rmsnorm_l2( + uint32_t l2_input, + uint32_t l2_result, + uint16_t i_row, + uint16_t i_col, + uint16_t epsilon, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// RMS normalization with scale: R = (A / sqrt(mean(A^2) + epsilon)) * scale +void ggml_gtx_compute_rmsnorm_scale( + uint64_t ddr_input, + uint64_t ddr_scale, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint16_t epsilon, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +//================================================================== +// Softmax +//================================================================== + +// Row-wise softmax: R[i] = softmax(A[i]) +void ggml_gtx_compute_softmax( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// Row-wise softmax over compact L2-resident rows. +void ggml_gtx_compute_softmax_l2( + uint32_t l2_input, + uint32_t l2_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// Row-wise softmax for ggml_soft_max_ext: R = softmax(A*scale + mask) +void ggml_gtx_compute_softmax_ext( + uint64_t ddr_input, + uint64_t ddr_mask, + uint64_t ddr_result, + uint16_t ne0, + uint16_t ne1, + uint16_t ne2, + uint16_t ne3, + uint32_t dst_nb1, + uint32_t dst_nb2, + uint32_t dst_nb3, + uint16_t mask_ne2, + uint16_t mask_ne3, + uint32_t mask_nb1, + uint32_t mask_nb2, + uint32_t mask_nb3, + float scale, + float max_bias +); + +//================================================================== +// Matrix Multiplication +//================================================================== + +// Matrix multiplication: C = A x B^T +// A: [a_row, h_col], B: [b_row, h_col], C: [a_row, b_row] +void ggml_gtx_compute_matmul( + uint64_t ddr_a, + uint64_t ddr_b, + uint64_t ddr_result, + uint16_t a_row, + uint16_t h_col, + uint16_t b_row, + uint32_t h_col_origin, // Actual row stride of A in memory + uint32_t res_col_origin, // Actual row stride of result in memory + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// Matrix multiplication with bias: C = A x B^T + bias +void ggml_gtx_compute_matmul_bias( + uint64_t ddr_a, + uint64_t ddr_b, + uint64_t ddr_bias, + uint64_t ddr_result, + uint16_t a_row, + uint16_t h_col, + uint16_t b_row, + uint16_t h_col_origin, + uint16_t res_col_origin, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +//================================================================== +// Memory Operations +//================================================================== + +// Copy tensor: R = A +void ggml_gtx_compute_copy( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// CONT: make a tensor contiguous using explicit byte strides from the source view. +void ggml_gtx_compute_cont( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t out_ne0, + uint16_t out_ne1, + uint16_t out_ne2, + uint16_t out_ne3, + uint32_t src_nb0, + uint32_t src_nb1, + uint32_t src_nb2, + uint32_t src_nb3, + uint16_t elem_size +); + +// Transpose: R = A^T (for 2D tensor) +void ggml_gtx_compute_transpose( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint16_t depth, // For 3D transpose + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +//================================================================== +// SAM (Segment Anything Model) Operations +//================================================================== + +// GET_REL_POS: Extract relative position embeddings +// src0: [d, 2*L-1] (FP16 lookup table) +// dst: [d, w, w] (FP16 output) +// For each (i2, i1): pos = (w - i1 - 1) + i2; dst[i2][i1][:] = src0[pos][:] +void ggml_gtx_compute_get_rel_pos( + uint64_t ddr_src, // DDR address of input lookup table + uint64_t ddr_dst, // DDR address of output tensor + uint16_t d, // Embedding dimension (ne0) + uint16_t w, // Window size (ne1 = ne2) + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// WIN_PART: Partition 2D spatial feature map into windows +// src0: [d, w_in, h_in] (FP16, 2D spatial) +// dst: [d, w, w, npx*npy] (FP16, windowed output, must be pre-zeroed) +// npx = ceil(w_in/w), npy = ceil(h_in/w) +// Padding positions (outside src bounds) are left as zero. +void ggml_gtx_compute_win_part( + uint64_t ddr_src, // DDR address of input tensor + uint64_t ddr_dst, // DDR address of output tensor (pre-zeroed) + uint16_t d, // Embedding dimension (ne0) + uint16_t w_in, // Input width (ne1 of src) + uint16_t h_in, // Input height (ne2 of src) + uint16_t w, // Window size + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// WIN_UNPART: Reconstruct 2D spatial feature map from windowed partitions +// src0: [d, w, w, npx*npy] (FP16, windowed partitions) +// dst: [d, w0, h0] (FP16, reconstructed spatial) +// npx = ceil(w0/w) +// Reverse of WIN_PART: gathers rows from windows back to spatial positions. +void ggml_gtx_compute_win_unpart( + uint64_t ddr_src, // DDR address of input windowed tensor + uint64_t ddr_dst, // DDR address of output spatial tensor + uint16_t d, // Embedding dimension (ne0) + uint16_t w0, // Output width + uint16_t h0, // Output height + uint16_t w, // Window size + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// ADD_REL_POS: Add decomposed 2D relative position bias to attention matrix +// src0 (a): [ne0, ne1, ne2] F32 attention matrix (ne0 = w*w) +// src1 (pw): [w, w, nh, np] F32 width relative position bias +// src2 (ph): [w, w, nh, np] F32 height relative position bias +// dst: [ne0, ne1, ne2] F32 output = src0 + bias(src1, src2) +// Constraints: ne10*ne10 == ne0, ne11*ne12 == ne1, ne13 == ne2 +// Uses direct DDR F32 access (no GTX ISA — ISA supports FP16 only). +void ggml_gtx_compute_add_rel_pos( + uint64_t ddr_src0, // DDR address of attention matrix (F32) + uint64_t ddr_src1, // DDR address of width position bias (F32) + uint64_t ddr_src2, // DDR address of height position bias (F32) + uint64_t ddr_dst, // DDR address of output (F32) + uint16_t ne10, // Window size w (ne10 = ne11 = w) + uint16_t ne11, // Same as ne10 (w) + uint16_t ne12, // Number of heads + uint16_t ne13, // Number of patches + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +//================================================================== +// Diffusion / Mamba Operations +//================================================================== + +// TIMESTEP_EMBEDDING: Sinusoidal timestep embeddings for diffusion models +// src0: [n_timesteps] (F32 timestep values) +// dst: [dim, n_timesteps] (F32 embeddings) +// For j in [0, dim/2): embed[j] = cos(t*freq), embed[j+half] = sin(t*freq) +// where freq = exp(-log(max_period) * j / half) +void ggml_gtx_compute_timestep_embedding( + uint64_t ddr_src, // DDR address of timestep values (F32) + uint64_t ddr_dst, // DDR address of output embeddings (F32) + uint16_t n_timesteps, // Number of timesteps + uint16_t dim, // Embedding dimension + uint32_t max_period, // Max period (typically 10000) + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// SSM_CONV: 1D depthwise convolution for Mamba SSM +// src0 (conv_x): [ncs, nr, n_s] F32 sliding window input +// src1 (weight): [nc, nr] F32 conv1d weight +// dst: [nr, n_t, n_s] F32 output +// nc = d_conv, ncs = d_conv-1+n_t, nr = d_inner +void ggml_gtx_compute_ssm_conv( + uint64_t ddr_src0, // DDR address of conv_x (F32) + uint64_t ddr_src1, // DDR address of weight (F32) + uint64_t ddr_dst, // DDR address of output (F32) + uint16_t nc, // d_conv (kernel size) + uint16_t ncs, // d_conv - 1 + n_t (input width) + uint16_t nr, // d_inner + uint16_t n_t, // tokens per sequence + uint16_t n_s, // number of sequences + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// SSM_SCAN: Selective State Space Model scan (Mamba) +// 7 source tensors: s, x, dt, A, B, C, ids +// dst = [y_output, updated_state] +// Supports both Mamba-1 (per-state A) and Mamba-2 (scalar A). +void ggml_gtx_compute_ssm_scan( + uint64_t ddr_s, // DDR: previous state [nc, nr, nh, ns+] F32 + uint64_t ddr_x, // DDR: input [nr, nh, nt, ns] F32 + uint64_t ddr_dt, // DDR: timestep [nh, nt, ns] F32 + uint64_t ddr_A, // DDR: decay [1,nh] or [nc,nh] F32 + uint64_t ddr_B, // DDR: input gate [nc, ng, nt, ns] F32 + uint64_t ddr_C, // DDR: output gate [nc, ng, nt, ns] F32 + uint64_t ddr_ids, // DDR: sequence ids [ns] int32 + uint64_t ddr_dst, // DDR: output [y + s_new] F32 + uint16_t nc, // d_state + uint16_t nr, // dim + uint16_t nh, // n_head + uint16_t ng, // n_group + uint16_t nt, // tokens per sequence + uint16_t ns, // number of sequences + uint16_t a_ne0, // A.ne[0]: 1 for Mamba-2, nc for Mamba-1 + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// RoPE: Rotary Position Embedding +// src0: [ne0, ne1, ne2, ne3] F32 input tensor +// src1: [ne2] int32 position indices +// dst: [ne0, ne1, ne2, ne3] F32 output +// Supports NORMAL (mode=0) and NEOX (mode=2) modes. +void ggml_gtx_compute_rope( + uint64_t ddr_src0, // DDR address of input tensor (F32) + uint64_t ddr_src1, // DDR address of position indices (int32) + uint64_t ddr_dst, // DDR address of output (F32) + uint16_t ne0, // Inner dimension + uint16_t ne1, // Number of heads + uint16_t ne2, // Sequence length + uint16_t ne3, // Batch size + uint16_t n_dims, // Number of rotary dimensions + uint8_t mode, // 0=NORMAL, 2=NEOX + uint32_t freq_base_b, // Base frequency as F32 bits (default 0x461C4000 = 10000.0) + uint32_t freq_scale_b, // Frequency scale as F32 bits (default 0x3F800000 = 1.0) + uint32_t attn_factor_b, // Attention factor as F32 bits (default 0x3F800000 = 1.0) + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// CONV_TRANSPOSE_1D: 1D transposed convolution (deconvolution) +// src0 (kernel): [ne00, ne01, ne02] = [K, Cout, Cin] F32 +// src1 (input): [ne10, ne11] = [L, Cin] F32 +// dst: [L_out, Cout] F32, L_out = (L-1)*s0 + K +// Uses DDR_TEMP_0/1 for workspace (kernel/input rearrangement). +void ggml_gtx_compute_conv_transpose_1d( + uint64_t ddr_src0, // DDR address of kernel (F32) + uint64_t ddr_src1, // DDR address of input (F32) + uint64_t ddr_dst, // DDR address of output (F32) + uint16_t ne00, // Kernel size (K) + uint16_t ne01, // Output channels (Cout) + uint16_t ne02, // Input channels (Cin) + uint16_t ne10, // Input length (L) + uint16_t ne11, // Input channels (Cin, == ne02) + uint16_t s0, // Stride + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +//================================================================== +// High-Level Graph Compute +//================================================================== + +// Compute a graph using GTX NPU +ggml_gtx_status ggml_gtx_compute_graph_gtx( + ggml_gtx_context * ctx, + ggml_gtx_cgraph * graph +); + +// Compute a single tensor operation +ggml_gtx_status ggml_gtx_compute_tensor_gtx( + ggml_gtx_context * ctx, + ggml_gtx_tensor * tensor +); + +//================================================================== +// Masking Operations +//================================================================== + +// DIAG_MASK_INF: Set elements above the shifted diagonal to -INF +// Input/output: F32 tensor [nc, nr, nz]. n_past shifts the diagonal. +// Uses CPU volatile pointer writes (no GTX ISA needed). Can be in-place. +void ggml_gtx_compute_diag_mask_inf( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t nc, // number of columns (ne0) + uint16_t nr, // number of rows per head (ne1) + uint16_t nz, // number of heads * batch (ne2 * ne3) + int32_t n_past, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// DIAG_MASK_ZERO: Same as DIAG_MASK_INF but fills with 0.0f +void ggml_gtx_compute_diag_mask_zero( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t nc, + uint16_t nr, + uint16_t nz, + int32_t n_past, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +//================================================================== +// Embedding / Gather Operations +//================================================================== + +// GET_ROWS: Indexed row gather from embedding table. +// data[n_embd, n_vocab, n_channels, n_batches] -> dst[n_embd, n_rows, n_channels, n_batches] +// indices[n_rows, n_channels, n_batches] can be I32 or I64. +void ggml_gtx_compute_get_rows( + uint64_t ddr_data, // Embedding table [n_embd, n_vocab] FP16 + uint64_t ddr_indices, // Row indices [n_rows] I32 + uint64_t ddr_dst, // Output [n_embd, n_rows] FP16 + uint16_t n_embd, // Embedding dimension (ne0) + uint16_t n_vocab, + uint16_t n_rows, + uint16_t n_channels, + uint16_t n_batches, + uint16_t idx_size, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +//================================================================== +// Concatenation / Repeat Operations +//================================================================== + +// CONCAT: Concatenate two tensors along a specified dimension +// Both inputs same type. Output ne[dim] = a_ne[dim] + b_ne[dim]. +// Uses CPU volatile pointer copies. +void ggml_gtx_compute_concat( + uint64_t ddr_a, + uint64_t ddr_b, + uint64_t ddr_dst, + uint16_t ne0, // Output ne0 + uint16_t ne1, // Output ne1 + uint16_t ne2, // Output ne2 + uint16_t ne3, // Output ne3 + uint16_t a_ne_dim, // A's size along concat dim + int32_t dim, // Concat dimension (0-3) + uint16_t elem_size, // Element size in bytes (2=FP16, 4=F32) + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// REPEAT: Tile/broadcast tensor to fill output shape +// Each ne[d] must be integer multiple of ne0[d]. +// Uses CPU volatile pointer copies. +void ggml_gtx_compute_repeat( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t ne00, // Source dimensions + uint16_t ne01, + uint16_t ne02, + uint16_t ne03, + uint16_t ne0, // Output dimensions + uint16_t ne1, + uint16_t ne2, + uint16_t ne3, + uint16_t elem_size, // Element size (2=FP16, 4=F32) + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +//================================================================== +// Compound Activation Operations +//================================================================== + +// GELU_QUICK: x * sigmoid(1.702 * x) +// 3-step GTX ISA: __mul_vs(1.702) → __sigm → __mul_vv with bank remapping +void ggml_gtx_compute_gelu_quick( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// SWIGLU: swish(x) * gate. +// If ddr_gate is 0, each input row is split into [x | gate] and output width is n_cols / 2. +// If ddr_gate is non-zero, x and gate are provided as separate tensors and output width is n_cols. +void ggml_gtx_compute_swiglu( + uint64_t ddr_input, + uint64_t ddr_gate, + uint64_t ddr_result, + uint16_t n_rows, + uint16_t n_cols, + uint8_t swapped, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// Packed SWIGLU over compact L2-resident rows. Input row width is 2 * output width. +void ggml_gtx_compute_swiglu_l2( + uint32_t l2_input, + uint32_t l2_result, + uint16_t n_rows, + uint16_t n_cols, + uint8_t swapped, + uint8_t nest_id, + uint8_t spu_num_per_nest +); + +// GLU variants: split row in half → activation(first_half) * second_half +// glu_type: 0=SWIGLU(SiLU), 1=GEGLU(GELU), 2=REGLU(ReLU) +// swapped: if true, swap gate and input halves +void ggml_gtx_compute_glu( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t n_rows, // Number of rows + uint16_t n_cols, // Full row width (will be split in half) + uint8_t glu_type, // 0=SWIGLU, 1=GEGLU, 2=REGLU + uint8_t swapped, // 0=normal, 1=swapped + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +//================================================================== +// FP16 Conversion Helpers +//================================================================== + +// Convert F32 raw bits (uint32_t) to FP16 — ISS-safe, no float type used +uint16_t ggml_gtx_fp32_bits_to_fp16(uint32_t f32); + +// Convert FP16 to F32 raw bits (uint32_t) — ISS-safe, no float type used +uint32_t ggml_gtx_fp16_to_fp32_bits(uint16_t value); + +// Legacy wrappers using float type — WARNING: fmv.x.w returns 0 on ISS v1.1.4 +uint16_t ggml_gtx_fp32_to_fp16(float value); +float ggml_gtx_fp16_to_fp32(uint16_t value); + +//================================================================== +// Software F32 Arithmetic (uint32_t IEEE-754 bit manipulation) +// +// ISS v1.1.4 cannot convert float<->uint32_t (fmv.x.w returns 0). +// All pre-compute F32 math is done on raw IEEE-754 bits as uint32_t. +//================================================================== + +// Well-known F32 bit constants +#define GGML_GTX_F32B_ZERO 0x00000000u // 0.0f +#define GGML_GTX_F32B_ONE 0x3F800000u // 1.0f +#define GGML_GTX_F32B_NEG_ONE 0xBF800000u // -1.0f +#define GGML_GTX_F32B_TWO 0x40000000u // 2.0f +#define GGML_GTX_F32B_HALF 0x3F000000u // 0.5f + +// Basic F32 bit arithmetic +uint32_t ggml_gtx_f32b_neg(uint32_t a); +uint32_t ggml_gtx_f32b_abs(uint32_t a); +int32_t ggml_gtx_f32b_gt(uint32_t a, uint32_t b); +uint32_t ggml_gtx_f32b_from_int(int32_t n); +int32_t ggml_gtx_f32b_to_int(uint32_t a); +uint32_t ggml_gtx_f32b_add(uint32_t a, uint32_t b); +uint32_t ggml_gtx_f32b_sub(uint32_t a, uint32_t b); +uint32_t ggml_gtx_f32b_mul(uint32_t a, uint32_t b); +uint32_t ggml_gtx_f32b_div(uint32_t a, uint32_t b); + +// Transcendental functions (F32 bits in/out) +uint32_t ggml_gtx_f32b_exp(uint32_t x); // exp(x) +uint32_t ggml_gtx_f32b_log(uint32_t x); // ln(x) +uint32_t ggml_gtx_f32b_softplus(uint32_t x); // log(1 + exp(x)) + +//================================================================== +// FP16 Trigonometric Functions (GTX ISA) +//================================================================== + +// Compute sin(x) for n FP16 elements in L1 Bank A → Bank R +// Uses Taylor/Horner approximation. Must be called within thread scope. +// Bank B, C used as scratch. Input should be in [-π, π]. +void ggml_gtx_sin_v(uint32_t n); + +// Compute cos(x) for n FP16 elements in L1 Bank A → Bank R +// Uses Taylor/Horner approximation. Must be called within thread scope. +// Bank B used as scratch. Input should be in [-π, π]. +void ggml_gtx_cos_v(uint32_t n); + +//================================================================== +// Data Operations (CPU-based, no GTX ISA) +//================================================================== + +// PAD: Zero-pad a tensor on all 4 dimensions. F32 data. +void ggml_gtx_compute_pad( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t src_ne0, + uint16_t src_ne1, + uint16_t src_ne2, + uint16_t src_ne3, + uint16_t dst_ne0, + uint16_t dst_ne1, + uint16_t dst_ne2, + uint16_t dst_ne3, + uint16_t lp0, + uint16_t lp1, + uint16_t lp2, + uint16_t lp3 +); + +// SET: Embed src1 into a sub-view of src0. F32 data. +void ggml_gtx_compute_set( + uint64_t ddr_src0, + uint64_t ddr_src1, + uint64_t ddr_dst, + uint16_t src0_ne0, + uint16_t src0_ne1, + uint16_t src0_ne2, + uint16_t src0_ne3, + uint16_t src1_ne0, + uint16_t src1_ne1, + uint16_t src1_ne2, + uint16_t src1_ne3, + uint32_t nb1, + uint32_t nb2, + uint32_t nb3, + uint32_t offset, + bool inplace +); + +// CPY: Copy a source tensor into destination strides. +void ggml_gtx_compute_cpy( + uint64_t ddr_src, + uint64_t ddr_dst, + uint32_t ne0, + uint32_t ne1, + uint32_t ne2, + uint32_t ne3, + uint32_t src_ne0, + uint32_t src_ne1, + uint32_t src_ne2, + uint32_t src_ne3, + uint32_t src_nb0, + uint32_t src_nb1, + uint32_t src_nb2, + uint32_t src_nb3, + uint32_t dst_nb0, + uint32_t dst_nb1, + uint32_t dst_nb2, + uint32_t dst_nb3, + uint16_t elem_size +); + +// IM2COL: Rearrange input for conv-as-matmul. F32 input, FP16 output. +void ggml_gtx_compute_im2col( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t IW, + uint16_t IH, + uint16_t IC, + uint16_t N, + uint16_t KW, + uint16_t KH, + int16_t s0, + int16_t s1, + int16_t p0, + int16_t p1, + int16_t d0, + int16_t d1, + bool is_2D +); + +// POOL_2D: 2D max or average pooling. F32 data. +// pool_type: 0 = MAX, 1 = AVG +void ggml_gtx_compute_pool_2d( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t W, + uint16_t H, + uint16_t C, + uint16_t N, + uint16_t OW, + uint16_t OH, + uint16_t k0, + uint16_t k1, + int16_t s0, + int16_t s1, + int16_t p0, + int16_t p1, + uint8_t pool_type +); + +// GROUP_NORM: Group normalization (Stable Diffusion UNet). F32 data. +// y = (x - mean) / sqrt(var + eps), no learnable gamma/beta. +void ggml_gtx_compute_group_norm( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t ne0, + uint16_t ne1, + uint16_t ne2, + uint16_t ne3, + int32_t n_groups, + uint32_t eps_bits +); + +// CONV_2D: Direct 2D convolution. F32 data. +// kernel[KW,KH,IC,OC] × input[W,H,C,N] → output[OW,OH,OC,N] +void ggml_gtx_compute_conv_2d( + uint64_t ddr_kernel, + uint64_t ddr_input, + uint64_t ddr_dst, + uint16_t W, + uint16_t H, + uint16_t IC, + uint16_t N, + uint16_t KW, + uint16_t KH, + uint16_t OC, + uint16_t OW, + uint16_t OH, + int16_t s0, + int16_t s1, + int16_t p0, + int16_t p1, + int16_t d0, + int16_t d1 +); + +// CONV_2D_DW: Depthwise 2D convolution. F32 data. +// kernel[KW,KH,1,C] × input[W,H,C,N] → output[OW,OH,C,N] +void ggml_gtx_compute_conv_2d_dw( + uint64_t ddr_kernel, + uint64_t ddr_input, + uint64_t ddr_dst, + uint16_t W, + uint16_t H, + uint16_t C, + uint16_t N, + uint16_t KW, + uint16_t KH, + uint16_t OW, + uint16_t OH, + int16_t s0, + int16_t s1, + int16_t p0, + int16_t p1, + int16_t d0, + int16_t d1 +); + +// FLASH_ATTN_EXT: Fused flash attention (online softmax). F32 Q/K/V/mask. +// q[DK, N, n_head, ne3], k[DK, n_kv, n_head_kv, ne3], +// v[DV, n_kv, n_head_kv, ne3], mask[n_kv, N, ...] (FP16 or NULL) +// dst[DV, n_head, N, ne3] (permuted 0,2,1,3) +// op_params[0]=scale, op_params[1]=max_bias, op_params[2]=logit_softcap +void ggml_gtx_compute_flash_attn_ext( + uint64_t ddr_q, + uint64_t ddr_k, + uint64_t ddr_v, + uint64_t ddr_mask, + uint64_t ddr_dst, + uint16_t DK, + uint16_t DV, + uint16_t N, + uint16_t n_kv, + uint16_t n_head, + uint16_t n_head_kv, + uint16_t ne3, + uint32_t scale_bits, + uint32_t max_bias_bits, + uint32_t logit_softcap_bits +); + +// MUL_MAT_ID: MoE expert-indexed matrix multiply. F32 data. +// as[cols, rows, n_expert] × b[cols, n_expert_used, n_tokens] → dst[rows, n_expert_used, n_tokens] +// ids[n_expert_used, n_tokens] (I32): expert selection indices +void ggml_gtx_compute_mul_mat_id( + uint64_t ddr_as, + uint64_t ddr_b, + uint64_t ddr_ids, + uint64_t ddr_dst, + uint16_t cols, + uint16_t rows, + uint16_t n_expert, + uint16_t n_expert_used, + uint16_t n_tokens +); + +// GATED_LINEAR_ATTN: Gated linear attention (RWKV-style). +// k[S,H,T], v[S,H,T], q[S,H,T], g[S,H,T] (gate), state[S*S*H, n_seqs] +// Output: [S*H, T + S*n_seqs] (concat of output tokens and updated state) +void ggml_gtx_compute_gated_linear_attn( + uint64_t ddr_k, + uint64_t ddr_v, + uint64_t ddr_q, + uint64_t ddr_g, + uint64_t ddr_state, + uint64_t ddr_dst, + uint16_t S, + uint16_t H, + uint16_t T, + uint16_t n_seqs, + uint32_t scale_bits +); + +// LEAKY_RELU: y = x > 0 ? x : negative_slope * x +// GTX ISA: __lrelu (fixed 0.01) or composition for arbitrary slope. +void ggml_gtx_compute_leaky_relu( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest, + uint32_t negative_slope_bits // op_params[0], F32 bits +); + +// ELU: y = x > 0 ? x : exp(x) - 1 +// GTX ISA: Two-plan — Plan 1: exp(x)-1 via __exp_v+__add_vs → TEMP, +// Plan 2: element-wise select max(x, TEMP). +void ggml_gtx_compute_elu( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// HARDSIGMOID: y = clamp((x + 3) / 6, 0, 1) +// GTX ISA: __add_vs(3) → __mul_vs(1/6) → __clamp_min(0) → __clamp_max(1). +void ggml_gtx_compute_hardsigmoid( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// HARDSWISH: y = x * hardsigmoid(x) +// GTX ISA: Two-plan — Plan 1: hardsigmoid → TEMP, Plan 2: mul_tt(x, TEMP). +void ggml_gtx_compute_hardswish( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// SOFTPLUS: y = log(1 + exp(x)) +// GTX ISA: __exp_v → __add_vs(1.0) → __ln_v chain. +void ggml_gtx_compute_softplus( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// SIN: element-wise sin(x). GTX ISA FP16 tile plan with range reduction. +void ggml_gtx_compute_sin( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// COS: element-wise cos(x). GTX ISA FP16 tile plan with range reduction. +void ggml_gtx_compute_cos( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// L2_NORM: per-row L2 normalization. y = x / max(||x||_2, eps). +// GTX ISA: __mul_vv (sqr) + __accum (sum) + sqrt + __div_vs. +// Row-based: i_row = ne1*ne2*ne3, i_col = ne0. +void ggml_gtx_compute_l2_norm( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest, + uint32_t eps_bits +); + +// CUMSUM: per-row cumulative sum. y[i] = sum(x[0..i]). F32 CPU. +void ggml_gtx_compute_cumsum( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t ne0, + uint16_t ne1, + uint16_t ne2, + uint16_t ne3 +); + +// UPSCALE: Nearest-neighbor upscaling. F32 CPU. +// dst[i] = src[floor(i * src_ne / dst_ne)] for each dim. +void ggml_gtx_compute_upscale( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t src_ne0, + uint16_t src_ne1, + uint16_t src_ne2, + uint16_t src_ne3, + uint16_t dst_ne0, + uint16_t dst_ne1, + uint16_t dst_ne2, + uint16_t dst_ne3 +); + +// DIAG: Vector → diagonal matrix. F32 CPU. +// src[N,1,ne2,ne3] → dst[N,N,ne2,ne3], zero off-diagonal. +void ggml_gtx_compute_diag( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t n, + uint16_t ne2, + uint16_t ne3 +); + +// ROLL: Circular shift on all 4 dims. F32 CPU. +// dst[i] = src[(i - shift) mod ne] per dim. +void ggml_gtx_compute_roll( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t ne0, + uint16_t ne1, + uint16_t ne2, + uint16_t ne3, + int32_t s0, + int32_t s1, + int32_t s2, + int32_t s3 +); + +// PAD_REFLECT_1D: Reflect-pad dim 0. F32 CPU. +// Output ne0 = ne0 + p0 + p1. Reflects interior elements. +void ggml_gtx_compute_pad_reflect_1d( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t ne0, + uint16_t ne1, + uint16_t ne2, + uint16_t ne3, + uint16_t p0, + uint16_t p1 +); + +// SET_ROWS: Scatter rows by index (inverse of GET_ROWS). +// data[nc,nr,ne2,ne3] + idx[nr,ne_idx1,ne_idx2] + src_base[nc,n_vocab,ne2,ne3] +// -> dst[nc,n_vocab,ne2,ne3] +// indices can be I32 or I64. +void ggml_gtx_compute_set_rows( + uint64_t ddr_data, + uint64_t ddr_idx, + uint64_t ddr_src_base, + uint64_t ddr_dst, + uint16_t nc, + uint16_t nr, + uint16_t ne2, + uint16_t ne3, + uint16_t n_vocab, + uint16_t ne_idx1, + uint16_t ne_idx2, + uint16_t idx_size +); + +// OUT_PROD: Outer product. dst = src0 @ src1^T. F32 CPU. +// src0[m,n,q0,r0], src1[p,n,q1,r1] → dst[m,p,q1,r1] +// Broadcasting: src0 broadcastable to src1 on dims 2,3. +void ggml_gtx_compute_out_prod( + uint64_t ddr_src0, + uint64_t ddr_src1, + uint64_t ddr_dst, + uint16_t ne00, // m + uint16_t ne01, // n (contracted) + uint16_t ne02, // src0 dim 2 + uint16_t ne03, // src0 dim 3 + uint16_t ne10, // p + uint16_t ne2, // dst dim 2 + uint16_t ne3 // dst dim 3 +); + +// POOL_1D: 1D pooling (max or avg). F32 CPU. +// src[IW,C,ne2,ne3] → dst[OW,C,ne2,ne3] +// pool_type: 0=MAX, 1=AVG +void ggml_gtx_compute_pool_1d( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t IW, + uint16_t C, + uint16_t ne2, + uint16_t ne3, + uint16_t OW, + uint16_t k0, + int16_t s0, + int16_t p0, + uint8_t pool_type +); + +// CONV_TRANSPOSE_2D: Transposed 2D convolution. F32 CPU. +// kernel[KW,KH,Cout,Cin], input[SW,SH,Cin,N] → dst[OW,OH,Cout,N] +// OW = (SW-1)*stride + KW, OH = (SH-1)*stride + KH +void ggml_gtx_compute_conv_transpose_2d( + uint64_t ddr_kernel, + uint64_t ddr_input, + uint64_t ddr_dst, + uint16_t KW, + uint16_t KH, + uint16_t Cout, + uint16_t Cin, + uint16_t SW, + uint16_t SH, + uint16_t N, + uint16_t OW, + uint16_t OH, + int16_t stride +); + +// GELU_ERF: Exact GELU via error function. +// GTX ISA: __gelu activation intrinsic (R→A direction). +void ggml_gtx_compute_gelu_erf( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// XIELU: Xi Exponential Linear Unit (parameterized). F32 CPU. +// Params are pre-computed by graph constructor. +void ggml_gtx_compute_xielu( + uint64_t ddr_src, + uint64_t ddr_dst, + uint32_t n_total, + float alpha_n, + float alpha_p, + float beta, + float eps +); + +// SWIGLU_OAI: OpenAI SwiGLU variant. F32 CPU. +// out_glu = min(x,limit) * sigmoid(alpha * min(x,limit)) +// dst = out_glu * (clamp(g, -limit, limit) + 1) +// Supports single-tensor (row split in half) or split-tensor (src0=x, src1=g). +// For single-tensor: n_cols = full row width, split into two halves. +// For split-tensor: pass src1_addr != 0, n_cols = row width (no halving). +void ggml_gtx_compute_swiglu_oai( + uint64_t ddr_src0, + uint64_t ddr_src1, // 0 if single-tensor (split in half) + uint64_t ddr_dst, + uint16_t n_rows, + uint16_t n_cols, // Full row width (if single-tensor) or per-tensor width (if split) + uint8_t swapped, // 0=normal, 1=swapped (only for single-tensor) + float alpha, // default 1.702 + float limit // default 7.0 +); + +// GEGLU_ERF: Gated GELU with erf activation. F32 CPU. +// dst = (0.5 * x * (1 + erf(x / sqrt(2)))) * g +// Supports single-tensor (row split in half) or split-tensor. +void ggml_gtx_compute_geglu_erf( + uint64_t ddr_src0, + uint64_t ddr_src1, // 0 if single-tensor (split in half) + uint64_t ddr_dst, + uint16_t n_rows, + uint16_t n_cols, // Full row width (if single-tensor) or per-tensor width (if split) + uint8_t swapped // 0=normal, 1=swapped (only for single-tensor) +); + +// GEGLU_QUICK: Gated GELU-quick with ISA. FP16 GTX ISA. +// activation(x) = x * sigmoid(1.702 * x) +// dst = activation(x) * g +// Uses __mul_vs + __sigm + __mul_vv ISA composition. +// Single-tensor only (row split in half, FP16). +void ggml_gtx_compute_geglu_quick( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t n_rows, + uint16_t n_cols, // Full row width (will be split in half) + uint8_t swapped, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +//================================================================== +// Phase 5 Batch 1: Utility / Postprocess Ops +//================================================================== + +// FILL: Fill every element with a constant FP16 value. +// GTX ISA: __fill DMA intrinsic. +void ggml_gtx_compute_fill( + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest, + uint32_t value_bits // op_params[0], F32 bits of fill constant +); + +// EXPM1: y = exp(x) - 1. +// GTX ISA: __exp_v(A→R) + __add_vs(-1.0) composition. +void ggml_gtx_compute_expm1( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest +); + +// ARGMAX: For each row, find the index of the maximum element. +// src0: F32 [ne0, ne1], dst: I32 [ne1] (one index per row). +void ggml_gtx_compute_argmax( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t ne0, // row width + uint16_t ne1 // number of rows +); + +// COUNT_EQUAL: Count elements where src0[i] == src1[i]. +// src0: I32, src1: I32 (same shape). dst: single I64 scalar. +void ggml_gtx_compute_count_equal( + uint64_t ddr_src0, + uint64_t ddr_src1, + uint64_t ddr_dst, + uint32_t total_elements +); + +// TRI: Triangular matrix mask. +// src0: F32 [N, N, ...], dst: same shape. +// tri_type: 0=UPPER_DIAG, 1=UPPER, 2=LOWER_DIAG, 3=LOWER +// Copies elements within the triangle; zeros outside. +void ggml_gtx_compute_tri( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t n, // matrix size (N, must be square ne0==ne1) + uint16_t ne2, + uint16_t ne3, + int32_t tri_type +); + +// ACC: dst = copy(src0); view(dst, offset, strides) += src1 +// op_params: [nb1, nb2, nb3, offset, inplace] +void ggml_gtx_compute_acc( + uint64_t ddr_src0, + uint64_t ddr_src1, + uint64_t ddr_dst, + uint16_t ne00, + uint16_t ne01, + uint16_t ne02, + uint16_t ne03, + uint16_t ne10, + uint16_t ne11, + uint16_t ne12, + uint16_t ne13, + uint32_t nb1, + uint32_t nb2, + uint32_t nb3, + uint32_t offset, + bool inplace +); + +// ADD_ID: dst[row] = src0[row] + src1[ids[row], :] +// Used for MoE routing. src2 = I32 index tensor. +void ggml_gtx_compute_add_id( + uint64_t ddr_src0, + uint64_t ddr_src1, + uint64_t ddr_ids, + uint64_t ddr_dst, + uint16_t ne0, + uint16_t ne1, + uint16_t ne2, + uint16_t ne3 +); + +// ARGSORT: Per-row sort returning I32 indices +// sort_order: 0=ASC, 1=DESC +void ggml_gtx_compute_argsort( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + int32_t sort_order +); + +// TOP_K: Per-row top-k selection returning I32 indices +// k = number of top elements per row +void ggml_gtx_compute_top_k( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint16_t k +); + +// CONV_3D: 3D convolution (direct, no scratch buffer) +// kernel[OC*IC, KD, KH, KW], src[N*IC, ID, IH, IW] +// dst[N*OC, OD, OH, OW] +// op_params: [s0,s1,s2, p0,p1,p2, d0,d1,d2, IC, N, OC] +void ggml_gtx_compute_conv_3d( + uint64_t ddr_kernel, + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t IW, uint16_t IH, uint16_t ID, + uint16_t KW, uint16_t KH, uint16_t KD, + uint16_t OW, uint16_t OH, uint16_t OD, + int16_t s0, int16_t s1, int16_t s2, + int16_t p0, int16_t p1, int16_t p2, + int16_t d0, int16_t d1, int16_t d2, + uint16_t IC, uint16_t N, uint16_t OC +); + +// RWKV_WKV6: RWKV v6 WKV attention mechanism +// 6 inputs: k[T,C], v[T,C], r[T,C], time_faaaa[C], time_decay[T,C], state[head_size,C,n_seqs] +// Output: [T,C] concatenated with new state [head_size,C,n_seqs] +void ggml_gtx_compute_rwkv_wkv6( + uint64_t ddr_k, + uint64_t ddr_v, + uint64_t ddr_r, + uint64_t ddr_time_faaaa, + uint64_t ddr_time_decay, + uint64_t ddr_state, + uint64_t ddr_dst, + uint16_t T, + uint16_t C, + uint16_t HEADS, + uint16_t n_seqs +); + +// RWKV_WKV7: RWKV v7 WKV attention mechanism +// 7 inputs: r[T,C], w[T,C], k[T,C], v[T,C], a[T,C], b[T,C], state[head_size,C,n_seqs] +// Output: [T,C] concatenated with new state [head_size,C,n_seqs] +void ggml_gtx_compute_rwkv_wkv7( + uint64_t ddr_r, + uint64_t ddr_w, + uint64_t ddr_k, + uint64_t ddr_v, + uint64_t ddr_a, + uint64_t ddr_b, + uint64_t ddr_state, + uint64_t ddr_dst, + uint16_t T, + uint16_t C, + uint16_t HEADS, + uint16_t n_seqs +); + +// IM2COL_3D: 3D volume unfold for conv-as-GEMM +// src0 = kernel [OC*IC, KD, KH, KW] (shape only) +// src1 = input [N*IC, ID, IH, IW] +// dst = [N*OD, OH, OW, IC*KD*KH*KW] +// op_params: [s0,s1,s2, p0,p1,p2, d0,d1,d2, IC] +void ggml_gtx_compute_im2col_3d( + uint64_t ddr_src1, + uint64_t ddr_dst, + uint16_t IW, uint16_t IH, uint16_t ID, + uint16_t KW, uint16_t KH, uint16_t KD, + uint16_t OW, uint16_t OH, uint16_t OD, + int16_t s0, int16_t s1, int16_t s2, + int16_t p0, int16_t p1, int16_t p2, + int16_t d0, int16_t d1, int16_t d2, + uint16_t IC, uint16_t N +); + +// SOLVE_TRI: Forward substitution for lower-triangular system Ax=B +// src0 = A [n, n, B1, B2] (lower triangular), src1 = B [k, n, B1, B2] +// dst = X [k, n, B1, B2] where AX = B +void ggml_gtx_compute_solve_tri( + uint64_t ddr_src0, + uint64_t ddr_src1, + uint64_t ddr_dst, + uint16_t n, + uint16_t k, + uint16_t ne02, + uint16_t ne03 +); + +#ifdef __cplusplus +} +#endif + +#endif // GGML_GTX_COMPUTE_H diff --git a/depend/ggml-gtx/include/ggml-gtx-config.h b/depend/ggml-gtx/include/ggml-gtx-config.h new file mode 100644 index 0000000..ca6cb91 --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx-config.h @@ -0,0 +1,318 @@ +/** + * @file ggml_gtx_gtx_config.h + * @brief GTX NPU Hardware Configuration + * + * Defines memory map, NEST/SPU configuration, and peripheral addresses + * for the GTX NPU baremetal environment. + * + * GTX NPU Architecture: + * - 1 NEST (Neural Engine SubsysTem) + * - 1 SPU (Scalar Processing Unit) per NEST + * + * Memory Hierarchy: + * - DDR: Up to 112GB (4 channels) + * - L2: 10MB + * - L1: 384KB + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef GGML_GTX_GTX_CONFIG_H +#define GGML_GTX_GTX_CONFIG_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/*============================================================================= + * System Clock Configuration + *============================================================================*/ + +#define GGML_GTX_XIN_HZ 25000000UL /* 25 MHz external clock */ +#define GGML_GTX_CORE_HZ 800000000UL /* 800 MHz core clock */ +#define GGML_GTX_DDR_HZ 250000000UL /* 250 MHz DDR clock */ +#define GGML_GTX_UART_HZ 50000000UL /* 50 MHz UART clock */ +#define GGML_GTX_TIMER_HZ 10000000UL /* 10 MHz timer clock */ + +/*============================================================================= + * NPU Architecture Configuration + *============================================================================*/ + +#define GGML_GTX_NUM_NESTS 1 /* Number of NESTs */ +#define GGML_GTX_SPUS_PER_NEST 1 /* SPUs per NEST */ +#define GGML_GTX_TOTAL_SPUS 1 /* Total SPUs */ + +/*============================================================================= + * L2 Memory Configuration (Per-NEST Scratchpad) + * + * The n1s1 FPGA exposes one 10MB shared scratchpad. + * All NESTs see the SAME address (0x2000_0000). + * Hardware routes accesses based on NEST context set by __start_plan(nest_id). + *============================================================================*/ + +#define GGML_GTX_L2_BASE 0x20000000UL /* Same for all NESTs */ +#define GGML_GTX_L2_SIZE_PER_NEST (10UL * 1024 * 1024) /* 10MB visible */ +#define GGML_GTX_L2_TOTAL_PHYSICAL GGML_GTX_L2_SIZE_PER_NEST + +/* Get L2 base address (same for all NESTs) */ +static inline uintptr_t ggml_gtx_l2_base(unsigned nest) { + (void)nest; /* NEST selection is via __start_plan(), not address */ + return GGML_GTX_L2_BASE; +} + +/*============================================================================= + * L1 Memory Configuration (Per-SPU Scratchpad) + * + * L1 is a 384KB private scratchpad per SPU, totaling 24MB physical. + * All SPUs see the SAME address (0x1000_0000). + * Hardware routes accesses based on SPU context set by __start_plan(). + * + * Bank Layout (per SPU, all SPUs see same addresses): + * Bank A: 0x1000_0000 - 0x1001_FFFF (128KB) - Input/Weight data + * Bank B: 0x1002_0000 - 0x1002_FFFF (64KB) - Additional input + * Bank C: 0x1003_0000 - 0x1004_FFFF (128KB) - Output data + * Bank R: 0x1005_0000 - 0x1005_FFFF (64KB) - Accumulator/Scratch + *============================================================================*/ + +#define GGML_GTX_L1_BASE 0x10000000UL /* Same for all SPUs */ +#define GGML_GTX_L1_SIZE_PER_SPU (384UL * 1024) /* 384KB visible per SPU */ +#define GGML_GTX_L1_TOTAL_PHYSICAL GGML_GTX_L1_SIZE_PER_SPU + +/* L1 Bank Offsets (within each SPU's 384KB) */ +#define GGML_GTX_L1_BANK_A_OFFSET 0x00000 /* Bank A: 128KB */ +#define GGML_GTX_L1_BANK_A_SIZE (128UL * 1024) +#define GGML_GTX_L1_BANK_B_OFFSET 0x20000 /* Bank B: 64KB */ +#define GGML_GTX_L1_BANK_B_SIZE (64UL * 1024) +#define GGML_GTX_L1_BANK_C_OFFSET 0x30000 /* Bank C: 128KB */ +#define GGML_GTX_L1_BANK_C_SIZE (128UL * 1024) +#define GGML_GTX_L1_BANK_R_OFFSET 0x50000 /* Bank R: 64KB */ +#define GGML_GTX_L1_BANK_R_SIZE (64UL * 1024) + +/* Bank identifiers */ +typedef enum { + GGML_GTX_L1_BANK_A = 0, + GGML_GTX_L1_BANK_B = 1, + GGML_GTX_L1_BANK_C = 2, + GGML_GTX_L1_BANK_R = 3, +} ggml_gtx_l1_bank_t; + +/* Get L1 bank offset */ +static inline uint32_t ggml_gtx_l1_bank_offset(ggml_gtx_l1_bank_t bank) { + static const uint32_t offsets[] = { + GGML_GTX_L1_BANK_A_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, + GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_L1_BANK_R_OFFSET + }; + return (bank < 4) ? offsets[bank] : 0; +} + +/* Get L1 bank size */ +static inline size_t ggml_gtx_l1_bank_size(ggml_gtx_l1_bank_t bank) { + static const size_t sizes[] = { + GGML_GTX_L1_BANK_A_SIZE, + GGML_GTX_L1_BANK_B_SIZE, + GGML_GTX_L1_BANK_C_SIZE, + GGML_GTX_L1_BANK_R_SIZE + }; + return (bank < 4) ? sizes[bank] : 0; +} + +/*============================================================================= + * DDR Memory Configuration + * + * GTX NPU supports up to 4 DDR channels with up to 32GB each (112GB total). + * The simulator uses a unified memory model at 0x80000000. + *============================================================================*/ + +/* DDR Channel Addresses (64-bit) */ +#define GGML_GTX_DDR0_BASE 0x0400000000ULL /* Channel 0: 16GB */ +#define GGML_GTX_DDR1_BASE 0x0800000000ULL /* Channel 1: 32GB */ +#define GGML_GTX_DDR2_BASE 0x1000000000ULL /* Channel 2: 32GB */ +#define GGML_GTX_DDR3_BASE 0x1800000000ULL /* Channel 3: 32GB */ + +#define GGML_GTX_DDR_CHANNEL0_SIZE (16ULL * 1024 * 1024 * 1024) /* 16GB */ +#define GGML_GTX_DDR_CHANNEL1_SIZE (32ULL * 1024 * 1024 * 1024) /* 32GB */ +#define GGML_GTX_DDR_CHANNEL2_SIZE (32ULL * 1024 * 1024 * 1024) /* 32GB */ +#define GGML_GTX_DDR_CHANNEL3_SIZE (32ULL * 1024 * 1024 * 1024) /* 32GB */ +#define GGML_GTX_DDR_TOTAL_SIZE (112ULL * 1024 * 1024 * 1024) /* 112GB */ + +/* Unified memory for simulator (alias at 0x80000000) */ +#define GGML_GTX_UNIFIED_BASE 0x80000000UL +#define GGML_GTX_UNIFIED_SIZE (4ULL * 1024 * 1024 * 1024) /* 4GB */ + +/* Override default memory configuration for ggml_gtx_memory.h */ +#undef GGML_GTX_DDR_BASE +#undef GGML_GTX_DDR_SIZE +#undef GGML_GTX_L2_SIZE +#undef GGML_GTX_L1_SIZE + +#define GGML_GTX_DDR_BASE GGML_GTX_UNIFIED_BASE +#define GGML_GTX_DDR_SIZE GGML_GTX_UNIFIED_SIZE +#define GGML_GTX_L2_SIZE GGML_GTX_L2_SIZE_PER_NEST /* 10MB visible */ +#define GGML_GTX_L1_SIZE GGML_GTX_L1_SIZE_PER_SPU /* 384KB visible */ + +/*============================================================================= + * Peripheral Base Addresses + *============================================================================*/ + +/* On-chip Memory */ +#define GGML_GTX_ROM_RAM0_BASE 0x00000000UL +#define GGML_GTX_ROM_RAM0_SIZE (512UL * 1024 * 1024) /* 512MB */ +#define GGML_GTX_SRAM1_BASE 0x20000000UL +#define GGML_GTX_SRAM1_SIZE (1UL * 1024 * 1024) /* 1MB */ + +/* Peripheral Region (64KB each) */ +#define GGML_GTX_QSPI_BASE 0x30000000UL +#define GGML_GTX_UART_BASE 0x30010000UL +#define GGML_GTX_HOST_IF_BASE 0x30020000UL +#define GGML_GTX_HOST_UART_BASE 0x30030000UL +#define GGML_GTX_HOST_SPI_BASE 0x30040000UL +#define GGML_GTX_TIMER0_BASE 0x30050000UL +#define GGML_GTX_TIMER1_BASE 0x30060000UL +#define GGML_GTX_GPIO_BASE 0x30070000UL +#define GGML_GTX_SYS_CFG_BASE 0x30080000UL +#define GGML_GTX_WATCHDOG_BASE 0x30090000UL +#define GGML_GTX_DDR_MONITOR_BASE 0x300A0000UL +#define GGML_GTX_TEMPSENSOR_BASE 0x300B0000UL +#define GGML_GTX_DMA_CTRL_BASE 0x300C0000UL +#define GGML_GTX_I2C_MASTER_BASE 0x300D0000UL + +/* DDR Controller Addresses */ +#define GGML_GTX_DDR_CTRL0_BASE 0x34000000UL +#define GGML_GTX_DDR_CTRL1_BASE 0x36000000UL +#define GGML_GTX_DDR_CTRL2_BASE 0x38000000UL +#define GGML_GTX_DDR_CTRL3_BASE 0x3A000000UL +#define GGML_GTX_DDR_PHY_OFFSET 0x01000000UL + +/* PCIe Addresses */ +#define GGML_GTX_PCIE_DBI_BASE 0x40000000UL +#define GGML_GTX_PCIE_PHY_BASE 0x40800000UL +#define GGML_GTX_PCIE_CFG_BASE 0x40A00000UL +#define GGML_GTX_GSF_CSR_BASE 0x50000000UL +#define GGML_GTX_PCIE_SLAVE_BASE 0x60000000UL + +/* SNOR (Serial NOR Flash) */ +#define GGML_GTX_SNOR_BASE 0x70000000UL +#define GGML_GTX_SNOR_SIZE (256UL * 1024 * 1024) /* 256MB */ + +/*============================================================================= + * RISC-V Timer Addresses (mtime compatible) + *============================================================================*/ + +#define GGML_GTX_MTIME_BASE 0x00600000UL +#define GGML_GTX_MTIME_CTRL (GGML_GTX_MTIME_BASE + 0x00) +#define GGML_GTX_MTIME_DIV (GGML_GTX_MTIME_BASE + 0x04) +#define GGML_GTX_MTIME_LO (GGML_GTX_MTIME_BASE + 0x08) +#define GGML_GTX_MTIME_HI (GGML_GTX_MTIME_BASE + 0x0C) +#define GGML_GTX_MTIMECMP_LO (GGML_GTX_MTIME_BASE + 0x10) +#define GGML_GTX_MTIMECMP_HI (GGML_GTX_MTIME_BASE + 0x14) + +/*============================================================================= + * IRQ Numbers + *============================================================================*/ + +typedef enum { + /* System Peripherals (0-18) */ + GGML_GTX_IRQ_UART = 0, + GGML_GTX_IRQ_HOST_IF = 1, + GGML_GTX_IRQ_HOST_UART = 2, + GGML_GTX_IRQ_HOST_SPI0 = 3, + GGML_GTX_IRQ_HOST_SPI1 = 4, + GGML_GTX_IRQ_QSPI = 5, + GGML_GTX_IRQ_WATCHDOG = 6, + GGML_GTX_IRQ_TIMER0 = 7, + GGML_GTX_IRQ_TIMER1 = 8, + GGML_GTX_IRQ_TEMPSENSOR0 = 9, + GGML_GTX_IRQ_TEMPSENSOR1 = 10, + GGML_GTX_IRQ_TEMPSENSOR2 = 11, + GGML_GTX_IRQ_TEMPSENSOR3 = 12, + GGML_GTX_IRQ_DMA = 13, + GGML_GTX_IRQ_GPIO0 = 14, + GGML_GTX_IRQ_GPIO1 = 15, + GGML_GTX_IRQ_GPIO2 = 16, + GGML_GTX_IRQ_GPIO3 = 17, + GGML_GTX_IRQ_DDR_MONITOR = 18, + + /* DDR Channel IRQs (19-62) */ + GGML_GTX_IRQ_DDR0_BASE = 19, /* 11 IRQs per channel */ + GGML_GTX_IRQ_DDR1_BASE = 30, + GGML_GTX_IRQ_DDR2_BASE = 41, + GGML_GTX_IRQ_DDR3_BASE = 52, + + /* PCIe and Others (63-69) */ + GGML_GTX_IRQ_PCIE0 = 63, + GGML_GTX_IRQ_PCIE1 = 64, + GGML_GTX_IRQ_PCIE2 = 65, + GGML_GTX_IRQ_PCIE3 = 66, + GGML_GTX_IRQ_I2CM = 67, + GGML_GTX_IRQ_PMU = 68, + GGML_GTX_IRQ_GTX = 69, + + GGML_GTX_IRQ_COUNT = 70, +} ggml_gtx_irq_t; + +#define GGML_GTX_DDR_IRQS_PER_CHANNEL 11 + +/*============================================================================= + * NEST/SPU Utility Functions + *============================================================================*/ + +/* Get NEST ID from SPU ID */ +static inline unsigned ggml_gtx_spu_to_nest(unsigned spu_id) { + return spu_id / GGML_GTX_SPUS_PER_NEST; +} + +/* Get local SPU ID within NEST */ +static inline unsigned ggml_gtx_spu_local_id(unsigned spu_id) { + return spu_id % GGML_GTX_SPUS_PER_NEST; +} + +/* Get global SPU ID from NEST and local ID */ +static inline unsigned ggml_gtx_spu_global_id(unsigned nest, unsigned local_id) { + return nest * GGML_GTX_SPUS_PER_NEST + local_id; +} + +/* Get DDR controller base for a channel */ +static inline uintptr_t ggml_gtx_ddr_ctrl_base(unsigned channel) { + static const uintptr_t bases[] = { + GGML_GTX_DDR_CTRL0_BASE, + GGML_GTX_DDR_CTRL1_BASE, + GGML_GTX_DDR_CTRL2_BASE, + GGML_GTX_DDR_CTRL3_BASE + }; + return (channel < 4) ? bases[channel] : 0; +} + +/* Get DDR PHY base for a channel */ +static inline uintptr_t ggml_gtx_ddr_phy_base(unsigned channel) { + return ggml_gtx_ddr_ctrl_base(channel) + GGML_GTX_DDR_PHY_OFFSET; +} + +/*============================================================================= + * Memory Allocation Budgets + * + * Recommended memory allocation for GGML operations. + *============================================================================*/ + +/* Per-NEST budgets */ +#define GGML_GTX_L2_COMPUTE_BUDGET (8UL * 1024 * 1024) /* 8MB for compute */ +#define GGML_GTX_L2_BUFFER_BUDGET (2UL * 1024 * 1024) /* 2MB for buffering */ + +/* Per-SPU budgets */ +#define GGML_GTX_L1_INPUT_BUDGET (192UL * 1024) /* Banks A+B: 192KB */ +#define GGML_GTX_L1_OUTPUT_BUDGET (128UL * 1024) /* Bank C: 128KB */ +#define GGML_GTX_L1_SCRATCH_BUDGET (64UL * 1024) /* Bank R: 64KB */ + +/* Double buffering tile sizes */ +#define GGML_GTX_L2_TILE_SIZE (2UL * 1024 * 1024) /* 2MB per tile */ +#define GGML_GTX_L1_TILE_SIZE (64UL * 1024) /* 64KB per tile */ + +#ifdef __cplusplus +} +#endif + +#endif /* GGML_GTX_GTX_CONFIG_H */ diff --git a/depend/ggml-gtx/include/ggml-gtx-plan.h b/depend/ggml-gtx/include/ggml-gtx-plan.h new file mode 100644 index 0000000..b5f18f2 --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx-plan.h @@ -0,0 +1,418 @@ +/** + * @file ggml_gtx_plan.h + * @brief GGML Baremetal Plan Execution API + * + * High-level API for NEST/SPU parallel execution. + * + * Execution Model: + * __split() + * __start_plan(nest_id) + * __start_shared(nest_id) + * __credit_ld() // Signal TMU: DDR->L2 load starting + * __load() // DDR -> L2 + * __credit_chk() // Wait for TMU completion + * __store() // L2 -> DDR + * __credit_st() // Signal SMU: L2->DDR store starting + * __end_shared(nest_id) + * __start_thread(spu_id) + * __credit_chk() // Wait for SMU ready + * __load() // L2 -> L1 + * __credit_ld() // Signal TMU: L1->L2 load starting + * // compute + * __store() // L1 -> L2 + * __credit_st() // Signal SMU: L2->L1 store starting + * __end_thread(spu_id) + * __end_plan(nest_id) + * __join() + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef GGML_GTX_PLAN_H +#define GGML_GTX_PLAN_H + +#include +#include +#include "ggml-gtx-config.h" +#include "intrinsics/intrin.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/*============================================================================= + * Plan Context + *============================================================================*/ + +/** + * @brief Plan execution context + */ +typedef struct ggml_gtx_plan_ctx { + uint32_t nest_mask; /* Active NESTs bitmask */ + uint32_t spu_mask[GGML_GTX_NUM_NESTS]; + uint32_t current_nest; /* Current NEST being configured */ + uint32_t current_spu; /* Current SPU being configured */ + bool in_shared; /* Inside shared region */ + bool in_thread; /* Inside thread region */ +} ggml_gtx_plan_ctx_t; + +/** + * @brief Tile descriptor for DMA operations + */ +typedef struct ggml_gtx_tile { + uint64_t ddr_addr; /* DDR address */ + uint32_t l2_offset; /* L2 offset within NEST */ + uint32_t l1_offset; /* L1 offset within SPU */ + uint16_t width; /* Tile width in elements */ + uint16_t height; /* Tile height in rows */ + uint32_t ddr_stride; /* DDR row stride in bytes */ + uint32_t l2_stride; /* L2 row stride in bytes */ + uint32_t l1_stride; /* L1 row stride in bytes */ +} ggml_gtx_tile_t; + +/*============================================================================= + * Plan Initialization + *============================================================================*/ + +/** + * @brief Initialize plan context with all NESTs and SPUs + */ +static inline void ggml_gtx_plan_init_all(ggml_gtx_plan_ctx_t *ctx) { + const uint32_t nest_mask = (1u << GGML_GTX_NUM_NESTS) - 1u; + const uint32_t spu_mask = (1u << GGML_GTX_SPUS_PER_NEST) - 1u; + ctx->nest_mask = nest_mask; + for (uint32_t i = 0; i < GGML_GTX_NUM_NESTS; i++) { + ctx->spu_mask[i] = spu_mask; + } + ctx->current_nest = 0; + ctx->current_spu = 0; + ctx->in_shared = false; + ctx->in_thread = false; +} + +/** + * @brief Initialize plan context with specific NESTs + */ +static inline void ggml_gtx_plan_init(ggml_gtx_plan_ctx_t *ctx, uint32_t nest_mask) { + const uint32_t valid_nests = (1u << GGML_GTX_NUM_NESTS) - 1u; + const uint32_t valid_spus = (1u << GGML_GTX_SPUS_PER_NEST) - 1u; + ctx->nest_mask = nest_mask & valid_nests; + for (uint32_t i = 0; i < GGML_GTX_NUM_NESTS; i++) { + ctx->spu_mask[i] = (ctx->nest_mask & (1u << i)) ? valid_spus : 0u; + } + ctx->current_nest = 0; + ctx->current_spu = 0; + ctx->in_shared = false; + ctx->in_thread = false; +} + +/** + * @brief Set active SPUs for a specific NEST + */ +static inline void ggml_gtx_plan_set_spus(ggml_gtx_plan_ctx_t *ctx, + uint32_t nest_id, + uint32_t spu_mask) { + if (nest_id < GGML_GTX_NUM_NESTS) { + const uint32_t valid_spus = (1u << GGML_GTX_SPUS_PER_NEST) - 1u; + ctx->spu_mask[nest_id] = spu_mask & valid_spus; + } +} + +/*============================================================================= + * Plan Execution Macros (Wrappers around intrinsics) + *============================================================================*/ + +/** + * @brief Start parallel execution across NESTs + * Call before __start_plan() to fork execution + */ +#define ggml_gtx_split() __split() + +/** + * @brief End parallel execution and synchronize + * Call after __end_plan() to join all NESTs + */ +#define ggml_gtx_join() __join() + +/** + * @brief Start plan execution on a NEST + * @param nest_id NEST ID (0-3) + */ +#define ggml_gtx_start_plan(nest_id) __start_p(nest_id) + +/** + * @brief End plan execution on a NEST + * @param nest_id NEST ID (0-3) + */ +#define ggml_gtx_end_plan(nest_id) __end_p(nest_id) + +/** + * @brief Start shared memory region (SMU context) + * Operations in this region use L2 shared memory + */ +#define ggml_gtx_start_shared() __start_s() + +/** + * @brief End shared memory region + */ +#define ggml_gtx_end_shared() __end_s() + +/** + * @brief Start thread execution on an SPU + * @param spu_id Local SPU ID within NEST (0-15) + */ +#define ggml_gtx_start_thread(spu_id) __start_t(spu_id) + +/** + * @brief End thread execution on an SPU + * @param spu_id Local SPU ID within NEST (0-15) + */ +#define ggml_gtx_end_thread(spu_id) __end_t(spu_id) + +/*============================================================================= + * Credit-based Synchronization + *============================================================================*/ + +/** + * @brief Signal load credit (producer ready) + * Called before DMA load to notify consumers not to access the target + * @param target_spu SPU bitmask (for TMU notification) + * @param target_nest NEST bitmask (for inter-NEST notification) + */ +#define ggml_gtx_credit_load(target_spu, target_nest) \ + __credit_ld(target_spu, target_nest) + +/** + * @brief Signal store credit (consumer ready) + * Called before DMA store to notify producers not to access the source + * @param target_spu SPU bitmask + */ +#define ggml_gtx_credit_store(target_spu) \ + __credit_st(target_spu) + +/** + * @brief Wait for credit (synchronization point) + * Blocks until the specified SPUs have completed their operations + * @param target_spu SPU bitmask to wait for + */ +#define ggml_gtx_credit_check(target_spu) \ + __credit_chk(target_spu) + +/*============================================================================= + * DMA Operations with Credit Management + *============================================================================*/ + +/** + * @brief Load tile from DDR to L2 with credit management + * For use in shared region (__start_shared / __end_shared) + */ +static inline void ggml_gtx_tile_ddr_to_l2(const ggml_gtx_tile_t *tile, + uint64_t target_spu_mask) { + /* Signal threads: DDR->L2 load starting */ + __credit_ld(target_spu_mask, 0); + + /* DMA: DDR -> L2 */ + __load(tile->ddr_addr, + GGML_GTX_L2_BASE + tile->l2_offset, + tile->ddr_stride, + tile->width * sizeof(uint16_t), /* Assume FP16 */ + tile->height, + tile->l2_stride); + + /* Memory barrier */ + __msync(); +} + +/** + * @brief Store tile from L2 to DDR with credit management + * For use in shared region + */ +static inline void ggml_gtx_tile_l2_to_ddr(const ggml_gtx_tile_t *tile, + uint64_t target_spu_mask) { + /* Wait for threads to complete their L2 writes */ + __credit_chk(target_spu_mask); + + /* DMA: L2 -> DDR */ + __store(GGML_GTX_L2_BASE + tile->l2_offset, + tile->ddr_addr, + tile->ddr_stride, + tile->width * sizeof(uint16_t), + tile->height, + tile->l2_stride); + + /* Signal threads: L2->DDR store complete */ + __credit_st(target_spu_mask); + + /* Memory barrier */ + __msync(); +} + +/** + * @brief Load tile from L2 to L1 with credit management + * For use in thread region (__start_thread / __end_thread) + */ +static inline void ggml_gtx_tile_l2_to_l1(const ggml_gtx_tile_t *tile, + uint64_t notify_mask) { + /* Wait for SMU to complete L2 preparation */ + __credit_chk(notify_mask); + + /* DMA: L2 -> L1 (internal copy) */ + __copy(GGML_GTX_L2_BASE + tile->l2_offset, + GGML_GTX_L1_BASE + tile->l1_offset, + tile->l2_stride, + tile->width * sizeof(uint16_t), + tile->height, + tile->l1_stride); + + /* Signal SMU: L2->L1 load starting */ + __credit_ld(notify_mask, 0); + + /* Memory barrier */ + __msync(); +} + +/** + * @brief Store tile from L1 to L2 with credit management + * For use in thread region + */ +static inline void ggml_gtx_tile_l1_to_l2(const ggml_gtx_tile_t *tile, + uint64_t notify_mask) { + /* DMA: L1 -> L2 (internal copy) */ + __copy(GGML_GTX_L1_BASE + tile->l1_offset, + GGML_GTX_L2_BASE + tile->l2_offset, + tile->l1_stride, + tile->width * sizeof(uint16_t), + tile->height, + tile->l2_stride); + + /* Signal SMU: L1->L2 store complete */ + __credit_st(notify_mask); + + /* Memory barrier */ + __msync(); +} + +/*============================================================================= + * Double-Buffered Execution Template + *============================================================================*/ + +/** + * @brief Double buffer state + */ +typedef struct ggml_gtx_double_buffer { + uint32_t l2_buf[2]; /* L2 buffer offsets (ping-pong) */ + uint32_t l1_buf[2]; /* L1 buffer offsets (ping-pong) */ + uint32_t buf_size; /* Buffer size in bytes */ + uint32_t current; /* Current buffer index (0 or 1) */ +} ggml_gtx_double_buffer_t; + +/** + * @brief Initialize double buffer + */ +static inline void ggml_gtx_double_buffer_init(ggml_gtx_double_buffer_t *db, + uint32_t l2_base_offset, + uint32_t l1_base_offset, + uint32_t buf_size) { + db->l2_buf[0] = l2_base_offset; + db->l2_buf[1] = l2_base_offset + buf_size; + db->l1_buf[0] = l1_base_offset; + db->l1_buf[1] = l1_base_offset + buf_size; + db->buf_size = buf_size; + db->current = 0; +} + +/** + * @brief Swap double buffer + */ +static inline void ggml_gtx_double_buffer_swap(ggml_gtx_double_buffer_t *db) { + db->current ^= 1; +} + +/** + * @brief Get current L2 buffer offset + */ +static inline uint32_t ggml_gtx_double_buffer_l2(const ggml_gtx_double_buffer_t *db) { + return db->l2_buf[db->current]; +} + +/** + * @brief Get current L1 buffer offset + */ +static inline uint32_t ggml_gtx_double_buffer_l1(const ggml_gtx_double_buffer_t *db) { + return db->l1_buf[db->current]; +} + +/*============================================================================= + * High-Level Plan Execution Helpers + *============================================================================*/ + +/** + * @brief Execute a function on all active NESTs in parallel + * @param ctx Plan context + * @param func Function to execute (receives nest_id) + */ +#define ggml_gtx_foreach_nest(ctx, func) do { \ + ggml_gtx_split(); \ + for (uint32_t _n = 0; _n < GGML_GTX_NUM_NESTS; _n++) { \ + if ((ctx)->nest_mask & (1 << _n)) { \ + ggml_gtx_start_plan(_n); \ + func(_n); \ + ggml_gtx_end_plan(_n); \ + } \ + } \ + ggml_gtx_join(); \ +} while(0) + +/** + * @brief Execute a function on all active SPUs within current NEST + * @param ctx Plan context + * @param nest_id Current NEST ID + * @param func Function to execute (receives spu_id) + */ +#define ggml_gtx_foreach_spu(ctx, nest_id, func) do { \ + for (uint32_t _s = 0; _s < GGML_GTX_SPUS_PER_NEST; _s++) { \ + if ((ctx)->spu_mask[nest_id] & (1 << _s)) { \ + ggml_gtx_start_thread(_s); \ + func(_s); \ + ggml_gtx_end_thread(_s); \ + } \ + } \ +} while(0) + +/*============================================================================= + * Synchronization Barriers + *============================================================================*/ + +/** + * @brief Global barrier across all active NESTs + */ +static inline void ggml_gtx_barrier_global(void) { + __bar(); +} + +/** + * @brief Flush all pending operations + */ +static inline void ggml_gtx_flush(void) { + __flush(); +} + +/** + * @brief Memory synchronization barrier + */ +static inline void ggml_gtx_msync(void) { + __msync(); +} + +/** + * @brief Wait for specified clock cycles + */ +static inline void ggml_gtx_wait_cycles(uint32_t cycles) { + __wait(cycles); +} + +#ifdef __cplusplus +} +#endif + +#endif /* GGML_GTX_PLAN_H */ diff --git a/depend/ggml-gtx/include/ggml-gtx-port.h b/depend/ggml-gtx/include/ggml-gtx-port.h new file mode 100644 index 0000000..1e963fd --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx-port.h @@ -0,0 +1,212 @@ +//================================================================== +// GGML Baremetal - Platform Abstraction Layer +// +// This file provides platform-specific implementations that must +// be customized for your target hardware. +// +// Users should implement these functions for their specific +// baremetal environment. +//================================================================== + +#ifndef GGML_GTX_PORT_H +#define GGML_GTX_PORT_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +//================================================================== +// Memory Configuration +// +// Define these in your linker script or platform header: +// - DDR_BASE: Start address of DDR memory +// - DDR_SIZE: Total DDR size +// - L2_BASE: Start address of L2 scratchpad (if available) +// - L2_SIZE: L2 scratchpad size +// - L1_BASE: Start address of L1 scratchpad (if available) +// - L1_SIZE: L1 scratchpad size +//================================================================== + +#ifndef DDR_BASE +#define DDR_BASE 0x80000000ULL +#endif + +#ifndef DDR_SIZE +#define DDR_SIZE (116ULL * 1024 * 1024 * 1024) // 128 GB +#endif + +#ifndef L2_BASE +#define L2_BASE 0x20000000ULL +#endif + +#ifndef L2_SIZE +#define L2_SIZE (10 * 1024 * 1024) // 10 MB on n1s1 FPGA +#endif + +#ifndef L1_BASE +#define L1_BASE 0x10000000ULL +#endif + +#ifndef L1_SIZE +#define L1_SIZE (384 * 1024) // 384 KB +#endif + +//================================================================== +// Memory Management (Static Pool Based) +// +// These must be implemented by the user. +// Default implementations use simple bump allocators. +//================================================================== + +// Memory pool structure +#define GGML_GTX_MAX_POOL_BLOCKS 256 + +typedef struct { + size_t offset; + size_t size; + bool in_use; +} ggml_gtx_pool_block; + +typedef struct { + uint8_t * base; + size_t size; + size_t used; + size_t alignment; + size_t n_blocks; + ggml_gtx_pool_block blocks[GGML_GTX_MAX_POOL_BLOCKS]; +} ggml_gtx_mem_pool; + +// Initialize a memory pool at the given address +void ggml_gtx_pool_init(ggml_gtx_mem_pool * pool, void * base, size_t size, size_t alignment); + +// Allocate from pool (returns NULL if no space) +void * ggml_gtx_pool_alloc(ggml_gtx_mem_pool * pool, size_t size); + +// Free an allocation from pool (returns false for invalid or double free) +bool ggml_gtx_pool_free(ggml_gtx_mem_pool * pool, void * ptr); + +// Reset pool (mark all memory as free) +void ggml_gtx_pool_reset(ggml_gtx_mem_pool * pool); + +// Get remaining space +size_t ggml_gtx_pool_remaining(const ggml_gtx_mem_pool * pool); + +// Get the largest contiguous free payload that can satisfy an aligned alloc +size_t ggml_gtx_pool_largest_free(const ggml_gtx_mem_pool * pool); + +//================================================================== +// Memory Copy / Set Functions +// +// Optimized implementations for your platform. +// Default implementations use simple loops. +//================================================================== + +void * ggml_gtx_memcpy(void * dst, const void * src, size_t n); +void * ggml_gtx_memset(void * dst, int c, size_t n); +void * ggml_gtx_memset_f32(float * dst, float value, size_t count); +void * ggml_gtx_memset_f16(uint16_t * dst, uint16_t value, size_t count); + +//================================================================== +// DMA Operations (Optional) +// +// For hardware with DMA engines, implement these for faster +// memory transfers. +//================================================================== + +typedef enum { + GGML_GTX_DMA_DDR_TO_L2, + GGML_GTX_DMA_L2_TO_DDR, + GGML_GTX_DMA_DDR_TO_L1, + GGML_GTX_DMA_L1_TO_DDR, + GGML_GTX_DMA_L2_TO_L1, + GGML_GTX_DMA_L1_TO_L2, +} ggml_gtx_dma_dir; + +// Start async DMA transfer +int ggml_gtx_dma_start(void * dst, const void * src, size_t size, ggml_gtx_dma_dir dir); + +// Wait for DMA completion +void ggml_gtx_dma_wait(int dma_handle); + +// Synchronous DMA (start + wait) +void ggml_gtx_dma_copy(void * dst, const void * src, size_t size, ggml_gtx_dma_dir dir); + +//================================================================== +// Timing / Performance Counters +//================================================================== + +// Get current cycle count (for performance measurement) +uint64_t ggml_gtx_get_cycles(void); + +// Get time in microseconds (if RTC available) +uint64_t ggml_gtx_get_time_us(void); + +//================================================================== +// Debug / Logging +//================================================================== + +// Print a string (must be implemented for your UART/console) +void ggml_gtx_print(const char * str); + +// Printf-like function (optional, can be stubbed out) +void ggml_gtx_printf(const char * fmt, ...); + +// Abort execution (for fatal errors) +void ggml_gtx_halt(void); + +//================================================================== +// FP16 <-> FP32 Conversion +// +// If your hardware has native FP16 support, optimize these. +//================================================================== + +uint16_t ggml_gtx_fp32_to_fp16(float f); +float ggml_gtx_fp16_to_fp32(uint16_t h); + +// Vectorized versions (for bulk conversion) +void ggml_gtx_fp32_to_fp16_row(const float * src, uint16_t * dst, size_t n); +void ggml_gtx_fp16_to_fp32_row(const uint16_t * src, float * dst, size_t n); + +//================================================================== +// Hardware-Specific Accelerator Interface +// +// For NPU/accelerator integration (GTX in your case) +//================================================================== + +#ifdef GGML_GTX_USE_GTX + +// Include GTX intrinsics (from local intrinsics folder) +#include "intrinsics/intrin.h" +#include "intrinsics/gtx_utils.h" + +// GTX-accelerated matrix multiplication +void ggml_gtx_gtx_mul_mat( + const void * A, const void * B, void * C, + int M, int K, int N, + int type_a, int type_b, int type_c); + +// GTX-accelerated softmax +void ggml_gtx_gtx_softmax(const void * src, void * dst, int n); + +// GTX-accelerated activation functions +void ggml_gtx_gtx_gelu(const void * src, void * dst, int n); +void ggml_gtx_gtx_silu(const void * src, void * dst, int n); +void ggml_gtx_gtx_relu(const void * src, void * dst, int n); + +// GTX-accelerated normalization +void ggml_gtx_gtx_rms_norm(const void * src, void * dst, int n, float eps); + +// GTX synchronization +void ggml_gtx_gtx_sync(void); + +#endif // GGML_GTX_USE_GTX + +#ifdef __cplusplus +} +#endif + +#endif // GGML_GTX_PORT_H diff --git a/depend/ggml-gtx/include/ggml-gtx-server.h b/depend/ggml-gtx/include/ggml-gtx-server.h new file mode 100644 index 0000000..8cf94ec --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx-server.h @@ -0,0 +1,495 @@ +/** + * @file ggml_gtx_server.h + * @brief GGML Baremetal PCIe Server Header + * + * PCIe-based RPC server for GTX NPU. + * Implements command handling similar to ggml-rpc. + * + * Communication Flow: + * 1. Device waits in WFI (Wait For Interrupt) + * 2. Host sends ELBI interrupt (BAR0 offset 0x0E40) + * 3. Device wakes up, processes message queue + * 4. Device sends MSI to host when complete + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef GGML_GTX_SERVER_H +#define GGML_GTX_SERVER_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/*============================================================================= + * RPC Commands (aligned with ggml-rpc) + *============================================================================*/ + +typedef enum { + /* Buffer Management */ + GGML_GTX_CMD_ALLOC_BUFFER = 0, /* Allocate device memory */ + GGML_GTX_CMD_GET_ALIGNMENT = 1, /* Get buffer alignment requirement */ + GGML_GTX_CMD_GET_MAX_SIZE = 2, /* Get maximum buffer size */ + GGML_GTX_CMD_BUFFER_GET_BASE = 3, /* Get buffer base address */ + GGML_GTX_CMD_FREE_BUFFER = 4, /* Free device memory */ + GGML_GTX_CMD_BUFFER_CLEAR = 5, /* Clear buffer to value */ + + /* Tensor Operations */ + GGML_GTX_CMD_SET_TENSOR = 6, /* Upload tensor data (Host -> Device) */ + GGML_GTX_CMD_SET_TENSOR_HASH = 7, /* Upload with hash validation */ + GGML_GTX_CMD_GET_TENSOR = 8, /* Download tensor data (Device -> Host) */ + GGML_GTX_CMD_COPY_TENSOR = 9, /* Copy tensor on device */ + + /* Compute */ + GGML_GTX_CMD_GRAPH_COMPUTE = 10, /* Execute compute graph */ + + /* Device Info */ + GGML_GTX_CMD_GET_DEVICE_MEMORY = 11, /* Query free/total device memory */ + GGML_GTX_CMD_INIT_TENSOR = 12, /* Initialize tensor */ + GGML_GTX_CMD_GET_ALLOC_SIZE = 13, /* Get allocation size for tensor */ + + /* Handshake */ + GGML_GTX_CMD_HELLO = 14, /* Handshake (must be 14 for ggml-rpc compat) */ + GGML_GTX_CMD_DEVICE_COUNT = 15, /* Query device count */ + + /* Extended */ + GGML_GTX_CMD_GRAPH_RECOMPUTE = 16, /* Recompute graph */ + + GGML_GTX_CMD_COUNT +} ggml_gtx_cmd_t; + +/*============================================================================= + * Server Configuration + *============================================================================*/ + +/** + * @brief Server configuration + */ +typedef struct { + void * msg_queue_addr; /* Message queue base address */ + size_t msg_queue_size; /* Message queue size */ + void * tensor_pool_addr; /* Tensor memory pool base */ + size_t tensor_pool_size; /* Tensor memory pool size */ + uint32_t msi_vector; /* MSI vector for completion notification */ +} ggml_gtx_server_config_t; + +/** + * @brief Server statistics + */ +typedef struct { + uint64_t commands_processed; + uint64_t bytes_received; + uint64_t bytes_sent; + uint64_t compute_cycles; + uint32_t errors; + uint32_t last_error_cmd; +} ggml_gtx_server_stats_t; + +/** + * @brief Server state + */ +typedef enum { + GGML_GTX_SERVER_STATE_UNINITIALIZED = 0, + GGML_GTX_SERVER_STATE_IDLE, + GGML_GTX_SERVER_STATE_PROCESSING, + GGML_GTX_SERVER_STATE_COMPUTING, + GGML_GTX_SERVER_STATE_ERROR, +} ggml_gtx_server_state_t; + +/*============================================================================= + * Graph Compute Limits + *============================================================================*/ + +/** Maximum number of tensors in a serialized graph (nodes + all dependencies) */ +#define GGML_GTX_GRAPH_MAX_TENSORS 4096 + +/* Descriptor-based graph execution ABI */ +#define GGML_GTX_GRAPH_V2_MAGIC 0x32585447u /* "GTX2" little-endian */ +#define GGML_GTX_GRAPH_V2_VERSION 1u + +/* Launch memory modes. + * + * DDR_DIRECT descriptors carry original DDR tensor addresses. Firmware-side + * wrappers may materialize tiles into L2 and call kernels with L2_STASHED_N1. + * L2_STASHED_N1 kernels must not perform DDR<->L2 transfers. + */ +#define GGML_GTX_MEMORY_MODE_DDR_DIRECT 0u +#define GGML_GTX_MEMORY_MODE_L2_STASHED_N1 1u + +#define GGML_GTX_DESC_FLAG_STASH_CHAIN 0x00000001u +/* The host has already selected one contiguous tile. Firmware maps raw + * memory views into L2 but must not derive an OP-specific tile/schedule. */ +#define GGML_GTX_DESC_FLAG_STASH_HOST_TILED 0x00000002u +/* Execution-memory policy chosen by the host lowerer. Exactly one of these + * flags is required for new descriptors: L2_REQUIRED admits only the generic + * raw-view mapper, while DDR_ALLOWED executes as a coherent DDR boundary. */ +#define GGML_GTX_DESC_FLAG_L2_REQUIRED 0x00000004u +#define GGML_GTX_DESC_FLAG_DDR_ALLOWED 0x00000008u + +/* A small non-DDR-backed L2 slot is kept available even for kernels that + * advertise zero scratch. This prevents ordinary scratch use from forcing a + * stash mapping to be reclaimed and written back. */ +#define GGML_GTX_L2_MIN_SCRATCH_BYTES (64u * 1024u) + +typedef enum { + GGML_GTX_KERNEL_INVALID = 0, + GGML_GTX_KERNEL_ADD_F16 = 1, + GGML_GTX_KERNEL_MUL_F16 = 2, + GGML_GTX_KERNEL_SCALE_F16 = 3, + GGML_GTX_KERNEL_RMS_NORM_F16 = 4, + GGML_GTX_KERNEL_SOFT_MAX_F16 = 5, + GGML_GTX_KERNEL_MUL_MAT_F16 = 6, + GGML_GTX_KERNEL_ROPE_F16 = 7, + GGML_GTX_KERNEL_GET_ROWS_F16 = 8, + GGML_GTX_KERNEL_SET_ROWS_F16 = 9, + GGML_GTX_KERNEL_IM2COL_F16 = 10, + GGML_GTX_KERNEL_SWIGLU_F16 = 11, + GGML_GTX_KERNEL_CONT_F16 = 12, + GGML_GTX_KERNEL_CPY_F16 = 13, + GGML_GTX_KERNEL_ADD_F16_F32_TO_F16 = 14, + GGML_GTX_KERNEL_GELU_F16 = 15, + GGML_GTX_KERNEL_NORM_F16 = 16, + GGML_GTX_KERNEL_REPEAT_F16 = 17, + GGML_GTX_KERNEL_CONCAT_F16 = 18, + GGML_GTX_KERNEL_CONV_2D_F16 = 19, + GGML_GTX_KERNEL_CONV_2D_DW_F16 = 20, + GGML_GTX_KERNEL_UPSCALE_F16 = 21, + GGML_GTX_KERNEL_SIN_F16 = 22, + GGML_GTX_KERNEL_ABS_F16 = 23, + GGML_GTX_KERNEL_NEG_F16 = 24, + GGML_GTX_KERNEL_SGN_F16 = 25, + GGML_GTX_KERNEL_CEIL_F16 = 26, + GGML_GTX_KERNEL_FLOOR_F16 = 27, + GGML_GTX_KERNEL_TANH_F16 = 28, + GGML_GTX_KERNEL_SILU_F16 = 29, + GGML_GTX_KERNEL_COS_F16 = 30, + GGML_GTX_KERNEL_PAD_F16 = 31, + GGML_GTX_KERNEL_RELU_F16 = 32, + GGML_GTX_KERNEL_CONV_TRANSPOSE_2D_F16 = 33, + GGML_GTX_KERNEL_LEAKY_RELU_F16 = 34, + GGML_GTX_KERNEL_CLAMP_F16 = 35, +} ggml_gtx_kernel_id_t; + +/* GGML owns CONV op_params[0..5]. Host-lowered layout metadata uses the + * remaining descriptor slots and is consumed only by the uploaded kernel. */ +#define GGML_GTX_CONV_LAYOUT_PARAM 15 +#define GGML_GTX_CONV_LAYOUT_C_FAST 1 +#define GGML_GTX_CONV_LOGICAL_OW_PARAM 6 +#define GGML_GTX_CONV_LOGICAL_OH_PARAM 7 +#define GGML_GTX_CONV_LOGICAL_C_PARAM 8 +#define GGML_GTX_CONV_LOGICAL_IW_PARAM 9 +#define GGML_GTX_CONV_LOGICAL_IH_PARAM 10 +#define GGML_GTX_CONV_LOGICAL_KW_PARAM 11 +#define GGML_GTX_CONV_LOGICAL_KH_PARAM 12 + +/*============================================================================= + * RPC Message Structures (packed, 8-byte aligned) + *============================================================================*/ + +#pragma pack(push, 1) + +/** + * @brief RPC tensor descriptor for tensor transfer and copy commands. + * + * This remains compatible with the current tensor upload/download RPC path. + * GRAPH_COMPUTE no longer consumes serialized ggml tensors; it uses the v2 + * descriptor payload below. + */ +typedef struct { + uint64_t id; /* Tensor ID */ + uint32_t type; /* ggml_type */ + uint64_t buffer; /* Buffer pointer */ + uint32_t ne[4]; /* Shape */ + uint32_t nb[4]; /* Strides */ + uint32_t op; /* Operation */ + int32_t op_params[16]; /* Operation parameters */ + int32_t flags; /* Flags */ + uint64_t src[10]; /* Source tensors */ + uint64_t view_src; /* View source */ + uint64_t view_offs; /* View offset */ + uint64_t data; /* Data pointer */ + char name[64]; /* Tensor name */ + char padding[4]; /* Alignment padding */ +} ggml_gtx_rpc_tensor_t; + +/** + * @brief Descriptor-based graph payload header. + * + * Host staging format: + * | ggml_gtx_graph_v2_header_t | + * | desc[n_desc] (each sizeof(ggml_gtx_kernel_launch_desc_t)) | + */ +typedef struct { + uint32_t magic; + uint16_t version; + uint16_t header_size; + uint32_t device; + uint32_t n_desc; + uint32_t desc_size; + uint32_t flags; + uint32_t reserved; +} ggml_gtx_graph_v2_header_t; + +/** + * @brief Transport-independent kernel launch descriptor. + * + * Built-in firmware kernels and future uploaded kernels should share this + * launch ABI shape: kernel_id selects executable code, args describe raw + * device addresses and dimensions. + */ +typedef struct { + uint32_t kernel_id; + uint32_t abi_version; + uint32_t dtype; + uint32_t reserved0; + uint64_t args_addr; + uint32_t args_size; + uint32_t grid[3]; + uint32_t block[3]; + uint32_t shared_mem; + uint32_t flags; + + uint64_t dst_addr; + uint64_t src_addr[4]; + uint32_t ne[4]; + uint32_t nb[4]; + uint32_t src_type[4]; + uint32_t src_ne[4][4]; + uint32_t src_nb[4][4]; + int32_t op_params[16]; + + /* L2-stashed execution view. + * + * DDR tensor identity remains in src_addr/dst_addr plus ne/nb above. + * When memory_mode is L2_STASHED_N1, l2_* fields describe compact + * contiguous per-nest tiles and the kernel executes only L2->L1/SPU work. + */ + uint32_t memory_mode; + uint32_t nest_id; + uint64_t l2_dst_addr; + uint64_t l2_src_addr[4]; + uint32_t l2_ne[4]; + uint32_t elem_size; + /* args_addr/args_size above identify an optional host-uploaded opaque + * payload. The control core passes it directly to the uploaded kernel; + * firmware and stash never stage or interpret it in L2. + * + * l2_scratch_size is the kernel-owned requirement advertised by the host. + * Firmware fills addr/capacity without interpreting the kernel or its + * arguments. Scratch is never DDR-backed or written back by stash. */ + uint32_t l2_scratch_size; + uint32_t l2_scratch_addr; + uint32_t l2_scratch_capacity; + uint32_t reserved[4]; +} ggml_gtx_kernel_launch_desc_t; + +/** + * @brief Hello response + */ +typedef struct { + uint8_t major; /* Version major */ + uint8_t minor; /* Version minor */ + uint8_t patch; /* Version patch */ +} ggml_gtx_msg_hello_rsp_t; + +/** + * @brief Device count response + */ +typedef struct { + uint32_t device_count; +} ggml_gtx_msg_device_count_rsp_t; + +/** + * @brief Alloc buffer request + */ +typedef struct { + uint32_t device; + uint64_t size; +} ggml_gtx_msg_alloc_buffer_req_t; + +/** + * @brief Alloc buffer response + */ +typedef struct { + uint64_t remote_ptr; + uint64_t remote_size; +} ggml_gtx_msg_alloc_buffer_rsp_t; + +/** + * @brief Get alignment request + */ +typedef struct { + uint32_t device; +} ggml_gtx_msg_get_alignment_req_t; + +/** + * @brief Get alignment response + */ +typedef struct { + uint64_t alignment; +} ggml_gtx_msg_get_alignment_rsp_t; + +/** + * @brief Get max size request + */ +typedef struct { + uint32_t device; +} ggml_gtx_msg_get_max_size_req_t; + +/** + * @brief Get max size response + */ +typedef struct { + uint64_t max_size; +} ggml_gtx_msg_get_max_size_rsp_t; + +/** + * @brief Free buffer request + */ +typedef struct { + uint64_t remote_ptr; +} ggml_gtx_msg_free_buffer_req_t; + +/** + * @brief Buffer clear request + */ +typedef struct { + uint64_t remote_ptr; + uint8_t value; +} ggml_gtx_msg_buffer_clear_req_t; + +/** + * @brief Get tensor request + */ +typedef struct { + ggml_gtx_rpc_tensor_t tensor; + uint64_t offset; + uint64_t size; +} ggml_gtx_msg_get_tensor_req_t; + +/** + * @brief Copy tensor request + */ +typedef struct { + ggml_gtx_rpc_tensor_t src; + ggml_gtx_rpc_tensor_t dst; +} ggml_gtx_msg_copy_tensor_req_t; + +/** + * @brief Copy tensor response + */ +typedef struct { + uint8_t result; +} ggml_gtx_msg_copy_tensor_rsp_t; + +/** + * @brief Get device memory request + */ +typedef struct { + uint32_t device; +} ggml_gtx_msg_get_device_memory_req_t; + +/** + * @brief Get device memory response + */ +typedef struct { + uint64_t free_mem; + uint64_t total_mem; +} ggml_gtx_msg_get_device_memory_rsp_t; + +#pragma pack(pop) + +/*============================================================================= + * Server API + *============================================================================*/ + +/** + * @brief Initialize the RPC server + * @param config Server configuration + * @return 0 on success, negative on error + */ +int ggml_gtx_server_init(const ggml_gtx_server_config_t *config); + +/** + * @brief Run the server main loop + * + * This function never returns under normal operation. + * It enters WFI when idle and wakes on ELBI interrupt. + */ +void ggml_gtx_server_run(void); + +/** + * @brief Process one command iteration + * + * Called from interrupt handler or polling loop. + * Processes available messages and sends responses. + * + * @return Number of commands processed + */ +uint32_t ggml_gtx_server_process(void); + +/** + * @brief Get server state + * @return Current server state + */ +ggml_gtx_server_state_t ggml_gtx_server_get_state(void); + +/** + * @brief Get server statistics + * @param stats Pointer to store statistics + */ +void ggml_gtx_server_get_stats(ggml_gtx_server_stats_t *stats); + +/** + * @brief Reset server statistics + */ +void ggml_gtx_server_reset_stats(void); + +/** + * @brief Shutdown the server + */ +void ggml_gtx_server_shutdown(void); + +/*============================================================================= + * Interrupt Handlers + *============================================================================*/ + +/** + * @brief ELBI interrupt handler + * + * Called from vector_table.c when Host sends doorbell interrupt. + * Sets flag to wake up server loop. + * + * @param source Interrupt source (0-31) + * @param user_data User data (unused) + */ +void ggml_gtx_server_elbi_handler(uint32_t source, void *user_data); + +/** + * @brief Check if server has pending work + * @return true if work pending + */ +bool ggml_gtx_server_has_pending_work(void); + +/** + * @brief Clear pending work flag + */ +void ggml_gtx_server_clear_pending(void); + +/*============================================================================= + * Version Info + *============================================================================*/ + +#define GGML_GTX_SERVER_VERSION_MAJOR 0 +#define GGML_GTX_SERVER_VERSION_MINOR 1 +#define GGML_GTX_SERVER_VERSION_PATCH 0 + +#ifdef __cplusplus +} +#endif + +#endif /* GGML_GTX_SERVER_H */ diff --git a/depend/ggml-gtx/include/ggml-gtx.h b/depend/ggml-gtx/include/ggml-gtx.h new file mode 100644 index 0000000..ecb4d7d --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx.h @@ -0,0 +1,208 @@ +/** + * @file ggml-gtx.h + * @brief GGML GTX NPU Backend Header + * + * Host-side GGML backend for GTX NPU accelerator. + * Communicates with the NPU firmware via PCIe. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#ifdef GGML_GTX_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef GGML_GTX_BUILD +# define GGML_GTX_API __declspec(dllexport) extern +# else +# define GGML_GTX_API __declspec(dllimport) extern +# endif +# else +# define GGML_GTX_API __attribute__ ((visibility ("default"))) extern +# endif +#else +# define GGML_GTX_API extern +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/*============================================================================= + * GTX Device Info + *============================================================================*/ + +/** + * @brief GTX device information + */ +typedef struct ggml_gtx_device_info { + int device_id; /* Device index */ + char name[64]; /* Device name */ + char description[128]; /* Device description */ + uint64_t total_memory; /* Total device memory in bytes */ + uint64_t free_memory; /* Free device memory in bytes */ + int num_nests; /* Number of NESTs */ + int num_spus; /* Total number of SPUs */ + int pcie_gen; /* PCIe generation (1-5) */ + int pcie_width; /* PCIe lane width (x1-x16) */ + uint8_t firmware_version[3]; /* Firmware version (major, minor, patch) */ +} ggml_gtx_device_info_t; + +/*============================================================================= + * Backend Registration + *============================================================================*/ + +/** + * @brief Get the number of available GTX devices + * @return Number of GTX devices + */ +GGML_GTX_API int ggml_gtx_get_device_count(void); + +/** + * @brief Get device information + * @param device Device index + * @param info Pointer to store device info + * @return true on success + */ +GGML_GTX_API bool ggml_gtx_get_device_info(int device, ggml_gtx_device_info_t * info); + +/** + * @brief Initialize a GTX backend for a specific device + * @param device Device index (0-based) + * @return Backend handle, or NULL on failure + */ +GGML_GTX_API ggml_backend_t ggml_backend_gtx_init(int device); + +/** + * @brief Check if a backend is a GTX backend + * @param backend Backend handle + * @return true if GTX backend + */ +GGML_GTX_API bool ggml_backend_is_gtx(ggml_backend_t backend); + +/** + * @brief Get the device index for a GTX backend + * @param backend Backend handle + * @return Device index, or -1 on error + */ +GGML_GTX_API int ggml_backend_gtx_get_device(ggml_backend_t backend); + +/*============================================================================= + * Buffer Type + *============================================================================*/ + +/** + * @brief Get the buffer type for GTX device memory + * @param device Device index + * @return Buffer type handle + */ +GGML_GTX_API ggml_backend_buffer_type_t ggml_backend_gtx_buffer_type(int device); + +/** + * @brief Get the host buffer type for GTX (pinned memory for faster transfers) + * @param device Device index + * @return Buffer type handle, or NULL if not supported + */ +GGML_GTX_API ggml_backend_buffer_type_t ggml_backend_gtx_host_buffer_type(int device); + +/*============================================================================= + * Device Registration (for backend registry) + *============================================================================*/ + +/** + * @brief Get GTX backend registration + * @return Backend registration handle + */ +GGML_GTX_API ggml_backend_reg_t ggml_backend_gtx_reg(void); + +/*============================================================================= + * PCIe Configuration + *============================================================================*/ + +/** + * @brief PCIe device path (e.g., "/dev/gtx0" or "\\.\gtx0") + */ +typedef struct ggml_gtx_pcie_config { + const char * device_path; /* PCIe device path */ + int bar_index; /* BAR index for memory mapping (default: 0) */ + size_t msg_queue_size; /* Message queue size (default: 64KB) */ + int timeout_ms; /* Operation timeout in ms (default: 5000) */ +} ggml_gtx_pcie_config_t; + +/** + * @brief Initialize GTX with custom PCIe configuration + * @param device Device index + * @param config PCIe configuration + * @return Backend handle, or NULL on failure + */ +GGML_GTX_API ggml_backend_t ggml_backend_gtx_init_with_config( + int device, + const ggml_gtx_pcie_config_t * config +); + +/*============================================================================= + * Synchronization and Status + *============================================================================*/ + +/** + * @brief Synchronize with GTX device (wait for all pending operations) + * @param backend Backend handle + */ +GGML_GTX_API void ggml_backend_gtx_synchronize(ggml_backend_t backend); + +/** + * @brief Check if device is ready + * @param backend Backend handle + * @return true if ready + */ +GGML_GTX_API bool ggml_backend_gtx_is_ready(ggml_backend_t backend); + +/** + * @brief Get last error message + * @param backend Backend handle + * @return Error message, or NULL if no error + */ +GGML_GTX_API const char * ggml_backend_gtx_get_error(ggml_backend_t backend); + +/*============================================================================= + * Debug and Statistics + *============================================================================*/ + +/** + * @brief GTX statistics + */ +typedef struct ggml_gtx_stats { + uint64_t tensors_set; /* Number of tensor uploads */ + uint64_t tensors_get; /* Number of tensor downloads */ + uint64_t bytes_sent; /* Total bytes sent to device */ + uint64_t bytes_received; /* Total bytes received from device */ + uint64_t graph_computes; /* Number of graph computations */ + uint64_t total_compute_us; /* Total compute time in microseconds */ + uint64_t total_transfer_us; /* Total transfer time in microseconds */ +} ggml_gtx_stats_t; + +/** + * @brief Get statistics for a GTX backend + * @param backend Backend handle + * @param stats Pointer to store statistics + */ +GGML_GTX_API void ggml_backend_gtx_get_stats(ggml_backend_t backend, ggml_gtx_stats_t * stats); + +/** + * @brief Reset statistics + * @param backend Backend handle + */ +GGML_GTX_API void ggml_backend_gtx_reset_stats(ggml_backend_t backend); + +/** + * @brief Enable/disable debug logging + * @param enable true to enable + */ +GGML_GTX_API void ggml_gtx_set_debug(bool enable); + +#ifdef __cplusplus +} +#endif diff --git a/depend/ggml-gtx/include/ggml-gtx/ggml-gtx-kernel-args.h b/depend/ggml-gtx/include/ggml-gtx/ggml-gtx-kernel-args.h new file mode 100644 index 0000000..0ca30c9 --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx/ggml-gtx-kernel-args.h @@ -0,0 +1,50 @@ +#ifndef GGML_GTX_KERNEL_ARGS_H +#define GGML_GTX_KERNEL_ARGS_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Kernel-private payloads are shared by the host packer and the uploaded + * kernel only. Firmware transports these bytes without interpreting them. */ +#define GGML_GTX_MUL_MAT_ARGS_VERSION 1u +#define GGML_GTX_ADD_ARGS_VERSION 1u +#define GGML_GTX_MUL_ARGS_VERSION 1u + +typedef struct ggml_gtx_mul_mat_args { + uint32_t version; + uint32_t size; + uint32_t m; + uint32_t k; + uint32_t n; + uint32_t reserved[3]; +} ggml_gtx_mul_mat_args_t; + +typedef struct ggml_gtx_add_args { + uint32_t version; + uint32_t size; + uint32_t scalar_count; + uint32_t reserved; + uint16_t scalars[1]; +} ggml_gtx_add_args_t; + +#define GGML_GTX_ADD_ARGS_HEADER_SIZE offsetof(ggml_gtx_add_args_t, scalars) + +typedef struct ggml_gtx_mul_args { + uint32_t version; + uint32_t size; + uint32_t scalar_count; + uint32_t reserved; + uint16_t scalars[1]; +} ggml_gtx_mul_args_t; + +#define GGML_GTX_MUL_ARGS_HEADER_SIZE offsetof(ggml_gtx_mul_args_t, scalars) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/depend/ggml-gtx/include/ggml-gtx/ggml-gtx-kernel-common.h b/depend/ggml-gtx/include/ggml-gtx/ggml-gtx-kernel-common.h new file mode 100644 index 0000000..c12cdd0 --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx/ggml-gtx-kernel-common.h @@ -0,0 +1,22 @@ +#ifndef GGML_GTX_KERNEL_COMMON_H +#define GGML_GTX_KERNEL_COMMON_H + +#include "ggml-gtx-compute.h" +#include "ggml-gtx-port.h" +#include "ggml-gtx/intrinsics/gtx_csr.h" +#include "ggml-gtx/intrinsics/intrin.h" + +#ifdef GGML_GTX_KERNEL_DEBUG +static inline void ggml_gtx_debug_dump_fp16_head(const char * tag, uint64_t addr) { + volatile uint16_t * p = (volatile uint16_t *)(uintptr_t)addr; + ggml_gtx_printf("[%s] addr=%p head=[%04x %04x %04x %04x]\n", + tag, + (void *)(uintptr_t)addr, + p[0], p[1], p[2], p[3]); +} +#else +#define ggml_gtx_debug_dump_fp16_head(tag, addr) \ + do { (void) (tag); (void) (addr); } while (0) +#endif + +#endif diff --git a/depend/ggml-gtx/include/ggml-gtx/ggml-gtx-protocol.h b/depend/ggml-gtx/include/ggml-gtx/ggml-gtx-protocol.h new file mode 100644 index 0000000..a53b260 --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx/ggml-gtx-protocol.h @@ -0,0 +1,163 @@ +#ifndef GGML_GTX_PROTOCOL_H +#define GGML_GTX_PROTOCOL_H + +#include +#include + +#include "ggml-gtx-server.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define GGML_GTX_RPC_MAGIC 0x31585447u /* "GTX1" little-endian */ +#define GGML_GTX_RPC_VERSION 1u + +#define GGML_GTX_KERNEL_ARG_MAGIC 0x31415847u /* "GXA1" little-endian */ +#define GGML_GTX_KERNEL_ARG_VERSION 1u + +#define GTX_DDR_BASE 0x370000000ull +#define GTX_FIXED_ARG_ADDR (GTX_DDR_BASE + 0x01000000ull) +#define GTX_FIXED_ARG_REGION_SIZE 0x00100000ull +#define GTX_FIXED_SRC0_ADDR (GTX_DDR_BASE + 0x01100000ull) +#define GTX_FIXED_SRC1_ADDR (GTX_DDR_BASE + 0x01200000ull) +#define GTX_FIXED_DST_ADDR (GTX_DDR_BASE + 0x01300000ull) +#define GTX_FIXED_TENSOR_ARENA_ADDR GTX_FIXED_SRC0_ADDR +#define GTX_FIXED_TENSOR_ARENA_SIZE 0x7ea00000ull +#define GTX_FIXED_KERNEL_STAGING_ADDR (GTX_DDR_BASE + 0x7ff00000ull) +#define GTX_FIXED_KERNEL_STAGING_SIZE 0x00100000ull +#define GTX_FIXED_KERNEL_WORK_ADDR 0x03100000ull +#define GTX_FIXED_KERNEL_WORK_SIZE 0x00100000ull +#define GTX_STASH_STATS_ADDR (GTX_DDR_BASE + 0x7fe00000ull) +#define GTX_STASH_STATS_MAGIC 0x48534154u + +/* Legacy DDR ROPE sin/cos helper workspace. L2 ROPE uses its dynamically + * assigned scratch slot only for cosine and sine tables. */ +#define GGML_GTX_ROPE_SCRATCH_WORK_B_OFFSET 0x00050000u + +static inline uint32_t ggml_gtx_rope_scratch_size(uint32_t ne2, uint32_t n_dims) { + if (ne2 == 0u || n_dims < 2u || (n_dims & 1u) != 0u) return 0u; + const uint64_t pairs_padded = (((uint64_t)n_dims / 2u) + 15u) & ~15ull; + const uint64_t table_bytes = (uint64_t)ne2 * pairs_padded * sizeof(uint16_t); + const uint64_t bytes = 2u * table_bytes; + return bytes <= UINT32_MAX - 31u ? (uint32_t)((bytes + 31u) & ~31ull) : 0u; +} + +typedef enum ggml_gtx_rpc_cmd { + GGML_GTX_RPC_PING = 1, + GGML_GTX_RPC_UPLOAD_BYTES = 2, + GGML_GTX_RPC_DOWNLOAD_BYTES = 3, + GGML_GTX_RPC_LOAD_KERNEL = 4, + GGML_GTX_RPC_LAUNCH = 5, + GGML_GTX_RPC_COPY_TENSOR = 6, + GGML_GTX_RPC_STASH_SYNC = 7, + GGML_GTX_RPC_DISCONNECT = 8, +} ggml_gtx_rpc_cmd_t; + +typedef enum ggml_gtx_rpc_status { + GGML_GTX_RPC_STATUS_OK = 0, + GGML_GTX_RPC_STATUS_ERROR = 1, + GGML_GTX_RPC_STATUS_BAD_MAGIC = 2, + GGML_GTX_RPC_STATUS_BAD_VERSION = 3, + GGML_GTX_RPC_STATUS_BAD_SIZE = 4, + GGML_GTX_RPC_STATUS_BAD_CMD = 5, + GGML_GTX_RPC_STATUS_NOT_FOUND = 6, +} ggml_gtx_rpc_status_t; + +typedef enum ggml_gtx_kernel_format { + GGML_GTX_KERNEL_FORMAT_LLEXT = 1, +} ggml_gtx_kernel_format_t; + +typedef struct ggml_gtx_rpc_frame { + uint32_t magic; + uint16_t version; + uint16_t header_size; + uint16_t cmd; + uint16_t status; + uint32_t seq; + uint32_t payload_size; +} ggml_gtx_rpc_frame_t; + +typedef struct ggml_gtx_rpc_upload_bytes { + uint64_t addr; + uint32_t size; + uint32_t reserved; +} ggml_gtx_rpc_upload_bytes_t; + +typedef struct ggml_gtx_rpc_download_bytes { + uint64_t addr; + uint32_t size; + uint32_t reserved; +} ggml_gtx_rpc_download_bytes_t; + +typedef struct ggml_gtx_rpc_load_kernel { + uint32_t kernel_id; + uint32_t format; + uint64_t staging_addr; + uint32_t binary_size; + uint32_t reserved0; + uint64_t work_addr; + uint32_t work_size; + char entry_symbol[32]; +} ggml_gtx_rpc_load_kernel_t; + +typedef struct ggml_gtx_rpc_launch { + uint32_t kernel_id; + uint32_t reserved; + uint64_t arg_addr; +} ggml_gtx_rpc_launch_t; + +typedef struct ggml_gtx_rpc_copy_tensor { + ggml_gtx_rpc_tensor_t src; + ggml_gtx_rpc_tensor_t dst; +} ggml_gtx_rpc_copy_tensor_t; + +typedef struct ggml_gtx_rpc_copy_tensor_rsp { + uint8_t result; +} ggml_gtx_rpc_copy_tensor_rsp_t; + +typedef enum ggml_gtx_rpc_stash_sync_op { + GGML_GTX_RPC_STASH_FLUSH_ALL = 1, + GGML_GTX_RPC_STASH_INVALIDATE_RANGE = 2, + // Write back only dirty stash slots that overlap addr..addr + size. + GGML_GTX_RPC_STASH_FLUSH_RANGE = 3, + // Make DDR authoritative, then discard all stash residency metadata. + GGML_GTX_RPC_STASH_FLUSH_RESET = 4, +} ggml_gtx_rpc_stash_sync_op_t; + +typedef struct ggml_gtx_rpc_stash_sync { + uint32_t op; + uint32_t reserved; + uint64_t addr; + uint64_t size; +} ggml_gtx_rpc_stash_sync_t; + +typedef struct ggml_gtx_kernel_arg { + uint32_t magic; + uint16_t version; + uint16_t size; + uint32_t kernel_id; + uint32_t flags; + ggml_gtx_kernel_launch_desc_t desc; +} ggml_gtx_kernel_arg_t; + +static inline const void * ggml_gtx_kernel_args(const ggml_gtx_kernel_arg_t * arg, + uint32_t minimum_size) { + if (arg == NULL || arg->desc.args_addr == 0u || + arg->desc.args_size < minimum_size) { + return NULL; + } + return (const void *)(uintptr_t)arg->desc.args_addr; +} + +#define GGML_GTX_STATIC_ASSERT(cond, name) typedef char ggml_gtx_static_assert_##name[(cond) ? 1 : -1] +GGML_GTX_STATIC_ASSERT(sizeof(ggml_gtx_kernel_launch_desc_t) == 436, launch_desc_size); +GGML_GTX_STATIC_ASSERT(offsetof(ggml_gtx_kernel_launch_desc_t, memory_mode) == 340, launch_desc_memory_mode_offset); +GGML_GTX_STATIC_ASSERT(sizeof(ggml_gtx_kernel_arg_t) == 452, kernel_arg_size); +#undef GGML_GTX_STATIC_ASSERT + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/depend/ggml-gtx/include/ggml-gtx/intrinsics/gtx_compat.h b/depend/ggml-gtx/include/ggml-gtx/intrinsics/gtx_compat.h new file mode 100644 index 0000000..877e6b8 --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx/intrinsics/gtx_compat.h @@ -0,0 +1,42 @@ +// ============================================================================== +// GTX compatibility header — workaround for assembler limitations +// +// The standard assembler doesn't recognize exp.v and ln.v with 2-operand form. +// This header redefines them using .insn r encoding. +// +// Include this AFTER intrin.h to override the macros. +// ============================================================================== + +#ifndef GTX_COMPAT_H +#define GTX_COMPAT_H + +// exp.v: funct7=0x1C, funct3=1, opcode=0x0B +// R-type: .insn r opcode, func3, func7, rd, rs1, rs2 +#ifdef __exp_v +#undef __exp_v +#endif +#define __exp_v(vector_size, mode) \ + do { \ + register unsigned long _vs __asm__("a0") = (unsigned long)(vector_size); \ + register unsigned long _md __asm__("a1") = (unsigned long)(mode); \ + __asm__ volatile( \ + ".insn r 0x0b, 1, 0x1c, x0, %[r1], %[r2]\n" \ + : : [r1] "r"(_vs), [r2] "r"(_md) \ + ); \ + } while(0) + +// ln.v: funct7=0x1C, funct3=2, opcode=0x0B +#ifdef __ln_v +#undef __ln_v +#endif +#define __ln_v(vector_size, mode) \ + do { \ + register unsigned long _vs __asm__("a0") = (unsigned long)(vector_size); \ + register unsigned long _md __asm__("a1") = (unsigned long)(mode); \ + __asm__ volatile( \ + ".insn r 0x0b, 2, 0x1c, x0, %[r1], %[r2]\n" \ + : : [r1] "r"(_vs), [r2] "r"(_md) \ + ); \ + } while(0) + +#endif // GTX_COMPAT_H diff --git a/depend/ggml-gtx/include/ggml-gtx/intrinsics/gtx_csr.h b/depend/ggml-gtx/include/ggml-gtx/intrinsics/gtx_csr.h new file mode 100644 index 0000000..91375db --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx/intrinsics/gtx_csr.h @@ -0,0 +1,249 @@ +//================================================================== +// Copyright : (C) Supergate - All Rights Reserved +// Project : GSF / VTS +// Description : GTX architecture CSR definition +// Author : mh.kim ( NPU Div - NPU Core Team ) +// Last Update : 2025/12/15 +//================================================================== + +#ifndef GTX_CSR_H +#define GTX_CSR_H + +#ifdef __cplusplus +extern "C" { +#endif + + +//================================= +// Global (GSPR, GTX) / 000-3FF (Nest관련) +//================================= +// Run +#define RUN 0x000 + +// Command +#define OPERAND_1 0x001 +#define OPERAND_2 0x002 +#define OPERAND_3 0x003 +#define OPCODE 0x004 +// [추가] Nest Selection (공유 윈도우 제어의 핵심) +#define NEST_SELECT 0x005 // 현재 CPU 주소창에 연결할 Nest ID (0-3) 선택 +#define NEST_STATUS 0x382 // 각 Nest의 전원/연결 상태 확인 (is_nx_accessible용) +// Stack +#define STACK_INFO 0x010 +#define STACK_SAVE 0x011 + +// Performance - reserved + +// H/W Feature & Debug +#define COMMAND 0x300 +#define INTERRUPT_CONTROL 0x310 +#define INTERRUPT_VECTOR 0x311 +#define INTERRUPT_SOURCE 0x312 +#define SPU_BUSY 0x380 +#define SMU_BUSY 0x381 +#define INFO 0x3F0 +#define ID 0x3F1 + + +//================================= +// Nest (NSPR, NEST) / 400-7FF (SMU, THREAD관련) +//================================= +// Mask +#define THREAD 0x400 +#define SHARED 0x401 + +// Config +#define DATA_FORMAT 0x402 +#define OP_MODE 0x403 + +// Performance - reserved + +// H/W Feature & Debug +#define CLEAR 0x700 +#define NEST_CREDIT 0x701 +#define SDLE_STATUS 0x780 +#define CREDIT_COUNT 0x781 +#define CREDIT_ERROR 0x782 + + +//================================= +// Local (LSPR, SPU) / 800-BFF (SPU관련) +//================================= +// SCR +#define SGPR_0 0x800 +#define SGPR_1 0x801 +#define SGPR_2 0x802 +#define SGPR_3 0x803 +#define SGPR_4 0x804 +#define SGPR_5 0x805 +#define SGPR_6 0x806 +#define SGPR_7 0x807 +#define SGPR_8 0x808 +#define SGPR_9 0x809 +#define SGPR_10 0x80a +#define SGPR_11 0x80b +#define SGPR_12 0x80c +#define SGPR_13 0x80d +#define SGPR_14 0x80e +#define SGPR_15 0x80f + +#define SGPR_16 0x810 +#define SGPR_17 0x811 +#define SGPR_18 0x812 +#define SGPR_19 0x813 +#define SGPR_20 0x814 +#define SGPR_21 0x815 +#define SGPR_22 0x816 +#define SGPR_23 0x817 +#define SGPR_24 0x818 +#define SGPR_25 0x819 +#define SGPR_26 0x81a +#define SGPR_27 0x81b +#define SGPR_28 0x81c +#define SGPR_29 0x81d +#define SGPR_30 0x81e +#define SGPR_31 0x81f + +#define SGPR_32 0x820 +#define SGPR_33 0x821 +#define SGPR_34 0x822 +#define SGPR_35 0x823 +#define SGPR_36 0x824 +#define SGPR_37 0x825 +#define SGPR_38 0x826 +#define SGPR_39 0x827 +#define SGPR_40 0x828 +#define SGPR_41 0x829 +#define SGPR_42 0x82a +#define SGPR_43 0x82b +#define SGPR_44 0x82c +#define SGPR_45 0x82d +#define SGPR_46 0x82e +#define SGPR_47 0x82f + +#define SGPR_48 0x830 +#define SGPR_49 0x831 +#define SGPR_50 0x832 +#define SGPR_51 0x833 +#define SGPR_52 0x834 +#define SGPR_53 0x835 +#define SGPR_54 0x836 +#define SGPR_55 0x837 +#define SGPR_56 0x838 +#define SGPR_57 0x839 +#define SGPR_58 0x83a +#define SGPR_59 0x83b +#define SGPR_60 0x83c +#define SGPR_61 0x83d +#define SGPR_62 0x83e +#define SGPR_63 0x83f + +#define SGPR_64 0x840 +#define SGPR_65 0x841 +#define SGPR_66 0x842 +#define SGPR_67 0x843 +#define SGPR_68 0x844 +#define SGPR_69 0x845 +#define SGPR_70 0x846 +#define SGPR_71 0x847 +#define SGPR_72 0x848 +#define SGPR_73 0x849 +#define SGPR_74 0x84a +#define SGPR_75 0x84b +#define SGPR_76 0x84c +#define SGPR_77 0x84d +#define SGPR_78 0x84e +#define SGPR_79 0x84f + +#define SGPR_80 0x850 +#define SGPR_81 0x851 +#define SGPR_82 0x852 +#define SGPR_83 0x853 +#define SGPR_84 0x854 +#define SGPR_85 0x855 +#define SGPR_86 0x856 +#define SGPR_87 0x857 +#define SGPR_88 0x858 +#define SGPR_89 0x859 +#define SGPR_90 0x85a +#define SGPR_91 0x85b +#define SGPR_92 0x85c +#define SGPR_93 0x85d +#define SGPR_94 0x85e +#define SGPR_95 0x85f + +#define SGPR_96 0x860 +#define SGPR_97 0x861 +#define SGPR_98 0x862 +#define SGPR_99 0x863 +#define SGPR_100 0x864 +#define SGPR_101 0x865 +#define SGPR_102 0x866 +#define SGPR_103 0x867 +#define SGPR_104 0x868 +#define SGPR_105 0x869 +#define SGPR_106 0x86a +#define SGPR_107 0x86b +#define SGPR_108 0x86c +#define SGPR_109 0x86d +#define SGPR_110 0x86e +#define SGPR_111 0x86f + +#define SGPR_112 0x870 +#define SGPR_113 0x871 +#define SGPR_114 0x872 +#define SGPR_115 0x873 +#define SGPR_116 0x874 +#define SGPR_117 0x875 +#define SGPR_118 0x876 +#define SGPR_119 0x877 +#define SGPR_120 0x878 +#define SGPR_121 0x879 +#define SGPR_122 0x87a +#define SGPR_123 0x87b +#define SGPR_124 0x87c +#define SGPR_125 0x87d +#define SGPR_126 0x87e +#define SGPR_127 0x87f + +#define SPM_ADDR_A 0x900 +#define SPM_ADDR_B 0x901 +#define SPM_ADDR_C 0x902 +#define SPM_ADDR_R 0x903 + +// Performance - reserved + +// H/W Feature & Debug +#define CP_CONTROL 0xB00 +#define DMA_CONTROL 0xB20 +// [추가] DMA 상세 제어 (DDR -> L2 -> L1 복사 시 필수) +#define DMA_SRC_ADDR_LO 0xB21 // DMA 소스 주소 (DDR) 하위 32비트 +#define DMA_SRC_ADDR_HI 0xB22 // DMA 소스 주소 (DDR) 상위 32비트 +#define DMA_DST_ADDR 0xB23 // DMA 목적지 주소 (L2/L1 Window 주소) +#define DMA_TRANS_SIZE 0xB24 // 전송 크기 (Byte 단위) +// [추가] 동기화 (복사가 끝났는지 확인) +#define DMA_DONE_INTR 0xBA1 // DMA 완료 인터럽트 상태 및 클리어 + +#define MXE_CONTROL 0xB30 +#define SDE_CONTROL 0xB40 +#define PDE_CONTROL 0xB50 +#define SVR_TIMING 0xB60 +#define SPU_STATUS 0xB80 +#define DMA_STATUS 0xBA0 +#define MXE_STATUS 0xBB0 +#define SDE_STATUS 0xBC0 +#define PDE_STATUS 0xBD0 +#define CREDIT 0xBE0 +#define THREAD_ID 0xBFF + + +//================================= +// System / C00-FFF +//================================= +// reserved + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/depend/ggml-gtx/include/ggml-gtx/intrinsics/gtx_utils.h b/depend/ggml-gtx/include/ggml-gtx/intrinsics/gtx_utils.h new file mode 100644 index 0000000..a5ee849 --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx/intrinsics/gtx_utils.h @@ -0,0 +1,30 @@ +#ifndef GTX_UTILS_H +#define GTX_UTILS_H + +#include +#include +#include +#include +// #include "uart.hpp" +#include "ggml-gtx/sc_print.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// CSR 읽기 (RISC-V 표준 명령어 기반) +static inline uint32_t read_csr(uint32_t addr) { + uint32_t val; + __asm__ volatile("csrr %0, %1" : "=r"(val) : "i"(addr)); + return val; +} + +void __capture_l1_nest(uint16_t nest, uintptr_t dst); + +void __capture_l1(uintptr_t dst); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/depend/ggml-gtx/include/ggml-gtx/intrinsics/intrin.h b/depend/ggml-gtx/include/ggml-gtx/intrinsics/intrin.h new file mode 100644 index 0000000..7cc3162 --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx/intrinsics/intrin.h @@ -0,0 +1,44 @@ +//================================================================== +// Copyright : (C) Supergate - All Rights Reserved +// Project : GSF / VTS +// Description : GTX intrinsic include header file +// level 1: base instruction intrinsic +// level 2: convenience intrinsic for programming +// level 3: complex operation intrinsic consisting of compound instructions +// Author : mh.kim ( NPU Div - NPU Core Team ) +// Last Update : 2026/01/23 +//================================================================== + +#ifndef INTRIN_H +#define INTRIN_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "ggml-gtx/intrinsics/intrin_level1.h" +#include "ggml-gtx/intrinsics/intrin_level2.h" +#include "ggml-gtx/intrinsics/intrin_level3.h" +#include "ggml-gtx/intrinsics/gtx_compat.h" + + +//============================================= +// .insn register type +//============================================= +// [RISCV R type] +// +-------+-----+-----+-------+----+---------+ +// | func7 | rs2 | rs1 | func3 | rd | opcode7 | +// +-------+-----+-----+-------+----+---------+ +// 31 25 20 15 12 7 0 +// +// .insn r opcode7, func3, func7, rd, rs1, rs2 +// #define INSN_R_TYPE(opcode7 ,func7, func3, rd, rs1, rs2) +// asm volatile (".insn r 0xB, 0x58, 0, %3, %4, %5\n" +// : "=r" (rd) +// : "i" (opcode7), "i"(func3), "i"(func7), "r"(rs1), "r"(rs2)) + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/depend/ggml-gtx/include/ggml-gtx/intrinsics/intrin_level1.h b/depend/ggml-gtx/include/ggml-gtx/intrinsics/intrin_level1.h new file mode 100644 index 0000000..6f0c28e --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx/intrinsics/intrin_level1.h @@ -0,0 +1,969 @@ +//================================================================== +// Copyright : (C) Supergate - All Rights Reserved +// Project : GSF / VTS +// Description : GTX intrinsic definition (level 1) +// base instruction intrinsic +// Author : mh.kim ( NPU Div - NPU Core Team ) +// Last Update : 2026/01/23 +//================================================================== + +#ifndef INTRIN_LEVEL1_H +#define INTRIN_LEVEL1_H + +#include + +#define GTX_VEC_DTYPE_F16_F32_FLAG (1u << 30) +#define GTX_VEC_DTYPE_F32_F32_TO_F16_FLAG (1u << 29) + +#ifndef __gtx_giss_unmask_threads +#ifdef GGML_GTX_GISS_MASK_COMPAT +#define __gtx_giss_unmask_threads() \ + do { \ + uint64_t __gtx_giss_zero = 0; \ + uint64_t __gtx_giss_thread = 0x400; \ + __asm__ volatile ("opset %[r1], %[r2]\n" : : [r1] "r"(__gtx_giss_zero), [r2] "r"(__gtx_giss_zero)); \ + __asm__ volatile ("wrspr %[r1], %[r2]\n" : : [r1] "r"(__gtx_giss_thread), [r2] "r"(__gtx_giss_zero)); \ + } while (0) +#else +#define __gtx_giss_unmask_threads() do { } while (0) +#endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +//================================= +// Matrix multiplication +//================================= +// [mm.s] rs1 +// rs1 = col_B_size[63:48], col_A_size[31:16], row_A_size[15:0] +__attribute__((noinline)) void __mm_s(uint16_t row_A_size, uint16_t col_A_size, uint16_t col_B_size); + + +// [mm.o] rs1 rs3 +// rs1 = col_A_size[31:16] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __mm_o(uint16_t col_A_size, uint8_t result_svr_addr); + + +// [mm.v] rs1 rs3 +// rs1 = col_A_size[31:16] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __mm_v(uint16_t col_A_size, uint8_t result_svr_addr); + + +// [mm] rs1 +// rs1 = col_B_size[63:48], col_A_size[31:16], row_A_size[15:0] +__attribute__((noinline)) void __mm(uint16_t row_A_size, uint16_t col_A_size, uint16_t col_B_size); + + +// [mm.t] rs1 +// rs1 = col_B_size[63:48], col_A_size[31:16], row_A_size[15:0] +__attribute__((noinline)) void __mm_t(uint16_t row_A_size, uint16_t col_A_size, uint16_t col_B_size); + + +// [mmc.s] rs1 +// rs1 = col_B_size[63:48], col_A_size[31:16], row_A_size[15:0] +__attribute__((noinline)) void __mmc_s(uint16_t row_A_size, uint16_t col_A_size, uint16_t col_B_size); + + +// [mmc.o] rs1 rs3 +// rs1 = col_A_size[31:16] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __mmc_o(uint16_t col_A_size, uint8_t result_svr_addr); + + +// [mmc.v] rs1 rs3 +// rs1 = col_A_size[31:16] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __mmc_v(uint16_t col_A_size, uint8_t result_svr_addr); + + +// [mmc] rs1 +// rs1 = col_B_size[63:48], col_A_size[31:16], row_A_size[15:0] +__attribute__((noinline)) void __mmc(uint16_t row_A_size, uint16_t col_A_size, uint16_t col_B_size); + + +// [mmc.t] rs1 +// rs1 = col_B_size[63:48], col_A_size[31:16], row_A_size[15:0] +__attribute__((noinline)) void __mmc_t(uint16_t row_A_size, uint16_t col_A_size, uint16_t col_B_size); + + + +//================================= +// Convolution +//================================= +// [im2col.n] rs1 rs2 +// rs1 = col_A_size[31:16], row_A_size[15:0] +// rs2 = num_of_channel[47:32], stride[31:16], dilation[9:8], kernel_size[4:0] +__attribute__((noinline)) void __im2col_n(uint16_t row_A_size , uint16_t col_A_size, + uint8_t kernel_size, uint8_t dilation , uint16_t stride, uint16_t num_of_channel); + + +// [im2col.d] rs1 rs2 +// rs1 = col_A_size[31:16], row_A_size[15:0] +// rs2 = num_of_channel[47:32], stride[31:16], dilation[9:8], kernel_size[4:0] +__attribute__((noinline)) void __im2col_d(uint16_t row_A_size , uint16_t col_A_size, + uint8_t kernel_size, uint8_t dilation , uint16_t stride, uint16_t num_of_channel); + + + +//================================= +// Scalar calculation +//================================= +// [add.vs] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = scalar_value[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __add_vs(uint32_t vector_size, uint16_t scalar_value, uint16_t r2_sel); + + +// [sub.vs] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = scalar_value[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __sub_vs(uint32_t vector_size, uint16_t scalar_value, uint16_t r2_sel); + + +// [mul.vs] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = scalar_value[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __mul_vs(uint32_t vector_size, uint16_t scalar_value, uint16_t r2_sel); + + +// [div.vs] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = scalar_value[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __div_vs(uint32_t vector_size, uint16_t scalar_value, uint16_t r2_sel); + + +// [fmadd.vss] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = scalar_value1[31:16], scalar_value0[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __fmadd_vss(uint32_t vector_size, uint16_t scalar_value0, uint16_t scalar_value1, uint16_t r2_sel); + + +// [max.vs] rs1 rs2 rs3 rs4 +// rs1 = vector_size[23:0] +// rs2 = previous_max_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __max_vs(uint32_t vector_size, uint16_t previous_max_value, uint8_t result_svr_addr, uint16_t r2_sel); + + +// [min.vs] rs1 rs2 rs3 rs4 +// rs1 = vector_size[23:0] +// rs2 = previous_min_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __min_vs(uint32_t vector_size, uint16_t previous_min_value, uint8_t result_svr_addr, uint16_t r2_sel); + + +// [add.is] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr[4:0] +// rs2 = scalar_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __add_is(uint8_t src_svr_addr, uint16_t scalar_value, uint8_t result_svr_addr, uint16_t r2_sel); + + +// [sub.is] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr[4:0] +// rs2 = scalar_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __sub_is(uint8_t src_svr_addr, uint16_t scalar_value, uint8_t result_svr_addr, uint16_t r2_sel); + + +// [mul.is] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr[4:0] +// rs2 = scalar_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __mul_is(uint8_t src_svr_addr, uint16_t scalar_value, uint8_t result_svr_addr, uint16_t r2_sel); + + +// [mul.is.f32] rs1 rs2 rs3 +// rs1 = src_svr_addr[4:0] +// rs2 = FP32 scalar raw bits[31:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __mul_is_f32(uint8_t src_svr_addr, float scalar_value, uint8_t result_svr_addr); + + +// [div.is] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr[4:0] +// rs2 = scalar_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __div_is(uint8_t src_svr_addr, uint16_t scalar_value, uint8_t result_svr_addr, uint16_t r2_sel); + + +// [fmadd.iss] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr[4:0] +// rs2 = scalar_value1[31:16], scalar_value0[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __fmadd_iss(uint8_t src_svr_addr, uint16_t scalar_value0, uint16_t scalar_value1, uint8_t result_svr_addr, uint16_t r2_sel); + + +// [max.is] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr[4:0] +// rs2 = previous_max_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __max_is(uint8_t src_svr_addr, uint16_t previous_max_value, uint8_t result_svr_addr, uint16_t r2_sel); + + +// [min.is] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr[4:0] +// rs2 = previous_min_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __min_is(uint8_t src_svr_addr, uint16_t previous_min_value, uint8_t result_svr_addr, uint16_t r2_sel); + + + +//================================= +// Vector calculation +//================================= +// [add.vv] rs1 +// rs1 = vector_size[23:0] +#define __add_vv(vector_size)\ + __asm__ volatile("add.vv %[r1]\n" : : [r1] "r"(vector_size)); + + +// [sub.vv] rs1 +// rs1 = vector_size[23:0] +#define __sub_vv(vector_size)\ + __asm__ volatile("sub.vv %[r1]\n" : : [r1] "r"(vector_size)); + + +// [mul.vv] rs1 +// rs1 = vector_size[23:0] +#define __mul_vv(vector_size)\ + __asm__ volatile("mul.vv %[r1]\n" : : [r1] "r"(vector_size)); + + +// [mul.vv F16xF32] rs1 +// ADDRA is FP16, ADDRB is FP32, ADDRR is FP16. +// rs1 = F16_F32_FLAG[30] | vector_size[15:0] +#define __mul_vv_f16_f32(vector_size)\ + __asm__ volatile("mul.vv %[r1]\n" : : [r1] "r"(((uint32_t)(vector_size) & 0xFFFFu) | GTX_VEC_DTYPE_F16_F32_FLAG)); + + +// [mul.vv F32xF32] rs1 +// ADDRA is FP32, ADDRB is FP32, ADDRR is FP16. +// rs1 = F32_F32_TO_F16_FLAG[29] | vector_size[15:0] +#define __mul_vv_f32_f32(vector_size)\ + __asm__ volatile("mul.vv %[r1]\n" : : [r1] "r"(((uint32_t)(vector_size) & 0xFFFFu) | GTX_VEC_DTYPE_F32_F32_TO_F16_FLAG)); + + +// [div.vv] rs1 +// rs1 = vector_size[23:0] +#define __div_vv(vector_size)\ + __asm__ volatile("div.vv %[r1]\n" : : [r1] "r"(vector_size)); + + +// [fmadd.vvv] rs1 +// rs1 = vector_size[23:0] +#define __fmadd_vvv(vector_size)\ + __asm__ volatile("fmadd.vvv %[r1]\n" : : [r1] "r"(vector_size)); + + +// [sqrt.v] rs1 +// rs1 = vector_size[23:0] +#define __sqrt_v(vector_size)\ + __asm__ volatile("sqrt.v %[r1]\n" : : [r1] "r"(vector_size)); + + +// [exp.v] rs1 rs2 +// rs1 = vector_size[23:0] +// rs2 = mode[1:0] +#define __exp_v(vector_size, mode)\ + __asm__ volatile("exp.v %[r1]\n" : : [r1] "r"(vector_size)); + + +// [ln.v] rs1 rs2 +// rs1 = vector_size[23:0] +// rs2 = mode[1:0] +#define __ln_v(vector_size, mode)\ + __asm__ volatile("ln.v %[r1]\n" : : [r1] "r"(vector_size)); + + +// [abs.v] rs1 +// rs1 = vector_size[23:0] +#define __abs_v(vector_size)\ + __asm__ volatile("abs.v %[r1]\n" : : [r1] "r"(vector_size)); + + +// [neg.v] rs1 +// rs1 = vector_size[23:0] +#define __neg_v(vector_size)\ + __asm__ volatile("neg.v %[r1]\n" : : [r1] "r"(vector_size)); + + +// [sign.v] rs1 +// rs1 = vector_size[23:0] +#define __sign_v(vector_size)\ + __asm__ volatile("sign.v %[r1]\n" : : [r1] "r"(vector_size)); + + +// [step.v] rs1 +// rs1 = vector_size[23:0] +#define __step_v(vector_size)\ + __asm__ volatile("step.v %[r1]\n" : : [r1] "r"(vector_size)); + + +// [ceil.v] rs1 +// rs1 = vector_size[23:0] +#define __ceil_v(vector_size)\ + __asm__ volatile("ceil.v %[r1]\n" : : [r1] "r"(vector_size)); + + +// [trunc.v] rs1 +// rs1 = vector_size[23:0] +#define __trunc_v(vector_size)\ + __asm__ volatile("trunc.v %[r1]\n" : : [r1] "r"(vector_size)); + + +// [floor.v] rs1 +// rs1 = vector_size[23:0] +#define __floor_v(vector_size)\ + __asm__ volatile("floor.v %[r1]\n" : : [r1] "r"(vector_size)); + + +// [rne.v] rs1 +// rs1 = vector_size[23:0] +#define __rne_v(vector_size)\ + __asm__ volatile("rne.v %[r1]\n" : : [r1] "r"(vector_size)); + + +// [clamp.min] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = scalar_value[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __clamp_min(uint32_t vector_size, uint16_t scalar_value, uint16_t r2_sel); + + +// [clamp.max] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = scalar_value[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __clamp_max(uint32_t vector_size, uint16_t scalar_value, uint16_t r2_sel); + + +// [accum] rs1 +// rs1 = vector_size[23:0] +#define __accum(vector_size)\ + __asm__ volatile("accum %[r1]\n" : : [r1] "r"(vector_size)); + + +// [arange] rs1 rs2 +// rs1 = vector_size[23:0] +// rs2 = step[31:16], start_value[15:0] +__attribute__((noinline)) void __arange(uint32_t vector_size, uint16_t start_value, uint16_t step); + + +// [add.ii] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = src_svr_addr_B[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __add_ii(uint8_t src_svr_addr_A, uint8_t src_svr_addr_B, uint8_t result_svr_addr); + + +// [sub.ii] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = src_svr_addr_B[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __sub_ii(uint8_t src_svr_addr_A, uint8_t src_svr_addr_B, uint8_t result_svr_addr); + + +// [mul.ii] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = src_svr_addr_B[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __mul_ii(uint8_t src_svr_addr_A, uint8_t src_svr_addr_B, uint8_t result_svr_addr); + + +// [div.ii] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = src_svr_addr_B[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __div_ii(uint8_t src_svr_addr_A, uint8_t src_svr_addr_B, uint8_t result_svr_addr); + + +// [fmadd.iii] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = src_svr_addr_C[9:5], src_svr_addr_B[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __fmadd_iii(uint8_t src_svr_addr_A, uint8_t src_svr_addr_B, uint8_t src_svr_addr_C, uint8_t result_svr_addr); + + +// [sqrt.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __sqrt_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr); + + +// [exp.i] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = mode[1:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __exp_i(uint8_t src_svr_addr_A, uint8_t mode, uint8_t result_svr_addr); + + +// [ln.i] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = mode[1:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __ln_i(uint8_t src_svr_addr_A, uint8_t mode, uint8_t result_svr_addr); + + +// [abs.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __abs_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr); + + +// [neg.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __neg_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr); + + +// [sign.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __sign_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr); + + +// [step.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __step_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr); + + +// [ceil.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __ceil_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr); + + +// [trunc.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __trunc_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr); + + +// [floor.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __floor_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr); + + +// [rne.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __rne_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr); + + +// [and.ii] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = src_svr_addr_B[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __and_ii(uint8_t src_svr_addr_A, uint8_t src_svr_addr_B, uint8_t result_svr_addr); + + +// [or.ii] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = src_svr_addr_B[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __or_ii(uint8_t src_svr_addr_A, uint8_t src_svr_addr_B, uint8_t result_svr_addr); + + +// [not.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __not_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr); + + +// [shift.i] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = shift_mode[4], shift_num[3:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __shift_i(uint8_t src_svr_addr_A, uint8_t shift_num, uint8_t shift_mode, uint8_t result_svr_addr); + + + +//================================= +// Format conversion +//================================= +// [scvt.qh] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = offset[31:16], scale[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __scvt_qh(uint32_t vector_size, uint16_t scale, uint16_t offset, uint16_t r2_sel); + + +// [scvt.hq] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = offset[31:16], scale[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __scvt_hq(uint32_t vector_size, uint16_t scale, uint16_t offset, uint16_t r2_sel); + + +// [scvt.ih] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = offset[31:16], scale[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __scvt_ih(uint32_t vector_size, uint16_t scale, uint16_t offset, uint16_t r2_sel); + + +// [scvt.hi] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = offset[31:16], scale[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __scvt_hi(uint32_t vector_size, uint16_t scale, uint16_t offset, uint16_t r2_sel); + + +// [scvt.hn] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = offset[31:16], scale[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __scvt_hn(uint32_t vector_size, uint16_t scale, uint16_t offset, uint16_t r2_sel); + + +// [fcvt.sh] rd rs1 +// rd = result_data[63:0] +// rs1 = input_data[63:0] +#define __fcvt_sh(input_data)\ + ({uint64_t res; __asm__ volatile("fcvt.sh %[rd], %[r1]\n" : [rd] "=r"(res) : [r1] "r"(input_data)); res;}) + + +// [fcvt.hs] rd rs1 +// rd = result_data[63:0] +// rs1 = input_data[63:0] +#define __fcvt_hs(input_data)\ + ({uint64_t res; __asm__ volatile("fcvt.hs %[rd], %[r1]\n" : [rd] "=r"(res) : [r1] "r"(input_data)); res;}) + + +// [fcvt.dh] rd rs1 +// rd = result_data[63:0] +// rs1 = input_data[63:0] +#define __fcvt_dh(input_data)\ + ({uint64_t res; __asm__ volatile("fcvt.dh %[rd], %[r1]\n" : [rd] "=r"(res) : [r1] "r"(input_data)); res;}) + + +// [fcvt.hd] rd rs1 +// rd = result_data[63:0] +// rs1 = input_data[63:0] +#define __fcvt_hd(input_data)\ + ({uint64_t res; __asm__ volatile("fcvt.hd %[rd], %[r1]\n" : [rd] "=r"(res) : [r1] "r"(input_data)); res;}) + + + +//================================= +// Activation +//================================= +// [prelu] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = slop_value[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __prelu(uint32_t vector_size, uint16_t slop_value, uint16_t r2_sel); + + +// [gelu] rs1 +// rs1 = vector_size[23:0] +#define __gelu(vector_size)\ + __asm__ volatile("gelu %[r1]\n" : : [r1] "r"(vector_size)); + + +// [tanh] rs1 +// rs1 = vector_size[23:0] +#define __tanh(vector_size)\ + __asm__ volatile("tanh %[r1]\n" : : [r1] "r"(vector_size)); + + +// [sigm] rs1 +// rs1 = vector_size[23:0] +#define __sigm(vector_size)\ + __asm__ volatile("sigm %[r1]\n" : : [r1] "r"(vector_size)); + + +// [prelu.i] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr_A[4:0] +// rs2 = slop_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __prelu_i(uint8_t src_svr_addr_A, uint16_t slop_value, uint8_t result_svr_addr, uint16_t r2_sel); + + +// [gelu.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __gelu_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr); + + +// [tanh.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __tanh_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr); + + +// [sigm.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __sigm_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr); + + + +//================================= +// Softmax +//================================= +// [esum] rs1 rs2 rs3 rs4 +// rs1 = vertor_size[23:0] +// rs2 = accumulated_data[31:16], max_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __esum(uint32_t vector_size, uint16_t max_value, uint16_t accumulated_data, uint8_t result_svr_addr, uint16_t r2_sel); + + +// [softmax] rs1 rs2 rs4 +// rs1 = vertor_size[23:0] +// rs2 = esum_value[31:16], max_value[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __softmax(uint32_t vector_size, uint16_t max_value, uint16_t esum_value, uint16_t r2_sel); + + +// [esum.i] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr_A[4:0] +// rs2 = accumulated_data[31:16], max_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __esum_i(uint8_t src_svr_addr_A, uint16_t max_value, uint16_t accumulated_data, uint8_t result_svr_addr, uint16_t r2_sel); + + +// [softmax.i] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr_A[4:0] +// rs2 = esum_value[31:16], max_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __softmax_i(uint8_t src_svr_addr_A, uint16_t max_value, uint16_t esum_value, uint8_t result_svr_addr, uint16_t r2_sel); + + + +//================================= +// Pooling +//================================= +// [pool.m] rs1 rs2 +// rs1 = col_OUT[63:48], row_OUT[47:32], col_IN[31:16], row_IN[15:0] +// rs2 = col_stride[31:24], row_stride[23:16], col_K[15:8], row_K[7:0] +__attribute__((noinline)) void __pool_m(uint16_t row_IN, uint16_t col_IN, uint16_t row_OUT , uint16_t col_OUT , + uint8_t row_K , uint8_t col_K , uint8_t row_stride, uint8_t col_stride); + + +// [pool.a] rs1, rs2 +// rs1 = col_OUT[63:48], row_OUT[47:32], col_IN[31:16], row_IN[15:0] +// rs2 = k_value[47:32]. col_stride[31:24], row_stride[23:16], col_K[15:8], row_K[7:0] +__attribute__((noinline)) void __pool_a(uint16_t row_IN, uint16_t col_IN, uint16_t row_OUT , uint16_t col_OUT , + uint8_t row_K , uint8_t col_K , uint8_t row_stride, uint8_t col_stride, uint16_t k_value); + + + +//================================= +// Memory operation +//================================= +// [tpose] rs1 rs2 rs3 +// rs1 = dtype[56], rw_dir[49:48], src_addr[36:0] +// rs2 = dim2_size[63:48], dim1_size[47:32], dim0_size[31:16], dim2[9:8], dim1[5:4], dim0[1:0] +// rs3 = dst_addr[36:0] +__attribute__((noinline)) void __tpose(uint64_t src_addr , uint64_t dst_addr , + uint16_t dim2_size, uint16_t dim1_size, uint16_t dim0_size, uint8_t dim2, uint8_t dim1, uint8_t dim0, + uint8_t rw_dir , uint8_t dtype ); + + +// [fill] rs1 rs2 rs3 +// rs1 = dir[48], dst_addr[36:0] +// rs2 = height[63:48], length[47:32], write_stride[31:0] +// rs3 = fill_pattern[63:0] +__attribute__((noinline)) void __fill(uint64_t dst_addr, uint32_t write_stride, uint16_t length, uint16_t height, uint64_t fill_pattern, uint8_t dir); + + + +//================================= +// DMA +//================================= +// [load] rs1 rs2 rs3 +// rs1 = src_addr[63:27], dst_addr[26:0] +// rs2 = heigth[63:48], length[47:32], read_stride[31:0] +// rs3 = write_stride[31:0] +__attribute__((noinline)) void __load(uint64_t src_addr, uint32_t dst_addr, uint32_t read_stride, uint16_t length, uint16_t height, uint32_t write_stride); + + +// [store] rs1 rs2 rs3 +// rs1 = dst_addr[63:27], src_addr[26:0] +// rs2 = heigth[63:48], length[47:32], write_stride[31:0] +// rs3 = read_stride[31:0] +__attribute__((noinline)) void __store(uint32_t src_addr, uint64_t dst_addr, uint32_t write_stride, uint16_t length, uint16_t height, uint32_t read_stride); + + +// [copy] rs1 rs2 rs3 +// rs1 = dst_addr[58:32], src_addr[26:0] +// rs2 = heigth[63:48], length[47:32], read_stride[31:0] +// rs3 = write_stride[31:0] +__attribute__((noinline)) void __copy(uint32_t src_addr, uint32_t dst_addr, uint32_t read_stride, uint16_t length, uint16_t height, uint32_t write_stride); + + +// [load.svr] rs1 rs2 +// rs1 = addr_L1SPM[23:0] +// rs2 = svr_addr[4:0] +#define __load_svr(addr_L1SPM, svr_addr)\ + __asm__ volatile("load.svr %[r1], %[r2]\n" : : [r1] "r"(addr_L1SPM), [r2] "r"(svr_addr)); + + +// [store.svr] rs1 rs2 +// rs1 = addr_L1SPM[23:0] +// rs2 = svr_addr[4:0] +#define __store_svr(addr_L1SPM, svr_addr)\ + __asm__ volatile("store.svr %[r1], %[r2]\n" : : [r1] "r"(addr_L1SPM), [r2] "r"(svr_addr)); + + +// [mcast.s2l] rs1 rs2 rs3 +// rs1 = src_addr[58:32], dst_addr[23:0] +// rs2 = heigth[63:48], length[47:32], read_stride[31:0] +// rs3 = target_spu[63:0] +__attribute__((noinline)) void __mcast_s2l(uint32_t src_addr, uint32_t dst_addr, uint32_t read_stride, uint16_t length, uint16_t height, uint64_t target_spu); + + +// [mcast.g2s] rs1 rs2 rs3 +// rs1 = src_addr[63:27], dst_addr[26:0] +// rs2 = heigth[63:48], length[47:32], read_stride[31:0] +// rs3 = target_nest[63:0] +__attribute__((noinline)) void __mcast_g2s(uint64_t src_addr, uint32_t dst_addr, uint32_t read_stride, uint16_t length, uint16_t height, uint64_t target_nest); + + +// [mcast.s2s] rs1 rs2 rs3 +// rs1 = target_nest_sel[63], src_nest_id[61:56], dst_addr[55:27], src_addr[26:0] +// rs2 = heigth[63:48], length[47:32], read_stride[31:0] +// rs3 = target_nest[63:32], write_stride[31:0] +__attribute__((noinline)) void __mcast_s2s(uint32_t src_addr, uint32_t dst_addr, uint32_t read_stride, uint16_t length, uint16_t height, uint32_t write_stride, + uint8_t src_nest_id, uint8_t target_nest_sel, uint32_t target_nest); + + +// [copy.mem] rs1 rs2 rs3 +// rs1 = write_stride_L[63:48], src_addr_DDR[36:0] +// rs3 = heigth[63:48], length[47:32], read_stride[31:0] +// rs2 = write_stride_H[63:48], dst_addr_DDR[36:0] +__attribute__((noinline)) void __copy_mem(uint64_t src_addr_DDR, uint64_t dst_addr_DDR, uint32_t read_stride, uint16_t length, uint16_t height, uint16_t write_stride_L, uint16_t write_stride_H); + + + +//================================= +// SPR +//================================= +// [rdspr] rd rs1 +// rd = spr_data[63:0] +// rs1 = nest_id[29:24], spu_id[21:16], spr_addr[11:0] +__attribute__((noinline)) uint64_t __rdspr(uint16_t spr_addr, uint8_t nest_id, uint8_t spu_id); + + +// [wrspr] rs1 rs2 rs3 rs4 +// rs1 = wrstb_n[23:16], spr_addr[11:0] +// rs2 = spr_data[63:0] +// rs3 = target_mask[63:0] +__attribute__((noinline)) void __wrspr(uint16_t spr_addr, uint8_t wrstb_n, uint64_t spr_data, uint64_t target_mask); + + +// [opset] rs1 rs2 +// rs1 = target[0] +// rs2 = data[63:0] +#define __opset(target, data)\ + __asm__ volatile ("opset %[r1], %[r2]\n" : : [r1] "r"(target), [r2] "r"(data)); + + +// [cpsvr] rs1 rs2 +// rs1 = svr_addr[4:0] +// rs2 = byte_size[1:0] +#define __cpsvr(svr_addr, byte_size)\ + __asm__ volatile ("cpsvr %[r1], %[r2]\n" : : [r1] "r"(svr_addr), [r2] "r"(byte_size)); + + +// [mvsvr] rs1 rs2 rs3 +// rs1 = src_svr_addr[4:0] +// rs2 = dst_svr_addr[4:0] +// rs3 = wrstrb_n[31:0] +__attribute__((noinline)) void __mvsvr(uint8_t src_svr_addr, uint8_t dst_svr_addr, uint32_t wrstrb_n); + + + +//================================= +// Credit +//================================= +// [credit.ld] rs1 rs2 +// rs1 = target_spu[63:0] +// rs2 = target_nest[63:0] +#define __credit_ld(target_spu, target_nest)\ + __asm__ volatile ("credit.ld %[r1], %[r2]\n" : : [r1] "r"(target_spu), [r2] "r"(target_nest)); + + +// [credit.st] rs1 +// rs1 = target_spu[63:0] +#define __credit_st(target_spu)\ + __asm__ volatile ("credit.st %[r1]\n" : : [r1] "r"(target_spu)); + + +// [credit.chk] rs1 +// rs1 = target_spu[63:0] +#define __credit_chk(target_spu) \ + do { uint64_t __tmp = (uint64_t)(target_spu); \ + __asm__ volatile ("credit.chk %[r1]\n" : : [r1] "r"(__tmp)); } while(0) + + + +//================================= +// Micro Code +//================================= +// [mexec] rd rs1 +// rd = status[0] +// rs1 = start_addr[36:0] +// rs2 = target_nest[63:0] +#define __mexec(start_addr, target_nest)\ + ({uint64_t res; __asm__ volatile("mexec %[rd], %[r1], %[r2]\n" : [rd] "=r"(res) : [r1] "r"(start_addr), [r2] "r"(target_nest)); res;}) + + +// [mbar] +#define __mbar()\ + __asm__ volatile("mbar\n" : : ) + + +// [msync] +#define __msync()\ + __asm__ volatile("msync\n" : : ) + + +// [eom] +#define __eom()\ + __asm__ volatile("eom\n" : : ) + + + +//================================= +// Sync +//================================= +// [bar] rd +// rd = status[0] +#define __bar()\ + ({uint64_t res; __asm__ volatile("bar %[rd]\n" : [rd] "=r"(res) : ); res;}) + + +// [wait] rd rs1 +// rd = status[0] +// rs1 = wait_clk_count[31:0] +#define __wait(wait_clk_count)\ + ({uint64_t res; __asm__ volatile("wait %[rd], %[r1]\n" : [rd] "=r"(res) : [r1] "r"(wait_clk_count)); res;}) + + +// [intr] rs1 +// rs1 = intr[63:0] +#define __intr(intr)\ + __asm__ volatile("intr %[r1]\n" : : [r1] "r"(intr)) + + +// [flush] +#define __flush()\ + __asm__ volatile("flush\n" : : ) + + +// [halt] +#define __halt()\ + __asm__ volatile("halt\n" : : ) + + + +//================================= +// Warp operation +//================================= +// [start.t] rs1 or imm12 +#define __start_t(spu_id)\ + __asm__ volatile("start.t %[r1], %[imm]\n" : : [r1] "r"(spu_id), [imm] "i"(0x000)) + + +#define __start_ti(spu_id)\ + __asm__ volatile("start.t %[r1], %[imm]\n" : : [r1] "r"(0x0), [imm] "i"(spu_id | 0x400)) + + +// [end.t] rs1 or imm12 +#define __end_t(spu_id)\ + __asm__ volatile("end.t %[r1], %[imm]\n" : : [r1] "r"(spu_id), [imm] "i"(0x000)) + + +#define __end_ti(spu_id)\ + __asm__ volatile("end.t %[r1], %[imm]\n" : : [r1] "r"(0x0), [imm] "i"(spu_id | 0x400)) + + +// [start.s] +#define __start_s()\ + __asm__ volatile("start.s\n" : : ) + + +// [end.s] +#define __end_s()\ + __asm__ volatile("end.s\n" : : ) + + +// [start.p] rs1 or imm12 +#define __start_p(nest_id)\ + do { \ + __asm__ volatile("start.p %[r1], %[imm]\n" : : [r1] "r"(nest_id), [imm] "i"(0x000)); \ + __gtx_giss_unmask_threads(); \ + } while (0) + + +#define __start_pi(nest_id)\ + do { \ + __asm__ volatile("start.p %[r1], %[imm]\n" : : [r1] "r"(0x0), [imm] "i"(nest_id | 0x400)); \ + __gtx_giss_unmask_threads(); \ + } while (0) + + +// [end.p] rs1 or imm12 +#define __end_p(nest_id)\ + __asm__ volatile("end.p %[r1], %[imm]\n" : : [r1] "r"(nest_id), [imm] "i"(0x000)) + + +#define __end_pi(nest_id)\ + __asm__ volatile("end.p %[r1], %[imm]\n" : : [r1] "r"(0x0), [imm] "i"(nest_id | 0x400)) + + +// [split]/[join] +#if defined(GGML_GTX_KERNEL_UART_CHECKPOINTS) && GGML_GTX_KERNEL_UART_CHECKPOINTS +void ggml_gtx_kernel_uart_checkpoint(char phase, uint32_t source_line); + +#define __split() \ + do { \ + ggml_gtx_kernel_uart_checkpoint('S', (uint32_t) __LINE__); \ + __asm__ volatile("split\n" : : ); \ + } while (0) + +#define __join() \ + do { \ + __asm__ volatile("join\n" : : ); \ + ggml_gtx_kernel_uart_checkpoint('J', (uint32_t) __LINE__); \ + } while (0) +#else +#define __split()\ + __asm__ volatile("split\n" : : ) + +#define __join()\ + __asm__ volatile("join\n" : :) +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/depend/ggml-gtx/include/ggml-gtx/intrinsics/intrin_level2.h b/depend/ggml-gtx/include/ggml-gtx/intrinsics/intrin_level2.h new file mode 100644 index 0000000..2249d6c --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx/intrinsics/intrin_level2.h @@ -0,0 +1,209 @@ +//================================================================== +// Copyright : (C) Supergate - All Rights Reserved +// Project : GSF / VTS +// Description : GTX intrinsic definition (level 2) +// convenience intrinsic for programming +// Author : mh.kim ( NPU Div - NPU Core Team ) +// Last Update : 2026/01/23 +//================================================================== + +#ifndef INTRIN_LEVEL2_H +#define INTRIN_LEVEL2_H + +#include + +#ifndef __gtx_giss_unmask_threads +#ifdef GGML_GTX_GISS_MASK_COMPAT +#define __gtx_giss_unmask_threads() \ + do { \ + uint64_t __gtx_giss_zero = 0; \ + uint64_t __gtx_giss_thread = 0x400; \ + __asm__ volatile ("opset %[r1], %[r2]\n" : : [r1] "r"(__gtx_giss_zero), [r2] "r"(__gtx_giss_zero)); \ + __asm__ volatile ("wrspr %[r1], %[r2]\n" : : [r1] "r"(__gtx_giss_thread), [r2] "r"(__gtx_giss_zero)); \ + } while (0) +#else +#define __gtx_giss_unmask_threads() do { } while (0) +#endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +//================================= +// Auto setting +//================================= +// [init L1SPM bank addr] +// 4bit input: en_restore_R[3], en_restore_C[2], en_restore_B[1], en_restore_A[0] +// initial spm_addr_A - 0x00000 +// initial spm_addr_B - 0x20000 +// initial spm_addr_C - 0x30000 +// initial spm_addr_R - 0x50000 +__attribute__((noinline)) void __init_spm_addr(uint8_t target_spm_bank); + + +// [set L1SPM bank addr] +// spm_addr_A - 24bit +// spm_addr_B - 24bit +// spm_addr_C - 24bit +// spm_addr_R - 24bit +__attribute__((noinline)) void __set_spm_addr(uint32_t spm_addr_R, uint32_t spm_addr_C, uint32_t spm_addr_B, uint32_t spm_addr_A); + + +// [set L1SPM bank addr A] +// spm_addr_A - 24bit +__attribute__((noinline)) void __set_spm_addr_A(uint32_t spm_addr_A); + + +// [set L1SPM bank addr B] +// spm_addr_B - 24bit +__attribute__((noinline)) void __set_spm_addr_B(uint32_t spm_addr_B); + + +// [set L1SPM bank addr C] +// spm_addr_C - 24bit +__attribute__((noinline)) void __set_spm_addr_C(uint32_t spm_addr_C); + + +// [set L1SPM bank addr R] +// spm_addr_R - 24bit +__attribute__((noinline)) void __set_spm_addr_R(uint32_t spm_addr_R); + + + +//================================= +// DMA with credit +//================================= +// [load with credit] +// load rs1 = src_addr[63:27], dst_addr[26:0] +// load rs2 = heigth[63:48], length[47:32], read_stride[31:0] +// load rs3 = write_stride[31:0] +// additional = auto_credit(1bit), target_spu(64bit), target_nest(64bit) +// auto_credit 1: 'load' with 'credit_ld' +// 0: 'load' +__attribute__((noinline)) void __load_cr(uint64_t src_addr , uint32_t dst_addr , uint32_t read_stride, uint16_t length, uint16_t height, uint32_t write_stride, + uint8_t auto_credit, uint64_t target_spu, uint64_t target_nest); + + +// [store with credit] +// store rs1 = dst_addr[63:27], src_addr[26:0] +// store rs2 = heigth[63:48], length[47:32], write_stride[31:0] +// store rs3 = read_stride[31:0] +// additional = auto_credit(1bit), target_spu(64bit) +// auto_credit 1: 'store' with 'credit_st' +// 0: 'store' +__attribute__((noinline)) void __store_cr(uint32_t src_addr , uint64_t dst_addr , uint32_t write_stride, uint16_t length, uint16_t height, uint32_t read_stride, + uint8_t auto_credit, uint64_t target_spu); + + + +//================================= +// Renaming intrinsic for programming (same operation) +//================================= +// [sum] +// same as __mm_o +// rs1 = col_A_size[31:16] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __sum(uint16_t col_A_size, uint8_t result_svr_addr); + + +// [sum_acc] +// same as __mmc_o +// rs1 = col_A_size[31:16] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __sum_acc(uint16_t col_A_size, uint8_t result_svr_addr); + + +// [dot_product] +// same as __mm_v +// rs1 = col_A_size[31:16] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __dot_product(uint16_t col_A_size, uint8_t result_svr_addr); + + +// [dot_product_acc] +// same as __mmc_v +// rs1 = col_A_size[31:16] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __dot_product_acc(uint16_t col_A_size, uint8_t result_svr_addr); + + +// [start thread] +// same as __start_t +#define __start_thread(spu_id)\ + __asm__ volatile("start.t %[r1], %[imm]\n" : : [r1] "r"(spu_id), [imm] "i"(0x000)); + + +// [start thread immediate] +// same as __start_ti +#define __start_threadi(spu_id)\ + __asm__ volatile("start.t %[r1], %[imm]\n" : : [r1] "r"(0x0), [imm] "i"(spu_id | 0x400)); + + +// [end thread] +// same as __end_t +#define __end_thread(spu_id)\ + __asm__ volatile("end.t %[r1], %[imm]\n" : : [r1] "r"(spu_id), [imm] "i"(0x000)); + + +// [end thread immediate] +// same as __end_ti +#define __end_threadi(spu_id)\ + __asm__ volatile("end.t %[r1], %[imm]\n" : : [r1] "r"(0x0), [imm] "i"(spu_id | 0x400)); + + +// [start shared] +// same as __start_s +#define __start_shared()\ + __asm__ volatile("start.s\n" : : ); + + +// [end shared] +// same as __end_s +#define __end_shared()\ + __asm__ volatile("end.s\n" : : ); + + +// [start plan] +// same as __start_p +#define __start_plan(nest_id)\ + do { \ + __asm__ volatile("start.p %[r1], %[imm]\n" : : [r1] "r"(nest_id), [imm] "i"(0x000)); \ + __gtx_giss_unmask_threads(); \ + } while (0) + + +// [start plan immediate] +// same as __start_pi +#define __start_plani(nest_id)\ + do { \ + __asm__ volatile("start.p %[r1], %[imm]\n" : : [r1] "r"(0x0), [imm] "i"(nest_id | 0x400)); \ + __gtx_giss_unmask_threads(); \ + } while (0) + + +// [end plan] +// same as __end_p +#define __end_plan(nest_id)\ + __asm__ volatile("end.p %[r1], %[imm]\n" : : [r1] "r"(nest_id), [imm] "i"(0x000)); + + +// [end plan immediate] +// same as __end_pi +#define __end_plani(nest_id)\ + __asm__ volatile("end.p %[r1], %[imm]\n" : : [r1] "r"(0x0), [imm] "i"(nest_id | 0x400)); + + + +//================================= +// Usable function (not consisting of GTX-ISA) +//================================= +// [int16_to_fp16] +// integer 16 format convert to floating point 16 format (RNE role) +__attribute__((noinline)) uint16_t __int16_to_fp16(int16_t integer); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/depend/ggml-gtx/include/ggml-gtx/intrinsics/intrin_level3.h b/depend/ggml-gtx/include/ggml-gtx/intrinsics/intrin_level3.h new file mode 100644 index 0000000..597e357 --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx/intrinsics/intrin_level3.h @@ -0,0 +1,147 @@ +//================================================================== +// Copyright : (C) Supergate - All Rights Reserved +// Project : GSF / VTS +// Description : GTX intrinsic definition (level 3) +// complex operation intrinsic consisting of compound instructions +// Author : mh.kim ( NPU Div - NPU Core Team ) +// Last Update : 2026/01/23 +//================================================================== + +#ifndef INTRIN_LEVEL3_H +#define INTRIN_LEVEL3_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +//================================= +// Special function - memory +//================================= +// [pad] +// fill 0 to dst_addr & DMA to dst_addr per channel +__attribute__((noinline)) void __pad(uint64_t src_addr , uint64_t dst_addr , + uint32_t fill_stride , uint16_t length , uint16_t height, + uint8_t ystr , uint8_t yend , uint8_t xstr , uint8_t xend , uint64_t fill_pattern, + uint32_t channel_stride, uint16_t channel_num, uint8_t rw , uint8_t d_type); + + +// [load_3d] +// 3D load from L2SPM to L1SPM +// src_addr_L2SPM(28bit) / dst_addr_L1SPM(24bit) +// heigth(16bit) / length(16bit) / read_stride(32bit) / depth(16bit) / depth_stride(32bit) +__attribute__((noinline)) void __load_3d(uint32_t src_addr_L2SPM, uint32_t dst_addr_L1SPM, uint32_t read_stride, uint16_t length, uint16_t height, uint32_t depth_stride, uint16_t depth); + + +// [store_3d] +// 3D store from L1SPM to L2SPM +// src_addr_L1SPM(24bit) / dst_addr_L2SPM(28bit) +// heigth(16bit) / length(16bit) / write_stride(32bit) / depth(16bit) / depth_stride(32bit) +__attribute__((noinline)) void __store_3d(uint32_t src_addr_L1SPM, uint32_t dst_addr_L2SPM, uint32_t write_stride, uint16_t length, uint16_t height, uint32_t depth_stride, uint16_t depth); + + + +//================================= +// Special function - activation +//================================= +// [relu] +// relu(x) = (x < 0) ? 0 : x +// input : L1SPM_R_BANK (l1_R_addr) +// result: L1SPM_A_BANK (l1_A_addr) +__attribute__((noinline)) void __relu(uint32_t vector_size, uint32_t l1_A_addr, uint32_t l1_R_addr); + + +// [relu6] +// relu6(x) = (x < 0) ? 0 : +// (x > 6) ? 6 : x +// input : L1SPM_R_BANK (l1_R_addr) +// result: L1SPM_A_BANK (l1_A_addr) +__attribute__((noinline)) void __relu6(uint32_t vector_size, uint32_t l1_A_addr, uint32_t l1_R_addr); + + +// [lrelu] +// lrelu(x) = (x > 0) ? x : 0.01 * x +// input : L1SPM_R_BANK (l1_R_addr) +// result: L1SPM_A_BANK (l1_A_addr) +__attribute__((noinline)) void __lrelu(uint32_t vector_size); + + +// [silu] +// silu(x) = x * sigmoid(x) +// input : L1SPM_R_BANK (l1_R_addr) +// result: L1SPM_A_BANK (l1_A_addr) +__attribute__((noinline)) void __silu(uint32_t vector_size, uint32_t l1_A_addr, uint32_t l1_B_addr, uint32_t l1_R_addr); + + + +//================================= +// Special function - math +//================================= +// [var] +// variance(x) = sum(((x-mean(x))/sqrt(x_num))^2) +// input : L1SPM_A_BANK (l1_A_addr) +// mean : SVR_1 +// result: SVR_2 +__attribute__((noinline)) void __var(uint32_t vector_size, uint32_t l1_A_addr, uint32_t l1_B_addr, uint32_t l1_R_addr); + + + +//================================= +// Special function - normalization +//================================= +// [layernorm] +// layernorm(x) = (x-mean(x))/sqrt(var(x)+epsilon) +// input : L1SPM_A_BANK (l1_A_addr) +// result: L1SPM_R_BANK (l1_R_addr) +__attribute__((noinline)) void __layernorm(uint32_t vector_size, uint32_t l1_A_addr, uint32_t l1_B_addr, uint32_t l1_R_addr, uint16_t epsilon); + + +// [layernorm_aff] +// layernorm_aff(x) = (scale*((x-mean(x))/var(x)+epsilon)) + shift +// input : L1SPM_A_BANK (l1_A_addr) +// scale : L1SPM_B_BANK (l1_B_addr) +// shift : L1SPM_C_BANK (l1_C_addr) +// result: L1SPM_R_BANK (l1_R_addr) +__attribute__((noinline)) void __layernorm_aff(uint32_t vector_size, uint32_t l1_A_addr, uint32_t l1_B_addr, uint32_t l1_C_addr, uint32_t l1_R_addr, uint16_t epsilon); + + +// [rmsnorm] +// rmsnorm(x) = x / R(MS(x)+epsilon) +// input : L1SPM_A_BANK (l1_A_addr) +// result: L1SPM_R_BANK (l1_R_addr) +__attribute__((noinline)) void __rmsnorm(uint32_t vector_size, uint32_t l1_A_addr, uint32_t l1_B_addr, uint32_t l1_R_addr, uint16_t epsilon); + + +// [rmsnorm_scale] +// rmsnorm_scale(x) = (x/R(MS(x)+epsilon))* scale +// input : L1SPM_A_BANK (l1_A_addr) +// scale : L1SPM_B_BANK (l1_B_addr) +// result: L1SPM_R_BANK (l1_R_addr) +__attribute__((noinline)) void __rmsnorm_scale(uint32_t vector_size, uint32_t l1_A_addr, uint32_t l1_B_addr, uint32_t l1_R_addr, uint16_t epsilon); + + +// [batchnorm] +// batchnorm(x) = (x-mean)/sqrt(var_epsilon) +// input : L1SPM_A_BANK (l1_A_addr) +// mean : fp16 value +// var : fp16 value +// result: L1SPM_R_BANK (l1_R_addr) +__attribute__((noinline)) void __batchnorm(uint32_t vector_size, uint16_t mean, uint16_t var, uint32_t l1_A_addr, uint32_t l1_R_addr, uint16_t epsilon); + + +// [batchnorm_aff] +// batchnorm_aff(x) = (x-mean)/sqrt(var+epsilon)*scale + shift +// input : L1SPM_A_BANK (l1_A_addr) +// mean : fp16 value +// var : fp16 value +// scale : fp16 value +// shift : fp16 value +// result: L1SPM_R_BANK (l1_R_addr) +__attribute__((noinline)) void __batchnorm_aff(uint32_t vector_size, uint16_t mean, uint16_t var, uint16_t scale, uint16_t shift, uint32_t l1_A_addr, uint32_t l1_R_addr, uint16_t epsilon); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/depend/ggml-gtx/include/ggml-gtx/sc_print.h b/depend/ggml-gtx/include/ggml-gtx/sc_print.h new file mode 100644 index 0000000..b139484 --- /dev/null +++ b/depend/ggml-gtx/include/ggml-gtx/sc_print.h @@ -0,0 +1,17 @@ +/** + * @file sc_print.h + * @brief Simulator Console Print declarations for GTX kernel builds. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef GTX_SC_PRINT_H +#define GTX_SC_PRINT_H + +#include + +int putchar(int ch); +int printf(const char *format, ...); +int vprintf(const char *format, va_list va); + +#endif /* GTX_SC_PRINT_H */ diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-abs.c b/depend/ggml-gtx/src/ggml-gtx-kernel-abs.c new file mode 100644 index 0000000..e594c49 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-abs.c @@ -0,0 +1,6 @@ +#define GGML_GTX_UNARY_KERNEL_ID GGML_GTX_KERNEL_ABS_F16 +#define GGML_GTX_UNARY_KERNEL_FN ggml_gtx_kernel_abs_f16 +#define GGML_GTX_UNARY_INPUT_BANK GGML_GTX_UNARY_BANK_A +#define GGML_GTX_UNARY_OUTPUT_BANK GGML_GTX_UNARY_BANK_R +#define GGML_GTX_UNARY_APPLY(elems) __abs_v(elems) +#include "ggml-gtx-kernel-unary-ref.h" diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-add-fp32.c b/depend/ggml-gtx/src/ggml-gtx-kernel-add-fp32.c new file mode 100644 index 0000000..0eee3a9 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-add-fp32.c @@ -0,0 +1,281 @@ +//================================================================== +// GGML GTX kernel: ADD F16 + F32 vector -> F16 +//================================================================== + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#define GGML_GTX_ADD_FP32_L1_A 0x00000u +#define GGML_GTX_ADD_FP32_L1_B 0x20000u +#define GGML_GTX_ADD_FP32_L1_C 0x30000u +#define GGML_GTX_ADD_FP32_L1_R 0x50000u +#define GGML_GTX_ADD_FP32_CONST_ONE 0x3c00u + +void ggml_gtx_compute_add_f16_f32_to_f16_tv( + uint64_t ddr_a_f16, + uint64_t ddr_b_f32, + uint64_t ddr_result_f16, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest); + +int ggml_gtx_kernel_add_f16_f32(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_ADD_F16_F32_TO_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_ADD_F16_F32_TO_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + const uint64_t rows64 = (uint64_t) desc->ne[1] * + (uint64_t) desc->ne[2] * + (uint64_t) desc->ne[3]; + const uint64_t row_bytes_f16 = (uint64_t) desc->ne[0] * 2u; + const uint64_t row_bytes_f32 = (uint64_t) desc->ne[0] * 4u; + if (desc->dst_addr == 0 || + desc->src_addr[0] == 0 || + desc->src_addr[1] == 0 || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->src_type[1] != GGML_GTX_TYPE_F32 || + desc->ne[0] == 0 || + rows64 == 0 || + desc->ne[0] > UINT16_MAX || + desc->src_ne[0][0] != desc->ne[0] || + desc->src_ne[1][0] != desc->ne[0] || + desc->src_ne[0][1] != desc->ne[1] || + desc->src_ne[0][2] != desc->ne[2] || + desc->src_ne[0][3] != desc->ne[3] || + desc->src_ne[1][1] != 1 || + desc->src_ne[1][2] != 1 || + desc->src_ne[1][3] != 1 || + desc->nb[1] != row_bytes_f16 || + desc->src_nb[0][1] != row_bytes_f16 || + desc->src_nb[1][1] != row_bytes_f32 || + row_bytes_f16 > UINT16_MAX || + row_bytes_f16 > GGML_GTX_L1_BANK_SIZE || + row_bytes_f32 > UINT16_MAX || + row_bytes_f32 > GGML_GTX_L1_BANK_SIZE) { + return -2; + } + + const uint16_t cols = (uint16_t) desc->ne[0]; + const uint64_t max_rows_per_call = UINT16_MAX; + for (uint64_t row = 0; row < rows64; row += max_rows_per_call) { + const uint64_t remaining = rows64 - row; + const uint16_t rows_chunk = + (uint16_t) (remaining < max_rows_per_call ? remaining : max_rows_per_call); + const uint64_t offset = row * desc->nb[1]; + ggml_gtx_compute_add_f16_f32_to_f16_tv(desc->src_addr[0] + offset, + desc->src_addr[1], + desc->dst_addr + offset, + rows_chunk, + cols, + GGML_GTX_NEST_NUM, + GGML_GTX_SPU_NUM_PER_NEST); + } + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_add_f16_f32(arg); +} + +void ggml_gtx_printf(const char * fmt, ...) { + (void) fmt; +} + +void ggml_gtx_compute_add_f16_f32_to_f16_tv( + uint64_t ddr_a_f16, + uint64_t ddr_b_f32, + uint64_t ddr_result_f16, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest) +{ +#ifdef GGML_GTX_USE_GTX + if (i_row == 0 || i_col == 0 || spu_num_per_nest == 0) { + return; + } + + const uint8_t nest_id = 0; + const uint16_t not_use = 0xBEEF; + const uint32_t row_bytes_f16 = (uint32_t) i_col * 2u; + const uint32_t row_bytes_f32 = (uint32_t) i_col * 4u; + const uint16_t eff_thread = 1; + const uint16_t eff_thread_hex = GGML_GTX_EFF_MASK_16(eff_thread); + const uint16_t cal_quot = (uint16_t) (i_row / eff_thread); + const uint16_t cal_mod = (uint16_t) (i_row % eff_thread); + const uint32_t cal_max = + row_bytes_f16 != 0 ? (GGML_GTX_L1_BANK_SIZE / row_bytes_f16) : 0; + + (void) nest_num; + + if (row_bytes_f16 == 0 || + row_bytes_f32 == 0 || + row_bytes_f16 > UINT16_MAX || + row_bytes_f32 > UINT16_MAX || + row_bytes_f32 > GGML_GTX_L1_BANK_SIZE || + cal_max == 0) { + return; + } + + __mcast_g2s(ddr_b_f32, + GGML_GTX_L2_B, + row_bytes_f32, + (uint16_t) row_bytes_f32, + 1, + (uint64_t) 0x1u << nest_id); + + __mcast_s2l(GGML_GTX_L2_B, + GGML_GTX_ADD_FP32_L1_C, + row_bytes_f32, + (uint16_t) row_bytes_f32, + 1, + eff_thread_hex); + + uint32_t bank_offset_shared = 0; + uint32_t bank_offset_thread = 0; + + __split(); + + __start_plani(nest_id); + + __set_spm_addr(GGML_GTX_ADD_FP32_L1_R, + GGML_GTX_ADD_FP32_L1_C, + GGML_GTX_ADD_FP32_L1_B, + GGML_GTX_ADD_FP32_L1_A); + __wrspr(0x800, 0x0, GGML_GTX_ADD_FP32_CONST_ONE, 0x0); + __store_svr(GGML_GTX_ADD_FP32_L1_B, 0x0); + + __start_shared(); + + for (uint8_t thread_id = 0; thread_id < eff_thread; thread_id++) { + const uint16_t cal_num_per_thread = + (thread_id < cal_mod) ? (cal_quot + 1) : cal_quot; + const uint32_t offset_thread = + (((thread_id < cal_mod) ? + (uint32_t) thread_id * (cal_quot + 1) : + (uint32_t) cal_mod * (cal_quot + 1) + + (uint32_t) (thread_id - cal_mod) * cal_quot) * + row_bytes_f16); + + __load(ddr_a_f16 + offset_thread, + GGML_GTX_L2_A + bank_offset_shared, + row_bytes_f16, + (uint16_t) row_bytes_f16, + cal_num_per_thread, + row_bytes_f16); + + __credit_ld((uint16_t) (0x1u << thread_id), not_use); + bank_offset_shared += GGML_GTX_L2_BANK_OFFSET; + } + + bank_offset_shared = 0; + + for (uint8_t thread_id = 0; thread_id < eff_thread; thread_id++) { + const uint16_t target = (uint16_t) (0x1u << thread_id); + const uint16_t cal_num_per_thread = + (thread_id < cal_mod) ? (cal_quot + 1) : cal_quot; + const uint32_t offset_thread = + (((thread_id < cal_mod) ? + (uint32_t) thread_id * (cal_quot + 1) : + (uint32_t) cal_mod * (cal_quot + 1) + + (uint32_t) (thread_id - cal_mod) * cal_quot) * + row_bytes_f16); + + __credit_chk(target); + __store(GGML_GTX_L2_RESULT + bank_offset_shared, + ddr_result_f16 + offset_thread, + row_bytes_f16, + (uint16_t) row_bytes_f16, + cal_num_per_thread, + row_bytes_f16); + __credit_st(target); + bank_offset_shared += GGML_GTX_L2_BANK_OFFSET; + } + + __end_shared(); + + for (uint8_t thread_id = 0; thread_id < eff_thread; thread_id++) { + __start_thread(thread_id); + + const uint16_t cal_num_per_thread = + (thread_id < cal_mod) ? (cal_quot + 1) : cal_quot; + const uint16_t iteration = + (uint16_t) ((cal_num_per_thread + cal_max - 1) / cal_max); + const uint32_t l2_a_thread = GGML_GTX_L2_A + bank_offset_thread; + const uint32_t l2_res_thread = GGML_GTX_L2_RESULT + bank_offset_thread; + uint32_t offset_cal = 0; + + __credit_chk(not_use); + + for (uint16_t iter = 0; iter < iteration; iter++) { + const bool last = (iter + 1) == iteration; + const uint16_t cal_tile = + last ? + (uint16_t) (cal_num_per_thread - (uint32_t) iter * cal_max) : + (uint16_t) cal_max; + uint32_t l1_a_addr = GGML_GTX_ADD_FP32_L1_A; + uint32_t l1_r_addr = GGML_GTX_ADD_FP32_L1_R; + + __load(l2_a_thread + offset_cal, + l1_a_addr, + row_bytes_f16, + (uint16_t) row_bytes_f16, + cal_tile, + not_use); + __credit_ld(not_use, not_use); + + for (uint16_t row = 0; row < cal_tile; row++) { + __set_spm_addr_A(l1_a_addr); + __set_spm_addr_B(GGML_GTX_ADD_FP32_L1_B); + __set_spm_addr_C(GGML_GTX_ADD_FP32_L1_C); + __set_spm_addr_R(l1_r_addr); + + __mmc(i_col, 1, 1); + + l1_a_addr += row_bytes_f16; + l1_r_addr += row_bytes_f16; + } + + __set_spm_addr_A(GGML_GTX_ADD_FP32_L1_A); + __set_spm_addr_B(GGML_GTX_ADD_FP32_L1_B); + __set_spm_addr_C(GGML_GTX_ADD_FP32_L1_C); + __set_spm_addr_R(GGML_GTX_ADD_FP32_L1_R); + + __store(GGML_GTX_ADD_FP32_L1_R, + l2_res_thread + offset_cal, + row_bytes_f16, + (uint16_t) row_bytes_f16, + cal_tile, + not_use); + + offset_cal += row_bytes_f16 * cal_tile; + } + + __credit_st(not_use); + bank_offset_thread += GGML_GTX_L2_BANK_OFFSET; + + __end_thread(thread_id); + } + + __end_plani(nest_id); + + __join(); +#else + (void) ddr_a_f16; + (void) ddr_b_f32; + (void) ddr_result_f16; + (void) i_row; + (void) i_col; + (void) nest_num; + (void) spu_num_per_nest; +#endif +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-add.c b/depend/ggml-gtx/src/ggml-gtx-kernel-add.c new file mode 100644 index 0000000..947619a --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-add.c @@ -0,0 +1,208 @@ +//================================================================== +// GGML GTX kernel: ADD, L2-stashed single-nest tile +//================================================================== + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-kernel-args.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#define GGML_GTX_BINARY_L1_MAX_ELEMS ((64u * 1024u - 2u) / GGML_GTX_DTYPE_SIZE) + +static uint32_t ggml_gtx_binary_rows(const ggml_gtx_kernel_launch_desc_t * desc) { + return desc->ne[1] * desc->ne[2] * desc->ne[3]; +} + +static uint32_t ggml_gtx_binary_row_for_tid(uint32_t rows, uint32_t tid) { + return (uint32_t) (((uint64_t) rows * tid) / GGML_GTX_SPU_NUM_PER_NEST); +} + +static uint32_t ggml_gtx_binary_rhs_row(const ggml_gtx_kernel_launch_desc_t * desc, + uint32_t row) { + const uint32_t i1 = row % desc->ne[1]; + const uint32_t q = row / desc->ne[1]; + const uint32_t i2 = q % desc->ne[2]; + const uint32_t i3 = q / desc->ne[2]; + const uint32_t j1 = desc->src_ne[1][1] == 1u ? 0u : i1; + const uint32_t j2 = desc->src_ne[1][2] == 1u ? 0u : i2; + const uint32_t j3 = desc->src_ne[1][3] == 1u ? 0u : i3; + return (j3 * desc->src_ne[1][2] + j2) * desc->src_ne[1][1] + j1; +} + +static void ggml_gtx_compute_add_vv_l2(const ggml_gtx_kernel_launch_desc_t * desc) { + const uint32_t ne0 = desc->ne[0]; + const uint32_t rows = ggml_gtx_binary_rows(desc); + const uint32_t row_bytes = ne0 * GGML_GTX_DTYPE_SIZE; + const uint32_t rhs_row_bytes = desc->src_ne[1][0] * GGML_GTX_DTYPE_SIZE; + + __split(); + __start_plan((uint8_t) desc->nest_id); + for (uint32_t tid = 0u; tid < GGML_GTX_SPU_NUM_PER_NEST; ++tid) { + const uint32_t thread_row0 = ggml_gtx_binary_row_for_tid(rows, tid); + const uint32_t thread_row1 = ggml_gtx_binary_row_for_tid(rows, tid + 1u); + + __start_thread((uint8_t) tid); + __set_spm_addr(GGML_GTX_L1_BANK_R_OFFSET, + GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, + GGML_GTX_L1_BANK_A_OFFSET); + + uint32_t row = thread_row0; + while (row < thread_row1) { + const uint32_t i1 = row % desc->ne[1]; + const uint32_t rows_to_i1_boundary = desc->ne[1] - i1; + const uint32_t cols = ne0 < GGML_GTX_BINARY_L1_MAX_ELEMS ? + ne0 : GGML_GTX_BINARY_L1_MAX_ELEMS; + const uint32_t max_rows_l1 = GGML_GTX_BINARY_L1_MAX_ELEMS / cols; + uint32_t block_rows = thread_row1 - row; + if (block_rows > rows_to_i1_boundary) { + block_rows = rows_to_i1_boundary; + } + if (block_rows > max_rows_l1) { + block_rows = max_rows_l1; + } + + for (uint32_t col0 = 0u; col0 < ne0; col0 += cols) { + const uint32_t block_cols = ne0 - col0 < cols ? ne0 - col0 : cols; + const uint32_t block_elems = block_cols * block_rows; + const uint16_t block_bytes = (uint16_t) (block_cols * GGML_GTX_DTYPE_SIZE); + const uint32_t a_off = row * row_bytes + col0 * GGML_GTX_DTYPE_SIZE; + const uint32_t b_row = ggml_gtx_binary_rhs_row(desc, row); + const uint32_t b_off = b_row * rhs_row_bytes + + col0 * GGML_GTX_DTYPE_SIZE; + const uint32_t b_stride = desc->src_ne[1][1] == 1u ? + 0u : rhs_row_bytes; + + __load((uint32_t) desc->l2_src_addr[0] + a_off, + GGML_GTX_L1_BANK_A_OFFSET, + row_bytes, + block_bytes, + (uint16_t) block_rows, + block_bytes); + __load((uint32_t) desc->l2_src_addr[1] + b_off, + GGML_GTX_L1_BANK_B_OFFSET, + b_stride, + block_bytes, + (uint16_t) block_rows, + block_bytes); + __add_vv(block_elems); + __store(GGML_GTX_L1_BANK_R_OFFSET, + (uint32_t) desc->l2_dst_addr + a_off, + row_bytes, + block_bytes, + (uint16_t) block_rows, + block_bytes); + } + row += block_rows; + } + __end_thread((uint8_t) tid); + } + __end_plan((uint8_t) desc->nest_id); + __join(); +} + +static void ggml_gtx_compute_add_vs_l2(const ggml_gtx_kernel_launch_desc_t * desc, + const uint16_t * scalars) { + const uint32_t ne0 = desc->ne[0]; + const uint32_t rows = ggml_gtx_binary_rows(desc); + const uint32_t row_bytes = ne0 * GGML_GTX_DTYPE_SIZE; + const uint32_t cols = ne0 < GGML_GTX_BINARY_L1_MAX_ELEMS ? + ne0 : GGML_GTX_BINARY_L1_MAX_ELEMS; + const uint32_t max_rows_l1 = GGML_GTX_BINARY_L1_MAX_ELEMS / cols; + + uint32_t row = 0u; + while (row < rows) { + const uint32_t i1 = row % desc->ne[1]; + const uint32_t rows_to_i1_boundary = desc->ne[1] - i1; + uint32_t block_rows = rows - row; + if (block_rows > rows_to_i1_boundary) block_rows = rows_to_i1_boundary; + if (block_rows > max_rows_l1) block_rows = max_rows_l1; + + const uint16_t scalar = scalars[ggml_gtx_binary_rhs_row(desc, row)]; + for (uint32_t col0 = 0u; col0 < ne0; col0 += cols) { + const uint32_t block_cols = ne0 - col0 < cols ? ne0 - col0 : cols; + const uint32_t block_elems = block_cols * block_rows; + const uint16_t block_bytes = (uint16_t)(block_cols * GGML_GTX_DTYPE_SIZE); + const uint32_t a_off = row * row_bytes + col0 * GGML_GTX_DTYPE_SIZE; + const uint32_t l2_src = (uint32_t)desc->l2_src_addr[0] + a_off; + const uint32_t l2_dst = (uint32_t)desc->l2_dst_addr + a_off; + + __split(); + __start_plan((uint8_t)desc->nest_id); + __start_thread(0u); + __set_spm_addr(GGML_GTX_L1_BANK_R_OFFSET, + GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, + GGML_GTX_L1_BANK_A_OFFSET); + __load(l2_src, GGML_GTX_L1_BANK_A_OFFSET, + row_bytes, block_bytes, (uint16_t)block_rows, block_bytes); + __add_vs(block_elems, scalar, 0u); + __store(GGML_GTX_L1_BANK_R_OFFSET, l2_dst, + row_bytes, block_bytes, (uint16_t)block_rows, block_bytes); + __end_thread(0u); + __end_plan((uint8_t)desc->nest_id); + __join(); + } + row += block_rows; + } +} + +int ggml_gtx_kernel_add_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_ADD_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_ADD_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + const uint32_t elems = desc->ne[0]; + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_L2_STASHED_N1 || + desc->nest_id >= GGML_GTX_NEST_NUM || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->src_type[1] != GGML_GTX_TYPE_F16 || + desc->elem_size != GGML_GTX_DTYPE_SIZE || + elems == 0 || + desc->src_ne[0][0] != elems || + (desc->src_ne[1][0] != elems && desc->src_ne[1][0] != 1u) || + elems > UINT16_MAX) { + return -2; + } + + uint32_t rows = 1u; + for (uint32_t d = 1u; d < 4u; ++d) { + if (desc->src_ne[0][d] != desc->ne[d] || + (desc->src_ne[1][d] != 1u && desc->src_ne[1][d] != desc->ne[d]) || + desc->ne[d] == 0u || rows > UINT32_MAX / desc->ne[d]) { + return -2; + } + rows *= desc->ne[d]; + } + + if (desc->src_ne[1][0] == elems) { + ggml_gtx_compute_add_vv_l2(desc); + return 0; + } + + const ggml_gtx_add_args_t * params = + (const ggml_gtx_add_args_t *)ggml_gtx_kernel_args( + karg, GGML_GTX_ADD_ARGS_HEADER_SIZE); + const uint64_t expected_scalar_count = + (uint64_t)desc->src_ne[1][1] * desc->src_ne[1][2] * desc->src_ne[1][3]; + const uint64_t expected_size = GGML_GTX_ADD_ARGS_HEADER_SIZE + + expected_scalar_count * sizeof(uint16_t); + if (params == 0 || params->version != GGML_GTX_ADD_ARGS_VERSION || + params->size != karg->desc.args_size || params->size != expected_size || + params->scalar_count != expected_scalar_count) { + return -3; + } + ggml_gtx_compute_add_vs_l2(desc, params->scalars); + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_add_f16(arg); +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-ceil.c b/depend/ggml-gtx/src/ggml-gtx-kernel-ceil.c new file mode 100644 index 0000000..08fa66d --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-ceil.c @@ -0,0 +1,6 @@ +#define GGML_GTX_UNARY_KERNEL_ID GGML_GTX_KERNEL_CEIL_F16 +#define GGML_GTX_UNARY_KERNEL_FN ggml_gtx_kernel_ceil_f16 +#define GGML_GTX_UNARY_INPUT_BANK GGML_GTX_UNARY_BANK_A +#define GGML_GTX_UNARY_OUTPUT_BANK GGML_GTX_UNARY_BANK_R +#define GGML_GTX_UNARY_APPLY(elems) __ceil_v(elems) +#include "ggml-gtx-kernel-unary-ref.h" diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-clamp.c b/depend/ggml-gtx/src/ggml-gtx-kernel-clamp.c new file mode 100644 index 0000000..6922f22 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-clamp.c @@ -0,0 +1,95 @@ +//================================================================== +// GGML GTX kernel: CLAMP +// +// Descriptor wrapper around ggml clamp semantics. +// dst[i] = max(min(src0[i], max), min) +//================================================================== + +#include + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#define BANK_A GGML_GTX_L1_BANK_A_OFFSET +#define BANK_B GGML_GTX_L1_BANK_B_OFFSET +#define BANK_C GGML_GTX_L1_BANK_C_OFFSET +#define BANK_R GGML_GTX_L1_BANK_R_OFFSET +#define DTYPE 2u +#define SPU_NUM_PER_NEST GGML_GTX_SPU_NUM_PER_NEST +#define NOT_USE 0xBEEFu + +static inline uint32_t elems_for_tid(uint32_t total_elems, uint8_t tid) { + const uint32_t q = total_elems / SPU_NUM_PER_NEST; + const uint32_t r = total_elems % SPU_NUM_PER_NEST; + return tid < r ? q + 1u : q; +} + +static inline uint32_t start_for_tid(uint32_t total_elems, uint8_t tid) { + const uint32_t q = total_elems / SPU_NUM_PER_NEST; + const uint32_t r = total_elems % SPU_NUM_PER_NEST; + if (tid < r) { + return (uint32_t) tid * (q + 1u); + } + return r * (q + 1u) + ((uint32_t) tid - r) * q; +} + +int ggml_gtx_kernel_clamp_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_CLAMP_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_CLAMP_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + const uint32_t total_elems = desc->ne[0]; + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_L2_STASHED_N1 || + desc->nest_id >= GGML_GTX_NEST_NUM || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->elem_size != DTYPE || + total_elems == 0 || + total_elems > UINT16_MAX) { + return -2; + } + + const uint16_t clamp_min = ggml_gtx_fp32_bits_to_fp16((uint32_t) desc->op_params[0]); + const uint16_t clamp_max = ggml_gtx_fp32_bits_to_fp16((uint32_t) desc->op_params[1]); + + __split(); + __start_plan((uint8_t) desc->nest_id); + for (uint8_t tid = 0; tid < SPU_NUM_PER_NEST; tid++) { + const uint32_t elems_this_tid = elems_for_tid(total_elems, tid); + __start_thread(tid); + if (elems_this_tid > 0u) { + const uint32_t elem_start = start_for_tid(total_elems, tid); + const uint32_t chunk_off = elem_start * DTYPE; + const uint32_t chunk_bytes = elems_this_tid * DTYPE; + const uint16_t chunk_bytes_u16 = (uint16_t) chunk_bytes; + + __load((uint32_t) desc->l2_src_addr[0] + chunk_off, BANK_A, + chunk_bytes, chunk_bytes_u16, 1u, chunk_bytes_u16); + + __set_spm_addr(BANK_C, BANK_R, BANK_B, BANK_A); + __clamp_min(elems_this_tid, clamp_min, 0u); + + __set_spm_addr(BANK_R, BANK_C, BANK_B, BANK_C); + __clamp_max(elems_this_tid, clamp_max, 0u); + + __store(BANK_R, (uint32_t) desc->l2_dst_addr + chunk_off, + chunk_bytes, chunk_bytes_u16, 1u, chunk_bytes_u16); + } + __end_thread(tid); + } + __end_plan((uint8_t) desc->nest_id); + __join(); + + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_clamp_f16(arg); +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-common.c b/depend/ggml-gtx/src/ggml-gtx-kernel-common.c new file mode 100644 index 0000000..a6eb38d --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-common.c @@ -0,0 +1,597 @@ +//================================================================== +// GGML GTX kernel common utilities +//================================================================== + +#include "ggml-gtx/ggml-gtx-kernel-common.h" + +#if defined(GGML_GTX_KERNEL_UART_CHECKPOINTS) && GGML_GTX_KERNEL_UART_CHECKPOINTS +#ifndef GGML_GTX_KERNEL_UART_BASE +#define GGML_GTX_KERNEL_UART_BASE 0x10020000ull +#endif +#define GTX_KERNEL_UART_BASE ((uint64_t) GGML_GTX_KERNEL_UART_BASE) +#define GTX_KERNEL_UART_TXR_OFF 0x00ull +#define GTX_KERNEL_UART_SR_OFF 0x24ull +#define GTX_KERNEL_UART_TX_FULL (1u << 1) +#define GTX_KERNEL_UART_TIMEOUT 1000000u + +static volatile uint32_t * ggml_gtx_kernel_uart_reg(uint64_t offset) { + return (volatile uint32_t *)(uintptr_t)(GTX_KERNEL_UART_BASE + offset); +} + +static void ggml_gtx_kernel_uart_putc(char value) { + uint32_t timeout = GTX_KERNEL_UART_TIMEOUT; + while ((*ggml_gtx_kernel_uart_reg(GTX_KERNEL_UART_SR_OFF) & + GTX_KERNEL_UART_TX_FULL) != 0u && + timeout-- != 0u) { + __asm__ __volatile__("nop"); + } + if (timeout != 0u) { + *ggml_gtx_kernel_uart_reg(GTX_KERNEL_UART_TXR_OFF) = (uint8_t) value; + } +} + +static void ggml_gtx_kernel_uart_puts(const char * text) { + while (text != 0 && *text != '\0') { + if (*text == '\n') { + ggml_gtx_kernel_uart_putc('\r'); + } + ggml_gtx_kernel_uart_putc(*text++); + } +} + +void ggml_gtx_kernel_uart_checkpoint(char phase, uint32_t source_line) { + static const char hex[] = "0123456789abcdef"; + ggml_gtx_kernel_uart_puts("[GTX-KERNEL] "); + ggml_gtx_kernel_uart_putc(phase); + ggml_gtx_kernel_uart_puts("@0x"); + for (int shift = 28; shift >= 0; shift -= 4) { + ggml_gtx_kernel_uart_putc(hex[(source_line >> shift) & 0xfu]); + } + ggml_gtx_kernel_uart_putc('\n'); +} + +#endif + +uint16_t ggml_gtx_fp32_bits_to_fp16(uint32_t f32) { + // Software conversion (F32 bits -> FP16) + // Takes raw F32 IEEE-754 bits as uint32_t, returns FP16 bits. + // NOTE: __fcvt_sh returns 0 on ISS v1.1.4. + // NOTE: ISS has float<->int store-forwarding issues, so callers + // should read F32 data as uint32_t directly from DDR. + uint16_t sign = (f32 >> 16) & 0x8000; + int32_t exp = ((f32 >> 23) & 0xFF) - 127 + 15; + uint32_t frac = (f32 >> 13) & 0x3FF; + + if (exp <= 0) { + return sign; // Underflow to zero + } else if (exp >= 31) { + return sign | 0x7C00; // Overflow to infinity + } + return sign | ((uint16_t)exp << 10) | frac; +} + +uint32_t ggml_gtx_fp16_to_fp32_bits(uint16_t value) { + // Software conversion (FP16 -> F32 bits) + // Takes FP16 bits, returns raw F32 IEEE-754 bits as uint32_t. + // NOTE: __fcvt_hs returns 0 on ISS v1.1.4. + // NOTE: ISS has float<->int store-forwarding issues, so callers + // should write F32 data as uint32_t directly to DDR. + uint16_t sign = (value >> 15) & 1; + int32_t exp = (value >> 10) & 0x1F; + uint32_t frac = value & 0x3FF; + + uint32_t f32; + if (exp == 0) { + f32 = sign << 31; // Zero + } else if (exp == 31) { + f32 = (sign << 31) | 0x7F800000 | (frac << 13); // Inf/NaN + } else { + f32 = (sign << 31) | ((exp - 15 + 127) << 23) | (frac << 13); + } + + return f32; +} + +// Legacy wrappers (ggml_gtx_fp32_to_fp16 / ggml_gtx_fp16_to_fp32) are +// defined in ggml-gtx-port-riscv.c. Not duplicated here to avoid +// multiple-definition linker errors. + +//================================================================== +// Software F32 Arithmetic (uint32_t IEEE-754 bit manipulation) +// +// ISS v1.1.4 cannot convert float<->uint32_t (fmv.x.w returns 0). +// All F32 math is done on raw IEEE-754 bits as uint32_t. +// These are NOT speed-optimized; they handle the scalar pre-compute +// before __split() and the F32 bias accumulation in add_rel_pos. +//================================================================== + +// Well-known F32 bit constants +#define F32B_ZERO 0x00000000u // 0.0f +#define F32B_ONE 0x3F800000u // 1.0f +#define F32B_NEG_ONE 0xBF800000u // -1.0f +#define F32B_TWO 0x40000000u // 2.0f +#define F32B_HALF 0x3F000000u // 0.5f +#define F32B_LN2 0x3F317218u // ln(2) = 0.6931472 +#define F32B_LOG2E 0x3FB8AA3Bu // log2(e) = 1.442695 +#define F32B_INF 0x7F800000u // +infinity +#define F32B_SIGN_MASK 0x80000000u + +static inline int32_t f32b_is_zero(uint32_t a) { + return (a & ~F32B_SIGN_MASK) == 0; +} + +static inline int32_t f32b_is_neg(uint32_t a) { + return (a >> 31) & 1; +} + +uint32_t ggml_gtx_f32b_neg(uint32_t a) { + return a ^ F32B_SIGN_MASK; +} + +uint32_t ggml_gtx_f32b_abs(uint32_t a) { + return a & ~F32B_SIGN_MASK; +} + +// Compare: returns 1 if a > b (as F32), 0 otherwise. +int32_t ggml_gtx_f32b_gt(uint32_t a, uint32_t b) { + int32_t sa = f32b_is_neg(a); + int32_t sb = f32b_is_neg(b); + if (sa != sb) { + // Different signs: negative < positive (unless both zero) + if (f32b_is_zero(a) && f32b_is_zero(b)) return 0; + return sb; // a>b iff b is negative + } + // Same sign + if (sa) { + // Both negative: larger magnitude = smaller value + return (a & ~F32B_SIGN_MASK) < (b & ~F32B_SIGN_MASK); + } else { + return (a & ~F32B_SIGN_MASK) > (b & ~F32B_SIGN_MASK); + } +} + +// Integer to F32 bits +uint32_t ggml_gtx_f32b_from_int(int32_t n) { + if (n == 0) return F32B_ZERO; + uint32_t sign = 0; + uint32_t abs_n; + if (n < 0) { + sign = F32B_SIGN_MASK; + abs_n = (uint32_t)(-n); + } else { + abs_n = (uint32_t)n; + } + // Find highest bit position + int32_t shift = 0; + uint32_t tmp = abs_n; + while (tmp > 1) { tmp >>= 1; shift++; } + // exp = 127 + shift, mantissa = (abs_n << (23 - shift)) & 0x7FFFFF + uint32_t exp = (uint32_t)(127 + shift); + uint32_t mant; + if (shift >= 23) { + mant = (abs_n >> (shift - 23)) & 0x7FFFFFu; + } else { + mant = (abs_n << (23 - shift)) & 0x7FFFFFu; + } + return sign | (exp << 23) | mant; +} + +// F32 bits to integer (truncate toward zero) +int32_t ggml_gtx_f32b_to_int(uint32_t a) { + if (f32b_is_zero(a)) return 0; + int32_t exp = ((a >> 23) & 0xFF) - 127; + if (exp < 0) return 0; + uint32_t mant = (1u << 23) | (a & 0x7FFFFFu); // implicit 1 + int32_t result; + if (exp >= 23) { + result = (int32_t)(mant << (exp - 23)); + } else { + result = (int32_t)(mant >> (23 - exp)); + } + return f32b_is_neg(a) ? -result : result; +} + +// F32 addition: a + b (raw IEEE-754 bits) +uint32_t ggml_gtx_f32b_add(uint32_t a, uint32_t b) { + if (f32b_is_zero(a)) return b; + if (f32b_is_zero(b)) return a; + + uint32_t sign_a = a & F32B_SIGN_MASK; + uint32_t sign_b = b & F32B_SIGN_MASK; + int32_t exp_a = (a >> 23) & 0xFF; + int32_t exp_b = (b >> 23) & 0xFF; + // Mantissa with implicit 1 bit, shifted left by 2 for guard bits + uint32_t mant_a = ((1u << 23) | (a & 0x7FFFFFu)) << 2; + uint32_t mant_b = ((1u << 23) | (b & 0x7FFFFFu)) << 2; + + // Align exponents + if (exp_a > exp_b) { + int32_t shift = exp_a - exp_b; + if (shift >= 26) return a; // b too small + mant_b >>= shift; + } else if (exp_b > exp_a) { + int32_t shift = exp_b - exp_a; + if (shift >= 26) return b; // a too small + mant_a >>= shift; + exp_a = exp_b; + sign_a = sign_b; // will be corrected below + } + + uint32_t result_mant; + uint32_t result_sign; + if (sign_a == sign_b) { + // Same sign: add mantissas + result_mant = mant_a + mant_b; + result_sign = sign_a; + } else { + // Different signs: subtract mantissas + uint32_t abs_a = (a & ~F32B_SIGN_MASK); + uint32_t abs_b = (b & ~F32B_SIGN_MASK); + if (mant_a >= mant_b) { + result_mant = mant_a - mant_b; + // sign follows the larger magnitude + result_sign = (abs_a >= abs_b) ? (a & F32B_SIGN_MASK) + : (b & F32B_SIGN_MASK); + } else { + result_mant = mant_b - mant_a; + result_sign = (abs_b >= abs_a) ? (b & F32B_SIGN_MASK) + : (a & F32B_SIGN_MASK); + } + } + + if (result_mant == 0) return F32B_ZERO; + + // Normalize + while (result_mant >= (1u << 26)) { + result_mant >>= 1; + exp_a++; + } + while (result_mant < (1u << 25) && exp_a > 0) { + result_mant <<= 1; + exp_a--; + } + + if (exp_a >= 255) return result_sign | F32B_INF; + if (exp_a <= 0) return result_sign; // underflow + + // Remove guard bits and implicit 1 + uint32_t frac = (result_mant >> 2) & 0x7FFFFFu; + return result_sign | ((uint32_t)exp_a << 23) | frac; +} + +// F32 subtraction: a - b +uint32_t ggml_gtx_f32b_sub(uint32_t a, uint32_t b) { + return ggml_gtx_f32b_add(a, ggml_gtx_f32b_neg(b)); +} + +// F32 multiplication: a * b +uint32_t ggml_gtx_f32b_mul(uint32_t a, uint32_t b) { + if (f32b_is_zero(a) || f32b_is_zero(b)) return F32B_ZERO; + + uint32_t result_sign = (a ^ b) & F32B_SIGN_MASK; + int32_t exp_a = (a >> 23) & 0xFF; + int32_t exp_b = (b >> 23) & 0xFF; + int32_t exp_r = exp_a + exp_b - 127; + + uint32_t mant_a = (1u << 23) | (a & 0x7FFFFFu); + uint32_t mant_b = (1u << 23) | (b & 0x7FFFFFu); + + // 24x24 → 48 bit product using 16-bit chunks to avoid 64-bit overflow + uint32_t a_hi = mant_a >> 12; + uint32_t a_lo = mant_a & 0xFFF; + uint32_t b_hi = mant_b >> 12; + uint32_t b_lo = mant_b & 0xFFF; + + uint32_t prod_hi = a_hi * b_hi; // bits [47:24] approx + uint32_t prod_mid = a_hi * b_lo + a_lo * b_hi; // bits [35:12] + // prod_lo = a_lo * b_lo; // bits [23:0] — skip, only need top 24 bits + + // Combine: product ≈ (prod_hi << 24) + (prod_mid << 12) + prod_lo + // We need bits [47:24] of full product + uint32_t result_mant = prod_hi + (prod_mid >> 12); + + // Normalize: result_mant should have bit 23 as implicit 1 + // prod_hi is (12-bit * 12-bit) so up to 24 bits + if (result_mant >= (1u << 24)) { + result_mant >>= 1; + exp_r++; + } + // Shouldn't need to shift left in normal cases + + if (exp_r >= 255) return result_sign | F32B_INF; + if (exp_r <= 0) return result_sign; // underflow + + uint32_t frac = result_mant & 0x7FFFFFu; + return result_sign | ((uint32_t)exp_r << 23) | frac; +} + +// F32 division: a / b +uint32_t ggml_gtx_f32b_div(uint32_t a, uint32_t b) { + if (f32b_is_zero(a)) return F32B_ZERO; + if (f32b_is_zero(b)) return (a & F32B_SIGN_MASK) | F32B_INF; + + uint32_t result_sign = (a ^ b) & F32B_SIGN_MASK; + int32_t exp_a = (a >> 23) & 0xFF; + int32_t exp_b = (b >> 23) & 0xFF; + int32_t exp_r = exp_a - exp_b + 127; + + uint32_t mant_a = (1u << 23) | (a & 0x7FFFFFu); + uint32_t mant_b = (1u << 23) | (b & 0x7FFFFFu); + + // Long division: shift mant_a left to get more precision + // result = (mant_a << 23) / mant_b + uint64_t num = (uint64_t)mant_a << 23; + uint32_t result_mant = (uint32_t)(num / mant_b); + + // Normalize + while (result_mant >= (1u << 24)) { + result_mant >>= 1; + exp_r++; + } + while (result_mant < (1u << 23) && exp_r > 0) { + result_mant <<= 1; + exp_r--; + } + + if (exp_r >= 255) return result_sign | F32B_INF; + if (exp_r <= 0) return result_sign; + + uint32_t frac = result_mant & 0x7FFFFFu; + return result_sign | ((uint32_t)exp_r << 23) | frac; +} + +// F32 exp(x) — range reduction + polynomial +// exp(x) = 2^(x * log2(e)) = 2^n * 2^f where n=int, f=frac +uint32_t ggml_gtx_f32b_exp(uint32_t x) { + // exp(0) = 1 + if (f32b_is_zero(x)) return F32B_ONE; + + // Clamp to avoid overflow/underflow + // exp(88.7) > FLT_MAX, exp(-87.3) < FLT_MIN + // F32 bits for 88.0 ≈ 0x42B00000, -88.0 ≈ 0xC2B00000 + uint32_t abs_x = ggml_gtx_f32b_abs(x); + if (ggml_gtx_f32b_gt(abs_x, 0x42B00000u)) { + return f32b_is_neg(x) ? F32B_ZERO : F32B_INF; + } + + // x * log2(e) → get integer and fractional parts + uint32_t xlog2e = ggml_gtx_f32b_mul(x, F32B_LOG2E); + + // Floor to integer + int32_t n = ggml_gtx_f32b_to_int(xlog2e); + // Adjust floor for negative + uint32_t n_f32 = ggml_gtx_f32b_from_int(n); + if (ggml_gtx_f32b_gt(n_f32, xlog2e)) { + n--; + n_f32 = ggml_gtx_f32b_from_int(n); + } + + // f = x*log2(e) - n → in [0, 1) + uint32_t f = ggml_gtx_f32b_sub(xlog2e, n_f32); + + // 2^f via polynomial (minimax on [0,1)): + // 2^f ≈ 1 + 0.6931472*f + 0.2402265*f^2 + 0.0555041*f^3 + 0.00961813*f^4 + // Coefficients as F32 bits: + // c1 = 0.6931472 → 0x3F317218 + // c2 = 0.2402265 → 0x3E75FDF0 + // c3 = 0.0555041 → 0x3D635847 + // c4 = 0.00961813 → 0x3C1D9530 + #define EXP_C1 0x3F317218u + #define EXP_C2 0x3E75FDF0u + #define EXP_C3 0x3D635847u + #define EXP_C4 0x3C1D9530u + + // Horner: result = 1 + f*(c1 + f*(c2 + f*(c3 + f*c4))) + uint32_t p = ggml_gtx_f32b_mul(f, EXP_C4); + p = ggml_gtx_f32b_add(p, EXP_C3); + p = ggml_gtx_f32b_mul(p, f); + p = ggml_gtx_f32b_add(p, EXP_C2); + p = ggml_gtx_f32b_mul(p, f); + p = ggml_gtx_f32b_add(p, EXP_C1); + p = ggml_gtx_f32b_mul(p, f); + p = ggml_gtx_f32b_add(p, F32B_ONE); + + // Multiply by 2^n: adjust exponent by n + int32_t exp_p = (p >> 23) & 0xFF; + exp_p += n; + if (exp_p >= 255) return F32B_INF; + if (exp_p <= 0) return F32B_ZERO; + return (p & 0x807FFFFFu) | ((uint32_t)exp_p << 23); + + #undef EXP_C1 + #undef EXP_C2 + #undef EXP_C3 + #undef EXP_C4 +} + +// F32 log(x) — natural logarithm +// ln(x) = (exp_bits - 127) * ln(2) + ln(1 + f) +// where f = mantissa / 2^23 +uint32_t ggml_gtx_f32b_log(uint32_t x) { + // log(0) = -inf, log(neg) = NaN + if (f32b_is_zero(x)) return F32B_SIGN_MASK | F32B_INF; + if (f32b_is_neg(x)) return 0x7FC00000u; // NaN + if (x == F32B_ONE) return F32B_ZERO; + + int32_t exp_val = ((x >> 23) & 0xFF) - 127; + // f = mantissa as fraction in [1.0, 2.0) + // Reduce to [1, 2): set exponent to 127 + uint32_t f_bits = (x & 0x7FFFFFu) | 0x3F800000u; // 1.0 + frac + + // f_bits is now F32 for value in [1.0, 2.0) + // ln(f) ≈ polynomial on t = f - 1, t in [0, 1) + uint32_t t = ggml_gtx_f32b_sub(f_bits, F32B_ONE); + + // ln(1+t) ≈ t - t^2/2 + t^3/3 - t^4/4 + t^5/5 (Taylor series) + // For better accuracy: Padé or minimax + // Using: ln(1+t) ≈ t*(1 + t*(-0.5 + t*(0.3333 + t*(-0.25 + t*0.2)))) + // Coefficients as F32 bits: + // -0.5 → 0xBF000000 + // 0.33333 → 0x3EAAAAAB + // -0.25 → 0xBE800000 + // 0.2 → 0x3E4CCCCDu + #define LOG_C1 0xBF000000u + #define LOG_C2 0x3EAAAAAB + #define LOG_C3 0xBE800000u + #define LOG_C4 0x3E4CCCCDu + + uint32_t p = ggml_gtx_f32b_mul(t, LOG_C4); + p = ggml_gtx_f32b_add(p, LOG_C3); + p = ggml_gtx_f32b_mul(p, t); + p = ggml_gtx_f32b_add(p, LOG_C2); + p = ggml_gtx_f32b_mul(p, t); + p = ggml_gtx_f32b_add(p, LOG_C1); + p = ggml_gtx_f32b_mul(p, t); + p = ggml_gtx_f32b_add(p, F32B_ONE); + uint32_t ln_frac = ggml_gtx_f32b_mul(p, t); + + // ln(x) = exp_val * ln(2) + ln_frac + uint32_t exp_part = ggml_gtx_f32b_mul( + ggml_gtx_f32b_from_int(exp_val), F32B_LN2); + return ggml_gtx_f32b_add(exp_part, ln_frac); + + #undef LOG_C1 + #undef LOG_C2 + #undef LOG_C3 + #undef LOG_C4 +} + +// F32 softplus: log(1 + exp(x)) +// With clamp: x > 20 → x, x < -20 → 0 +uint32_t ggml_gtx_f32b_softplus(uint32_t x) { + // 20.0f = 0x41A00000 + #define F32B_20 0x41A00000u + if (ggml_gtx_f32b_gt(x, F32B_20)) return x; + if (ggml_gtx_f32b_gt(ggml_gtx_f32b_neg(x), F32B_20)) return F32B_ZERO; + uint32_t ex = ggml_gtx_f32b_exp(x); + uint32_t one_plus = ggml_gtx_f32b_add(F32B_ONE, ex); + return ggml_gtx_f32b_log(one_plus); + #undef F32B_20 +} + +//================================================================== +// Tiling Plan Initialization +//================================================================== + +void ggml_gtx_tile_plan_init( + ggml_gtx_tile_plan_t * plan, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest) +{ + plan->i_row = i_row; + plan->i_col = i_col; + plan->unit_per_thread = i_col * GGML_GTX_DTYPE_SIZE; + plan->max_cal_per_thread = GGML_GTX_L1_BANK_SIZE / plan->unit_per_thread; + + // Calculate effective nest count + uint32_t cal_num_nest = i_row; + plan->eff_nest_int = (cal_num_nest < nest_num) ? cal_num_nest : nest_num; + plan->eff_nest_hex = GGML_GTX_EFF_MASK_64(plan->eff_nest_int); + + // Distribute work across nests + uint32_t cal_quot_nest = cal_num_nest / plan->eff_nest_int; + uint32_t cal_mod_nest = cal_num_nest % plan->eff_nest_int; + + uint64_t cumulative_offset = 0; + + for (uint8_t i = 0; i < plan->eff_nest_int; i++) { + ggml_gtx_nest_tile_t * nest = &plan->nests[i]; + + // Work per nest + nest->cal_num = (i < cal_mod_nest) ? cal_quot_nest + 1 : cal_quot_nest; + + // Effective threads for this nest + nest->eff_thread_int = (nest->cal_num < spu_num_per_nest) ? + nest->cal_num : spu_num_per_nest; + nest->eff_thread_hex = GGML_GTX_EFF_MASK_16(nest->eff_thread_int); + + // Distribute work across threads + uint32_t cal_quot_thread = nest->cal_num / nest->eff_thread_int; + uint32_t cal_mod_thread = nest->cal_num % nest->eff_thread_int; + + for (uint8_t j = 0; j < nest->eff_thread_int; j++) { + ggml_gtx_thread_tile_t * thread = &nest->threads[j]; + + thread->cal_num = (j < cal_mod_thread) ? cal_quot_thread + 1 : cal_quot_thread; + thread->iteration = thread->cal_num / plan->max_cal_per_thread; + thread->remain = thread->cal_num % plan->max_cal_per_thread; + thread->offset = cumulative_offset; + + cumulative_offset += thread->cal_num * plan->unit_per_thread; + } + } +} + +void ggml_gtx_matmul_tile_init( + ggml_gtx_matmul_tile_t * plan, + uint16_t a_row, + uint16_t h_col, + uint16_t b_row, + uint16_t h_col_origin, + uint16_t res_col_origin, + uint16_t tile_a_row, + uint16_t tile_h_col, + uint16_t tile_b_row, + uint8_t nest_num, + uint8_t spu_num_per_nest) +{ + plan->a_row = a_row; + plan->h_col = h_col; + plan->b_row = b_row; + plan->h_col_origin = h_col_origin; + plan->res_col_origin = res_col_origin; + plan->tile_a_row = tile_a_row; + plan->tile_h_col = tile_h_col; + plan->tile_b_row = tile_b_row; + + // Calculate tile counts + plan->num_tile_row_a = GGML_GTX_CEIL_DIV(a_row, tile_a_row); + plan->num_tile_col_h = GGML_GTX_CEIL_DIV(h_col, tile_h_col); + plan->num_tile_row_b = GGML_GTX_CEIL_DIV(b_row, tile_b_row); + + // Last tile sizes + plan->last_row_a = GGML_GTX_LAST_TILE(a_row, tile_a_row); + plan->last_col_h = GGML_GTX_LAST_TILE(h_col, tile_h_col); + plan->last_row_b = GGML_GTX_LAST_TILE(b_row, tile_b_row); + + // Effective hardware (based on B tiling for nest distribution) + plan->eff_nest_int = (plan->num_tile_row_b < nest_num) ? + plan->num_tile_row_b : nest_num; + plan->eff_nest_hex = GGML_GTX_EFF_MASK_64(plan->eff_nest_int); + + // Distribute A rows across spreads (nests act as spreads for A tiling) + for (uint8_t spread_idx = 0; spread_idx < plan->eff_nest_int; spread_idx++) { + uint16_t row_a_quot = plan->num_tile_row_a / plan->eff_nest_int; + uint16_t row_a_mod = plan->num_tile_row_a % plan->eff_nest_int; + + plan->num_row_a_per_spread[spread_idx] = (spread_idx < row_a_mod) ? + row_a_quot + 1 : row_a_quot; + + plan->eff_thread_int[spread_idx] = (plan->num_row_a_per_spread[spread_idx] < spu_num_per_nest) ? + plan->num_row_a_per_spread[spread_idx] : spu_num_per_nest; + plan->eff_thread_hex[spread_idx] = GGML_GTX_EFF_MASK_16(plan->eff_thread_int[spread_idx]); + + for (uint8_t thread_id = 0; thread_id < plan->eff_thread_int[spread_idx]; thread_id++) { + uint16_t row_a_quot_thread = plan->num_row_a_per_spread[spread_idx] / plan->eff_thread_int[spread_idx]; + uint16_t row_a_mod_thread = plan->num_row_a_per_spread[spread_idx] % plan->eff_thread_int[spread_idx]; + + plan->num_row_a_per_thread[spread_idx][thread_id] = (thread_id < row_a_mod_thread) ? + row_a_quot_thread + 1 : row_a_quot_thread; + } + } + + // Distribute B rows across nests + for (uint8_t nest_id = 0; nest_id < plan->eff_nest_int; nest_id++) { + uint16_t row_b_quot = plan->num_tile_row_b / plan->eff_nest_int; + uint16_t row_b_mod = plan->num_tile_row_b % plan->eff_nest_int; + + plan->num_row_b_per_nest[nest_id] = (nest_id < row_b_mod) ? + row_b_quot + 1 : row_b_quot; + } +} + +//================================================================== +// Element-wise Operations (Tensor x Tensor) +//================================================================== diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-concat.c b/depend/ggml-gtx/src/ggml-gtx-kernel-concat.c new file mode 100644 index 0000000..246420a --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-concat.c @@ -0,0 +1,198 @@ +//================================================================== +// GGML GTX kernel: CONCAT +//================================================================== + +#include +#include +#include + +#include "ggml-gtx-baremetal.h" +#include "ggml-gtx-compute.h" +#include "ggml-gtx-port.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +int ggml_gtx_kernel_concat_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_CONCAT_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_CONCAT_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + const int32_t dim = desc->op_params[0]; + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_L2_STASHED_N1 || + desc->l2_dst_addr == 0 || + desc->l2_src_addr[0] == 0 || + desc->l2_src_addr[1] == 0 || + dim < 0 || + dim >= 4) { + return -2; + } + + for (int i = 0; i < 4; ++i) { + if (desc->ne[i] > UINT16_MAX || + desc->src_ne[0][i] > UINT16_MAX || + desc->src_ne[1][i] > UINT16_MAX) { + return -3; + } + const uint32_t expected = i == dim ? + desc->src_ne[0][i] + desc->src_ne[1][i] : desc->src_ne[0][i]; + if (desc->ne[i] != expected || + (i != dim && desc->src_ne[0][i] != desc->src_ne[1][i])) { + return -4; + } + } + const uint64_t dst_rows = (uint64_t)desc->ne[1] * desc->ne[2] * desc->ne[3]; + const uint64_t a_rows = (uint64_t)desc->src_ne[0][1] * desc->src_ne[0][2] * desc->src_ne[0][3]; + const uint64_t b_rows = (uint64_t)desc->src_ne[1][1] * desc->src_ne[1][2] * desc->src_ne[1][3]; + if (desc->ne[0] > UINT16_MAX / GGML_GTX_DTYPE_SIZE || + desc->src_ne[0][0] > UINT16_MAX / GGML_GTX_DTYPE_SIZE || + desc->src_ne[1][0] > UINT16_MAX / GGML_GTX_DTYPE_SIZE || + dst_rows > UINT16_MAX || a_rows > UINT16_MAX || b_rows > UINT16_MAX) { + return -5; + } + + ggml_gtx_compute_concat(desc->l2_src_addr[0], + desc->l2_src_addr[1], + desc->l2_dst_addr, + (uint16_t) desc->ne[0], + (uint16_t) desc->ne[1], + (uint16_t) desc->ne[2], + (uint16_t) desc->ne[3], + (uint16_t) desc->src_ne[0][dim], + dim, + GGML_GTX_DTYPE_SIZE, + 1, + 1); + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_concat_f16(arg); +} + +void ggml_gtx_compute_concat( + uint64_t ddr_a, + uint64_t ddr_b, + uint64_t ddr_dst, + uint16_t ne0, + uint16_t ne1, + uint16_t ne2, + uint16_t ne3, + uint16_t a_ne_dim, + int32_t dim, + uint16_t elem_size, + uint8_t nest_num, + uint8_t spu_num_per_nest) +{ + (void) nest_num; + (void) spu_num_per_nest; + + const uint32_t b_ne_dim = ((const uint32_t[]){ne0, ne1, ne2, ne3})[dim] - a_ne_dim; + const uint32_t a_ne[4] = { ne0, ne1, ne2, ne3 }; + const uint32_t b_ne[4] = { ne0, ne1, ne2, ne3 }; + uint32_t a_shape[4] = { a_ne[0], a_ne[1], a_ne[2], a_ne[3] }; + uint32_t b_shape[4] = { b_ne[0], b_ne[1], b_ne[2], b_ne[3] }; + a_shape[dim] = a_ne_dim; + b_shape[dim] = b_ne_dim; + +#ifdef GGML_GTX_USE_GTX + const uint32_t dst_row_bytes = (uint32_t)ne0 * elem_size; + const uint32_t a_row_bytes = a_shape[0] * elem_size; + const uint32_t b_row_bytes = b_shape[0] * elem_size; + + __start_plan(0); + __start_shared(); + if (dim == 0) { + const uint16_t rows = (uint16_t)((uint32_t)ne1 * ne2 * ne3); + __copy((uint32_t)ddr_a, (uint32_t)ddr_dst, + a_row_bytes, (uint16_t)a_row_bytes, rows, dst_row_bytes); + __copy((uint32_t)ddr_b, (uint32_t)ddr_dst + a_row_bytes, + b_row_bytes, (uint16_t)b_row_bytes, rows, dst_row_bytes); + } else if (dim == 1) { + const uint32_t dst_group_bytes = (uint32_t)ne1 * dst_row_bytes; + const uint32_t a_group_bytes = a_shape[1] * a_row_bytes; + const uint32_t b_group_bytes = b_shape[1] * b_row_bytes; + const uint32_t groups = (uint32_t)ne2 * ne3; + for (uint32_t g = 0; g < groups; ++g) { + __copy((uint32_t)ddr_a + g * a_group_bytes, + (uint32_t)ddr_dst + g * dst_group_bytes, + a_row_bytes, (uint16_t)a_row_bytes, + (uint16_t)a_shape[1], dst_row_bytes); + __copy((uint32_t)ddr_b + g * b_group_bytes, + (uint32_t)ddr_dst + g * dst_group_bytes + a_group_bytes, + b_row_bytes, (uint16_t)b_row_bytes, + (uint16_t)b_shape[1], dst_row_bytes); + } + } else if (dim == 2) { + const uint32_t dst_batch_bytes = (uint32_t)ne2 * ne1 * dst_row_bytes; + const uint32_t a_batch_bytes = a_shape[2] * a_shape[1] * a_row_bytes; + const uint32_t b_batch_bytes = b_shape[2] * b_shape[1] * b_row_bytes; + for (uint32_t i3 = 0; i3 < ne3; ++i3) { + __copy((uint32_t)ddr_a + i3 * a_batch_bytes, + (uint32_t)ddr_dst + i3 * dst_batch_bytes, + a_row_bytes, (uint16_t)a_row_bytes, + (uint16_t)(a_shape[2] * a_shape[1]), dst_row_bytes); + __copy((uint32_t)ddr_b + i3 * b_batch_bytes, + (uint32_t)ddr_dst + i3 * dst_batch_bytes + a_batch_bytes, + b_row_bytes, (uint16_t)b_row_bytes, + (uint16_t)(b_shape[2] * b_shape[1]), dst_row_bytes); + } + } else { + const uint32_t a_rows = a_shape[3] * a_shape[2] * a_shape[1]; + const uint32_t b_rows = b_shape[3] * b_shape[2] * b_shape[1]; + __copy((uint32_t)ddr_a, (uint32_t)ddr_dst, + a_row_bytes, (uint16_t)a_row_bytes, + (uint16_t)a_rows, dst_row_bytes); + __copy((uint32_t)ddr_b, (uint32_t)ddr_dst + a_rows * dst_row_bytes, + b_row_bytes, (uint16_t)b_row_bytes, + (uint16_t)b_rows, dst_row_bytes); + } + __end_shared(); + __end_plan(0); +#else + + volatile const uint8_t * a = (volatile const uint8_t *)(uintptr_t)ddr_a; + volatile const uint8_t * b = (volatile const uint8_t *)(uintptr_t)ddr_b; + volatile uint8_t * dst = (volatile uint8_t *)(uintptr_t)ddr_dst; + + const uint64_t a_nb0 = elem_size; + const uint64_t a_nb1 = (uint64_t) a_shape[0] * a_nb0; + const uint64_t a_nb2 = (uint64_t) a_shape[1] * a_nb1; + const uint64_t a_nb3 = (uint64_t) a_shape[2] * a_nb2; + const uint64_t b_nb0 = elem_size; + const uint64_t b_nb1 = (uint64_t) b_shape[0] * b_nb0; + const uint64_t b_nb2 = (uint64_t) b_shape[1] * b_nb1; + const uint64_t b_nb3 = (uint64_t) b_shape[2] * b_nb2; + + uint64_t dst_off = 0; + for (uint16_t i3 = 0; i3 < ne3; ++i3) { + for (uint16_t i2 = 0; i2 < ne2; ++i2) { + for (uint16_t i1 = 0; i1 < ne1; ++i1) { + for (uint16_t i0 = 0; i0 < ne0; ++i0) { + uint32_t idx[4] = { i0, i1, i2, i3 }; + const bool from_a = idx[dim] < a_ne_dim; + if (!from_a) { + idx[dim] -= a_ne_dim; + } + + const uint64_t src_off = + (uint64_t) idx[3] * (from_a ? a_nb3 : b_nb3) + + (uint64_t) idx[2] * (from_a ? a_nb2 : b_nb2) + + (uint64_t) idx[1] * (from_a ? a_nb1 : b_nb1) + + (uint64_t) idx[0] * (from_a ? a_nb0 : b_nb0); + volatile const uint8_t * src = from_a ? a : b; + for (uint16_t eb = 0; eb < elem_size; ++eb) { + dst[dst_off + eb] = src[src_off + eb]; + } + dst_off += elem_size; + } + } + } + } +#endif +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-cont.c b/depend/ggml-gtx/src/ggml-gtx-kernel-cont.c new file mode 100644 index 0000000..3ddea2f --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-cont.c @@ -0,0 +1,182 @@ +//================================================================== +// GGML GTX kernel: CONT +//================================================================== + +#include +#include +#include + +#include "ggml-gtx-baremetal.h" +#include "ggml-gtx-compute.h" +#include "ggml-gtx-port.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#ifdef GGML_GTX_USE_GTX +#include "ggml-gtx-plan.h" +#include "ggml-gtx-config.h" +#include "ggml-gtx/intrinsics/intrin.h" +#endif + +int ggml_gtx_kernel_cont_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_CONT_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_CONT_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + if (desc->dst_addr == 0 || + desc->src_addr[0] == 0 || + desc->ne[0] > UINT16_MAX || + desc->ne[1] > UINT16_MAX || + desc->ne[2] > UINT16_MAX || + desc->ne[3] > UINT16_MAX) { + return -2; + } + + const uint16_t elem_size = + desc->src_type[0] == GGML_GTX_TYPE_F32 ? 4 : 2; + ggml_gtx_compute_cont(desc->src_addr[0], + desc->dst_addr, + (uint16_t) desc->ne[0], + (uint16_t) desc->ne[1], + (uint16_t) desc->ne[2], + (uint16_t) desc->ne[3], + desc->src_nb[0][0], + desc->src_nb[0][1], + desc->src_nb[0][2], + desc->src_nb[0][3], + elem_size); + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_cont_f16(arg); +} + +void ggml_gtx_compute_cont( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t out_ne0, + uint16_t out_ne1, + uint16_t out_ne2, + uint16_t out_ne3, + uint32_t src_nb0, + uint32_t src_nb1, + uint32_t src_nb2, + uint32_t src_nb3, + uint16_t elem_size) +{ + uint8_t contiguous = 0; + const uint32_t row_bytes = (uint32_t)out_ne0 * elem_size; + const uint32_t plane_bytes = (uint32_t)out_ne1 * row_bytes; + const uint32_t block_bytes = (uint32_t)out_ne2 * plane_bytes; + + if (src_nb0 == elem_size) contiguous |= 1; + if (src_nb1 == row_bytes) contiguous |= 2; + if (src_nb2 == plane_bytes) contiguous |= 4; + if (src_nb3 == block_bytes) contiguous |= 8; + +#ifdef GGML_GTX_USE_GTX + if (row_bytes <= UINT16_MAX && contiguous == 0xF) { + __copy_mem(ddr_src, ddr_dst, + row_bytes, row_bytes, + (uint32_t)out_ne1 * out_ne2 * out_ne3, + row_bytes, row_bytes >> 16); + return; + } + + uint64_t dst_addr = ddr_dst; + + if (row_bytes <= UINT16_MAX && contiguous == 0x7) { + for (uint16_t b = 0; b < out_ne3; ++b) { + __copy_mem(ddr_src + (uint64_t)b * src_nb3, + dst_addr, + row_bytes, row_bytes, + (uint32_t)out_ne1 * out_ne2, + row_bytes, row_bytes >> 16); + dst_addr += block_bytes; + } + return; + } + + if (row_bytes <= UINT16_MAX && contiguous == 0x3) { + for (uint16_t b = 0; b < out_ne3; ++b) { + for (uint16_t ch = 0; ch < out_ne2; ++ch) { + __copy_mem(ddr_src + (uint64_t)b * src_nb3 + (uint64_t)ch * src_nb2, + dst_addr, + row_bytes, row_bytes, + out_ne1, + row_bytes, row_bytes >> 16); + dst_addr += plane_bytes; + } + } + return; + } + + if (row_bytes <= UINT16_MAX && contiguous == 0x1) { + for (uint16_t b = 0; b < out_ne3; ++b) { + for (uint16_t ch = 0; ch < out_ne2; ++ch) { + __copy_mem(ddr_src + (uint64_t)b * src_nb3 + (uint64_t)ch * src_nb2, + dst_addr, + src_nb1, (uint16_t)row_bytes, + out_ne1, + (uint16_t)row_bytes, row_bytes >> 16); + dst_addr += plane_bytes; + } + } + return; + } + + /* General permutation/gather: one strided DMA command per logical row + * instead of copying every element through the RISC-V core. */ + if (elem_size <= UINT16_MAX) { + for (uint16_t b = 0; b < out_ne3; ++b) { + for (uint16_t ch = 0; ch < out_ne2; ++ch) { + for (uint16_t row = 0; row < out_ne1; ++row) { + __copy_mem(ddr_src + (uint64_t)b * src_nb3 + + (uint64_t)ch * src_nb2 + (uint64_t)row * src_nb1, + dst_addr, + src_nb0, elem_size, + out_ne0, + elem_size, 0u); + dst_addr += row_bytes; + } + } + } + return; + } +#endif + + volatile uint8_t * src = (volatile uint8_t *)(uintptr_t)ddr_src; + volatile uint8_t * dst = (volatile uint8_t *)(uintptr_t)ddr_dst; + uint64_t dst_off = 0; + + for (uint16_t b = 0; b < out_ne3; ++b) { + for (uint16_t ch = 0; ch < out_ne2; ++ch) { + for (uint16_t row = 0; row < out_ne1; ++row) { + for (uint16_t col = 0; col < out_ne0; ++col) { + uint64_t src_off = (uint64_t)b * src_nb3 + (uint64_t)ch * src_nb2 + + (uint64_t)row * src_nb1 + (uint64_t)col * src_nb0; + for (uint16_t eb = 0; eb < elem_size; ++eb) { + dst[dst_off + eb] = src[src_off + eb]; + } + dst_off += elem_size; + } + } + } + } +} + +//================================================================== +// SET: Embed src1 into a sub-view of src0 at a given byte offset +// +// 1. Copy src0 → dst (if not in-place) +// 2. Copy src1 into dst at (offset, nb1, nb2, nb3) view strides +// +// Both tensors are F32. +//================================================================== diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-conv-2d-core.h b/depend/ggml-gtx/src/ggml-gtx-kernel-conv-2d-core.h new file mode 100644 index 0000000..310da69 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-conv-2d-core.h @@ -0,0 +1,526 @@ +#include "gtx/kernel_params.h" +#include "gtx/print.h" +#include "gtx/intrin.h" + +#include +#include +#include + + +#define FP16_B 2 +#define FP32_B 4 + +#define MAX_SPU_NUM 16 + +#define L1_CAP 0x60000 +#define L1_64K 0x10000 +#define L1_128K 0x20000 +#define L1_ALIGNMENT 0x100 +#define L2_ALIGNMENT 0x1000 +#define L2_CAP 0x1000000 + +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) + + +static const int MIN_OUT_T_LEN = 8; + +static inline int ceil_div(int a, int b){ return (a + b - 1u) / b; } +static inline int ceil_rem(int a, int b){ return (a + b - 1u) % b + 1; } +static inline int FIND_INPUT_LENGTH(int l_out, int k_eff, int stride) { + return (l_out - 1) * stride + k_eff; +} +static inline uint64_t align_up(uint64_t size, uint64_t align) { + return ((size) + (align) - 1) & ~((align) - 1); +} + +static inline uint64_t isqrt(uint64_t n) { + if (n < 2) return n; + uint64_t low = 0; + uint64_t high = (n / 2) + 1; + uint64_t res = 0; + while (low <= high) { + uint64_t mid = low + (high - low) / 2; + if (mid == 0) { low = mid + 1; continue; } + else if (mid <= n / mid) { low = mid + 1; res = mid; } + else { high = mid - 1; } + } + return res; +} +//=================== common ====================== + +typedef struct { + int c_in, h_in, w_in; + int c_out; + int k_size; + int pad_h, pad_w; + int stride, dilation; + bool bias; + + int hin_pad, win_pad; + int k_eff, k_n; + int h_out, w_out; +} ConvParams; + +typedef struct { + uint32_t input_addr; + uint32_t kernel_addr; + uint32_t bias_addr; + uint32_t bias_ones_addr; + uint32_t output_addr; + uint32_t total_bytes; +} L2Info; + +typedef struct { + // ==== 분할 계획 (Global) ==== + int cout_div, cout_rem_cnt, cout_len; + int h_div, h_rem_cnt, hout_len; + int w_div, w_rem_cnt, wout_len; + int use_spu; + + // ==== SPU별 할당 정보 (Per-SPU) - 추가된 부분 ==== + int start_cout[MAX_SPU_NUM]; + int start_hout[MAX_SPU_NUM]; + int start_wout[MAX_SPU_NUM]; + + int cout_assigned[MAX_SPU_NUM]; + int hout_assigned[MAX_SPU_NUM]; + int wout_assigned[MAX_SPU_NUM]; + +} SPUTileMap; + +typedef struct { + int cin_div; + int cout_div; + int w_div, h_div; + + int cin_len, cout_len; + int wout_len, hout_len; + + int work_per_spu; +} TileExecPlan; + +void print_convParam(ConvParams* p) { + gtx_printf("\n============= ConvParams ==============\n"); + gtx_printf("input = %d x %d x %d\n", p->c_in, p->h_in, p->w_in); + gtx_printf("knl = %d x %d x %d x %d\n",p->c_out, p->c_in, p->k_size, p->k_size); + gtx_printf("out(h x w) = %d x %d\n", p->h_out, p->w_out); + gtx_printf("dilation = %d\n", p->dilation); + gtx_printf("stride = %d\n", p->stride); + gtx_printf("pad (h, w) = %d, %d\n", p->pad_h, p->pad_w); + gtx_printf("bias = %s\n", p->bias? "true" : "false"); +} + + +void print_l2Info(L2Info* l2i) { + gtx_printf("\n============= l2 Info ==============\n"); + gtx_printf("input_addr: %8x\n", l2i->input_addr); + gtx_printf("kernel_addr: %8x\n", l2i->kernel_addr); + gtx_printf("bias_addr: %8x\n", l2i->bias_addr); + gtx_printf("bias_ones: %8x\n", l2i->bias_ones_addr); + gtx_printf("output_addr: %8x\n", l2i->output_addr); + gtx_printf("total_bytes: %8x\n", l2i->total_bytes); +} + +void print_sputilemap(SPUTileMap* stm){ + gtx_printf("\n==== SPUTileMap ====\n"); + gtx_printf("div: OC x H x W = %d x %d x %d\n", stm->cout_div, stm->h_div, stm->w_div); + gtx_printf("len: OC x H x W = %d x %d x %d\n", stm->cout_len, stm->hout_len, stm->wout_len); + gtx_printf("rem_cnt: OC, H, W = %d, %d, %d\n", stm->cout_rem_cnt, stm->h_rem_cnt, stm->w_rem_cnt); + gtx_printf("use_spu = %d\n", stm->use_spu); + gtx_printf("====================\n"); +} + + +void print_tileinfo(TileExecPlan* t) { + gtx_printf("\nTile Info\n"); + gtx_printf("==== tile Number of each dim ====\n"); + gtx_printf("cin | OC x H x W : %3d | %3d x %3d x %3d \n", t->cin_div, t->cout_div, t->h_div, t->w_div); + gtx_printf("\n==== tile length of each dim ====\n"); + gtx_printf("cin | OC x H x W : %3d | %3d x %3d x %3d \n\n", t->cin_len, t->cout_len, t->hout_len, t->wout_len); +} + +void derive_convParam(ConvParams* p) { + p->win_pad = p->w_in + p->pad_w*2; + p->hin_pad = p->h_in + p->pad_h*2; + p->k_n = p->k_size * p->k_size; + p->k_eff = (p->k_size-1) * p->dilation + 1; + p->h_out = (p->hin_pad - p->k_eff) / p->stride +1; + p->w_out = (p->win_pad - p->k_eff) / p->stride +1; +} + +bool calculate_l2_usage_bias(ConvParams* p, L2Info * l2i, uint32_t l2_offset, uint32_t l2_cap) { + uint64_t input_size = p->c_in * p->hin_pad * p->win_pad *FP16_B; + input_size = align_up(input_size, L2_ALIGNMENT); + uint64_t kernel_size = p->c_out * p->k_n*p->c_in*FP16_B; + uint64_t output_size = p->c_out * p->h_out * p->w_out * FP32_B; + + uint64_t bias_size = align_up(p->c_out*FP16_B, L1_ALIGNMENT); + uint64_t bias_ones_size = L1_64K; + kernel_size = align_up(kernel_size + bias_size, L2_ALIGNMENT); + output_size = align_up(output_size + bias_ones_size, L2_ALIGNMENT); + + uint64_t total_l2_usage = input_size + kernel_size + output_size; + + if(total_l2_usage > l2_cap) return false; + + l2i->input_addr = l2_offset; + l2i->kernel_addr = l2_offset + input_size; + l2i->output_addr = l2_offset + input_size + kernel_size; + l2i->total_bytes = total_l2_usage; + l2i->bias_addr = l2i->output_addr - bias_size; + l2i->bias_ones_addr = l2i->total_bytes - bias_ones_size; + + return true; +} + + +void set_single_conv_l2( + int nest_id, ConvParams* p, + uint32_t l2_offset, uint32_t l2_cap, L2Info* l2i, + uint64_t ddrinput, uint64_t ddr_kernel, uint64_t ddr_bias +) { + uint32_t input_row_bytes = p->w_in * FP16_B; + uint32_t input_cin_bytes = p->h_in * input_row_bytes; + uint32_t knl_read_length = p->c_in*p->k_n*FP16_B; + gtx_printf("\nset_single_conv_l2 start plan\n"); + __start_plan(nest_id); + __start_shared(); + // input load with padding + __pad(ddrinput, l2i->input_addr, p->w_in*FP16_B, p->w_in*FP16_B, p->h_in, + p->pad_h, p->pad_h, p->pad_w, p->pad_w, 0x0, input_cin_bytes, p->c_in, 0b10, 1); + + // kernel load + __load(ddr_kernel, l2i->kernel_addr, knl_read_length, knl_read_length, p->c_out, knl_read_length); + + // bias load and set ones(or zeros) + uint64_t bias_fill_pattern = (p->bias>0)? 0x3C003C003C003C00 : 0x0000; + uint32_t bias_length = p->c_out * FP16_B; + __load(ddr_bias, l2i->bias_addr, bias_length, bias_length, 1, bias_length); + __fill(l2i->bias_ones_addr, 0x1000, 0x0fff, 0x10, bias_fill_pattern, 0); + + // __credit_ld(0xFFFF, 0); + + __end_shared(); + __end_plan(nest_id); + gtx_printf("set_single_conv_l2 end plan\n\n"); +} + + +void store_conv_l2( + int nest_id, ConvParams* p, + uint32_t l2_out_addr, + uint64_t ddr_output){ + + uint32_t length = p->w_out * FP16_B; + uint32_t height = p->c_out * p->h_out; + + __start_plan(nest_id); + __start_shared(); + __store(l2_out_addr, ddr_output, length, length, height, length); + __end_shared(); + __end_plan(nest_id); +} + + +//===================nmconv core===================== + + +void tile_nmconv_general(ConvParams* p, SPUTileMap* stm, TileExecPlan* t, int spu_num) { + + const int HALO_RATIO = 3; + // 1. spu workload division ============================== + const int min_out_tile_len = MAX((p->k_size - p->stride) * HALO_RATIO, MIN_OUT_T_LEN); + const int cout_target = L1_64K / (min_out_tile_len * min_out_tile_len * min_out_tile_len); + + int block_num = 1; + int h_div = 1; + int w_div = 1; + int cout_div = 1; + int hout_len = p->h_out; + int cout_len = p->c_out; + int wout_len = p->w_out; + + while(block_num < spu_num) { + int ohow = hout_len * wout_len; + int cin_kn = p->c_in * p->k_n; + int out_bytes = cout_len * ohow * FP32_B; + int knl_bytes = cout_len * cin_kn * FP16_B; + int im2col_bytes = cin_kn * ohow * FP16_B; + // if(out_bytes <= L1_64K && im2col_bytes <= L1_64K && knl_bytes <= L1_64K) break; + + if(out_bytes > L1_64K || knl_bytes * 2 > im2col_bytes) { + cout_div *= 2; + cout_len = ceil_div(p->c_out, cout_div); + } else if(hout_len > min_out_tile_len) { + h_div *= 2; + hout_len = ceil_div(p->h_out, h_div); + } else if(wout_len > min_out_tile_len) { + w_div *= 2; + wout_len = ceil_div(p->w_out, w_div); + } else break; + block_num *=2; + } + + int cout_rem_cnt = ceil_rem(p->c_out, cout_div); + int h_rem_cnt = ceil_rem(p->h_out, h_div); + int w_rem_cnt = ceil_rem(p->w_out, w_div); + + stm->cout_div = cout_div; + stm->h_div = h_div; + stm->w_div = w_div; + stm->cout_len = cout_len; + stm->hout_len = hout_len; + stm->wout_len = wout_len; + stm->cout_rem_cnt = cout_rem_cnt; + stm->h_rem_cnt = h_rem_cnt; + stm->w_rem_cnt = w_rem_cnt; + stm->use_spu = block_num; + + int spu_idx = 0; + int curr_c = 0; + for (int cd = 0; cd < stm->cout_div; cd++) { + int curr_h = 0; + + int assigned_c = (cd < stm->cout_rem_cnt) ? stm->cout_len : stm->cout_len - 1; + + for (int hd = 0; hd < stm->h_div; hd++) { + int curr_w = 0; + int assigned_h = (hd < stm->h_rem_cnt) ? stm->hout_len : stm->hout_len - 1; + + for (int wd = 0; wd < stm->w_div; wd++) { + int assigned_w = (wd < stm->w_rem_cnt) ? stm->wout_len : stm->wout_len - 1; + + if (spu_idx < spu_num) { + stm->start_cout[spu_idx] = curr_c; + stm->start_hout[spu_idx] = curr_h; + stm->start_wout[spu_idx] = curr_w; + + stm->cout_assigned[spu_idx] = assigned_c; + stm->hout_assigned[spu_idx] = assigned_h; + stm->wout_assigned[spu_idx] = assigned_w; + spu_idx++; + } + curr_w += assigned_w; + } + curr_h += assigned_h; + } + curr_c += assigned_c; + } + + + // 2. Define the tile size for a single SPU workload. + int t_num_cin = 1, t_len_cin = p->c_in; + int t_num_cout = 1, t_len_cout = stm->cout_len; + int t_num_h = 1, t_len_hout = stm->hout_len; + int t_num_w = 1, t_len_wout = stm->wout_len; + + if(t_len_cout > cout_target) { + t_num_cout = ceil_div(t_len_cout, cout_target); + t_len_cout = ceil_div(t_len_cout, t_num_cout); + } + + int knl_bytes = p->k_n * FP16_B; // im2col bytes per pixel + int mmc_bytes_per_pixel = t_len_cout * FP32_B; + int max_img_pixel = L1_128K / MAX(mmc_bytes_per_pixel, knl_bytes); + max_img_pixel = MIN(max_img_pixel, L1_64K / FP16_B / (p->stride*p->stride)); + /* The GTX im2col unit does not support a 1x1 kernel. That path packs A + * with ordinary loads below, one command per output position; keep its + * per-plan command count bounded during bring-up. */ + if (p->k_size == 1) { + max_img_pixel = MIN(max_img_pixel, 64); + } + + // set tile img length + if(max_img_pixel < t_len_hout * t_len_wout) { + int ideal_img_len = (int)isqrt((uint64_t) max_img_pixel); + t_num_h = ceil_div(t_len_hout, ideal_img_len); + t_num_w = ceil_div(t_len_wout, ideal_img_len); + t_len_hout = ceil_div(stm->hout_len, t_num_h); + t_len_wout = ceil_div(stm->wout_len, t_num_w); + + while (t_len_hout * t_len_wout > max_img_pixel) { + if (t_len_hout >= t_len_wout && t_len_hout > 1) t_num_h++; + else t_num_w++; + + t_len_hout = ceil_div(stm->hout_len, t_num_h); + t_len_wout = ceil_div(stm->wout_len, t_num_w); + } + } + + // set cin length + const float in_im2col_ratio = (float)p->k_n / (float)(p->stride * p->stride); + if(in_im2col_ratio >= 2.0f) { + int im2col_bytes_per_cin = t_len_hout * t_len_wout * knl_bytes; + t_len_cin = MIN(L1_128K / im2col_bytes_per_cin, p->c_in); + t_num_cin = ceil_div(p->c_in, t_len_cin); + } else { + int input_bytes_per_cin = t_len_hout * t_len_wout * FP16_B; + t_len_cin = MIN(L1_64K / input_bytes_per_cin, p->c_in); + t_num_cin = ceil_div(p->c_in, t_len_cin); + } + + // set cout length (fix) + int new_t_len_cout = L1_64K / (t_len_cin * knl_bytes); + if(new_t_len_cout < t_len_cout) { + t_len_cout = new_t_len_cout; + t_num_cout = ceil_div(stm->cout_len, t_len_cout); + } + + t->cin_div = t_num_cin; + t->cout_div = t_num_cout; + t->w_div = t_num_w; + t->h_div = t_num_h; + + t->cin_len = t_len_cin; + t->cout_len = t_len_cout; + t->wout_len = t_len_wout; + t->hout_len = t_len_hout; + + t->work_per_spu = t_num_cout * t_num_h * t_num_w; +} + + +void conv_tile_dim4_with_bias(int nest_id, ConvParams* p, SPUTileMap* stm, TileExecPlan* t, L2Info* l2i) { + gtx_printf("\n\n============ conv accumulation with fp32 bias (Refactored) ==============\n"); + + const int use_spu = stm->use_spu; + + const uint32_t input_row_bytes = p->win_pad * FP16_B; + const uint32_t input_cin_bytes = p->hin_pad * input_row_bytes; + const uint32_t output_row_bytes = p->w_out * FP16_B; + const uint32_t output_cout_bytes = p->h_out * output_row_bytes; + + const uint32_t knl_img_bytes = p->k_n * FP16_B; + const uint32_t knl_cout_bytes = p->c_in * knl_img_bytes; + const int len_cin_rem = ceil_rem(p->c_in, t->cin_len); + + for(int ocid = 0; ocid < t->cout_div; ocid++) { + const bool is_c_rem = (ocid == t->cout_div - 1); + const int oc_idx_offset = ocid * t->cout_len; + + for(int hid = 0; hid < t->h_div; hid++) { + const bool is_h_rem = (hid == t->h_div - 1); + const int hout_idx_offset = hid * t->hout_len; + + for(int wid = 0; wid < t->w_div; wid++) { + const bool is_w_rem = (wid == t->w_div - 1); + const int wout_idx_offset = wid * t->wout_len; + + __split(); + __start_plan(nest_id); + for(int spu_id = 0; spu_id < use_spu; spu_id++) { + + const int cur_cout = is_c_rem ? ceil_rem(stm->cout_assigned[spu_id], t->cout_len) : t->cout_len; + const int cur_hout = is_h_rem ? ceil_rem(stm->hout_assigned[spu_id], t->hout_len) : t->hout_len; + const int cur_wout = is_w_rem ? ceil_rem(stm->wout_assigned[spu_id], t->wout_len) : t->wout_len; + + const int cur_hin = FIND_INPUT_LENGTH(cur_hout, p->k_eff, p->stride); + const int cur_win = FIND_INPUT_LENGTH(cur_wout, p->k_eff, p->stride); + + const int cout_idx = stm->start_cout[spu_id] + oc_idx_offset; + const int hout_idx = stm->start_hout[spu_id] + hout_idx_offset; + const int wout_idx = stm->start_wout[spu_id] + wout_idx_offset; + + const uint32_t bias_addr = l2i->bias_addr + (cout_idx * FP16_B); + const uint32_t knl_base = l2i->kernel_addr + (cout_idx * knl_cout_bytes); + const uint32_t out_addr = l2i->output_addr + (cout_idx * output_cout_bytes) + (hout_idx * output_row_bytes) + (wout_idx * FP16_B); + const uint32_t in_tile_off = (hout_idx * p->stride * input_row_bytes) + (wout_idx * p->stride * FP16_B); + + __start_thread(spu_id); + __set_spm_addr(BASE_L1_BANK_R, BASE_L1_BANK_C, BASE_L1_BANK_B, BASE_L1_BANK_A); + + const int bias_len = cur_cout * FP16_B; + const int ones_len = cur_wout * FP16_B; + + __load(l2i->bias_ones_addr, BASE_L1_BANK_A, ones_len, ones_len, cur_hout, ones_len); + __load(bias_addr, BASE_L1_BANK_B, bias_len, bias_len, 1, bias_len); + __mm_s(cur_hout * cur_wout, 1, cur_cout); + + for(int icid = 0; icid < t->cin_div; icid++) { + const bool is_cin_rem = (icid == t->cin_div - 1); + const int cur_cin = is_cin_rem ? len_cin_rem : t->cin_len; + const int ic_idx = icid * t->cin_len; + + const uint32_t in_addr = l2i->input_addr + (ic_idx * input_cin_bytes) + in_tile_off; + const uint32_t knl_ptr = knl_base + (ic_idx * knl_img_bytes); + const uint32_t knl_len = cur_cin * knl_img_bytes; + + if (p->k_size == 1) { + /* Direct 1x1 lowering. L2 input is planar + * [cin][hin][win], while MM consumes A as + * [hout*wout][cin]. Gather one FP16 value + * from every channel for each output point. */ + const uint16_t cin_bytes = + (uint16_t)((uint32_t)cur_cin * FP16_B); + for (int out_y = 0; out_y < cur_hout; ++out_y) { + for (int out_x = 0; out_x < cur_wout; ++out_x) { + const uint32_t out_pixel = + (uint32_t)out_y * (uint32_t)cur_wout + + (uint32_t)out_x; + const uint32_t input_pixel_off = + (uint32_t)out_y * (uint32_t)p->stride * input_row_bytes + + (uint32_t)out_x * (uint32_t)p->stride * FP16_B; + __load(in_addr + input_pixel_off, + BASE_L1_BANK_A + out_pixel * cin_bytes, + input_cin_bytes, + FP16_B, + (uint16_t)cur_cin, + FP16_B); + } + } + } else { + /* Padding is materialized in L2; pack the + * channel planes in L1 before im2col. */ + const uint32_t l1_input_channel_bytes = + (uint32_t)cur_hin * (uint32_t)cur_win * FP16_B; + for (int cin_idx = 0; cin_idx < cur_cin; ++cin_idx) { + __load(in_addr + (uint32_t)cin_idx * input_cin_bytes, + BASE_L1_BANK_R + (uint32_t)cin_idx * l1_input_channel_bytes, + input_row_bytes, + (uint16_t)((uint32_t)cur_win * FP16_B), + (uint16_t)cur_hin, + (uint32_t)cur_win * FP16_B); + } + __im2col_n(cur_hin, cur_win, p->k_size, + p->dilation, p->stride, cur_cin); + } + __load(knl_ptr, BASE_L1_BANK_B, knl_cout_bytes, knl_len, cur_cout, knl_len); + + const int M = cur_hout * cur_wout; + const int K = cur_cin * p->k_n; + + if(!is_cin_rem) { + __mmc_s(M, K, cur_cout); + } else { + __mmc_t(M, K, cur_cout); + /* MMC_T leaves a dense tile in L1 as + * [cout][cur_hout][cur_wout]. L2, however, + * keeps the full [cout][h_out][w_out] planes. + * Store each channel separately so both the + * compact L1 row/channel strides and the full + * L2 row/channel strides are respected. */ + const uint32_t l1_channel_bytes = + (uint32_t)cur_hout * (uint32_t)cur_wout * FP16_B; + for (uint32_t stored_channel = 0; + stored_channel < (uint32_t)cur_cout; + ++stored_channel) { + __store(BASE_L1_BANK_R + + stored_channel * l1_channel_bytes, + out_addr + + stored_channel * output_cout_bytes, + output_row_bytes, + (uint16_t)(cur_wout * FP16_B), + (uint16_t)cur_hout, + (uint32_t)cur_wout * FP16_B); + } + } + } + __end_thread(spu_id); + } + __end_plan(nest_id); + __join(); + } + } + } +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-conv-2d-dw-core.h b/depend/ggml-gtx/src/ggml-gtx-kernel-conv-2d-dw-core.h new file mode 100644 index 0000000..2ddfa28 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-conv-2d-dw-core.h @@ -0,0 +1,892 @@ +#include +#include +#include +#include "gtx/kernel_params.h" +#include "gtx/intrin.h" +#include "gtx/print.h" + + +#define FP16_B 2 +#define FP32_B 4 + +#define L1_CAP 0x60000 +#define L1_64K 0x10000 +#define L1_128K 0x20000 +#define L1_ALIGNMENT 0x100 +#define L2_ALIGNMENT 0x1000 +#define L2_CAP 0x1000000 + +#define ALIGN_B 0x20 + +#define BIAS_ONES_LENGTH 0x100 + +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#define FIND_INPUT_LENGTH(l_out, k_eff, stride) ((l_out - 1) * stride + k_eff) + +static const int MIN_OUT_T_LEN = 8; + +static inline int ceil_div(int a, int b){ return (a + b - 1u) / b; } +static inline int ceil_rem(int a, int b){ return (a + b - 1u) % b + 1; } +static inline uint64_t align_up(uint64_t size, uint64_t align) { + return ((size) + (align) - 1) & ~((align) - 1); +} + +static inline uint64_t isqrt(uint64_t n) { + if (n < 2) return n; + + uint64_t low = 0; + uint64_t high = (n / 2) + 1; + uint64_t res = 0; + + while (low <= high) { + uint64_t mid = low + (high - low) / 2; + if (mid == 0) { + low = mid + 1; + continue; + } + else if (mid <= n / mid) { + low = mid + 1; + res = mid; + } else { + high = mid - 1; + } + } + return res; +} + +//=========================//common//=============================== + +typedef struct { + int cin, hin, win; + int cout; + int k_size; + int pad_h, pad_w; + int stride, dilation; + bool bias; + + int hin_pad, win_pad; + int k_eff, k_n; + int hout, wout; + int win_b; + int cin_b; + int win_pad_b; + int cin_pad_b; + int wout_b; + int cout_b; + int knl_ch_b; + +} ConvParams; + +typedef struct { + uint32_t input_addr; + uint32_t kernel_addr; + uint32_t bias_addr; + uint32_t bias_ones_addr; + uint32_t output_addr; + uint32_t total_bytes; +} L2Info; + +typedef struct { + uint64_t input_addr; + uint64_t kernel_addr; + uint64_t bias_addr; + uint64_t output_addr; +} DDRInfo; + +typedef struct { + // ==== per spu ==== + int cout_div, cout_rem_cnt, cout_len; + int h_div, h_rem_cnt, hout_len; + int w_div, w_rem_cnt, wout_len; + int use_spu; + +}SPUTileMap; + +typedef struct { + int cin_div; + int cout_div; + int w_div, h_div; + + int cin_len, cout_len; + int wout_len, hout_len; + + int work_per_spu; +} TileExecPlan; + + +static void derive_convParam(ConvParams* p) { + p->win_pad = p->win + p->pad_w*FP16_B; + p->hin_pad = p->hin + p->pad_h*FP16_B; + p->k_n = p->k_size * p->k_size; + p->k_eff = (p->k_size-1) * p->dilation + 1; + p->hout = (p->hin_pad - p->k_eff) / p->stride +1; + p->wout = (p->win_pad - p->k_eff) / p->stride +1; + + p->win_b = p->win * FP16_B; + p->cin_b = p->win_b * p->hin; + p->win_pad_b = p->win_pad * FP16_B; + p->cin_pad_b = p->hin_pad * p->win_pad_b; + p->wout_b = p->wout * FP16_B; + p->cout_b = p->hout * p->wout_b; + p->knl_ch_b = p->k_n * FP16_B; +} + +static void store_conv_l2( + int nest_id, ConvParams* p, + uint32_t l2_out_addr, + uint64_t ddr_output){ + + uint32_t length = p->wout * FP16_B; + uint32_t height = p->cout * p->hout; + + __start_plan(nest_id); + __start_shared(); + __store(l2_out_addr, ddr_output, length, length, height, length); + __end_shared(); + __end_plan(nest_id); +} + + +//=====================dwconv only============================== + +static bool calculate_l2_usage_bias32_dwconv(ConvParams* p, L2Info * l2i, uint32_t l2_offset, uint32_t l2_cap) { + uint64_t input_size = p->cin * align_up(p->cin_pad_b, ALIGN_B); + input_size = align_up(input_size, L2_ALIGNMENT); + uint64_t kernel_size = p->cout * align_up(p->knl_ch_b, ALIGN_B); + kernel_size = align_up(kernel_size, L2_ALIGNMENT); + uint64_t output_size = align_up(p->cout_b, L2_ALIGNMENT); + + uint64_t bias_size = align_up(p->cout*FP16_B, L1_ALIGNMENT); + uint64_t bias_ones_size = BIAS_ONES_LENGTH; + + uint64_t total_l2_usage = input_size + kernel_size + bias_size + bias_ones_size + output_size; + + if(total_l2_usage > l2_cap) return false; + + l2i->input_addr = l2_offset; + l2i->kernel_addr = l2_offset + input_size; + l2i->bias_addr = l2i->kernel_addr + kernel_size; + l2i->output_addr = l2i->bias_addr + bias_size; + l2i->bias_ones_addr = l2_cap - bias_ones_size; + l2i->total_bytes = total_l2_usage; + + return true; +} + + + +static void tile_dwconv_workload(ConvParams* p, SPUTileMap* stm, TileExecPlan* t, int spu_num) { + + int c_div = MIN(p->cin, spu_num); + int len_c = ceil_div(p->cin, c_div); + int h_div = 1; + int w_div = 1; + int len_hout = p->hout; + int len_wout = p->wout; + int block_num = c_div; + + const int wh_ratio = 8; + + while(block_num*2 <= spu_num) { + // len_c == 1 + int im2col_bytes = len_hout * len_wout * p->knl_ch_b; + if(im2col_bytes <= L1_64K) break; + + if(len_hout * wh_ratio > len_wout) { + h_div *=2; + len_hout = ceil_div(p->hout, h_div); + }else { + w_div *=2; + len_wout = ceil_div(p->wout, w_div); + } + block_num *=2; + } + + int cout_rem_cnt = ceil_rem(p->cout, c_div); + int h_rem_cnt = ceil_rem(p->hout, h_div); + int w_rem_cnt = ceil_rem(p->wout, w_div); + + stm->cout_div = c_div; + stm->h_div = h_div; + stm->w_div = w_div; + stm->cout_len = len_c; + stm->hout_len = len_hout; + stm->wout_len = len_wout; + stm->cout_rem_cnt = cout_rem_cnt; + stm->h_rem_cnt = h_rem_cnt; + stm->w_rem_cnt = w_rem_cnt; + stm->use_spu = block_num; + + int t_num_cout = 1, t_len_cout = stm->cout_len; + int t_num_h = 1, t_len_hout = stm->hout_len; + int t_num_w = 1, t_len_wout = stm->wout_len; + + + ///////////////////////////// + int ohow = t_len_hout * t_len_wout; + int im2col_bytes = ohow * p->knl_ch_b; + int input_bytes = p->hin_pad * p->win_pad * FP16_B; + int output_bytes = ohow * FP32_B; + + + //k_n < ohow assure + int spu_max_c = L1_128K / MAX(im2col_bytes, output_bytes); + spu_max_c = MIN(spu_max_c, L1_64K / input_bytes); + + if(spu_max_c < 1) { + t_len_cout = 1; + t_num_cout = ceil_div(stm->cout_len, t_len_cout); + int max_img_pixel = L1_128K / MAX(FP32_B, p->knl_ch_b); + max_img_pixel = MIN(max_img_pixel, L1_64K / FP16_B / (p->stride * p->stride)); + int ideal_img_len = (int)isqrt((uint64_t) max_img_pixel); + t_num_h = ceil_div(t_len_hout, ideal_img_len); + t_num_w = ceil_div(t_len_wout, ideal_img_len); + t_len_hout = ceil_div(stm->hout_len, t_num_h); + t_len_wout = ceil_div(stm->wout_len, t_num_w); + + while (t_len_hout * t_len_wout > max_img_pixel) { + if (t_len_hout >= t_len_wout && t_len_hout > 1) t_num_h++; + else t_num_w++; + + t_len_hout = ceil_div(stm->hout_len, t_num_h); + t_len_wout = ceil_div(stm->wout_len, t_num_w); + } + } + else if(spu_max_c < t_len_cout) { + t_len_cout = spu_max_c; + t_num_cout = ceil_div(stm->cout_len, spu_max_c); + } + + t->cin_div = t_num_cout; + t->cout_div = t_num_cout; + t->w_div = t_num_w; + t->h_div = t_num_h; + + t->cin_len = t_len_cout; + t->cout_len = t_len_cout; + t->wout_len = t_len_wout; + t->hout_len = t_len_hout; + + t->work_per_spu = t_num_cout * t_num_h * t_num_w; + +} + +static void dwconv_2d_tile(int nest_id, ConvParams* p, SPUTileMap* stm, TileExecPlan* t, L2Info* l2i, DDRInfo* ddri) +{ + int use_spu = stm->use_spu; + + // 1. Pre-calculate SPU-specific base offsets + // SPU별로 할당된 시작 채널 인덱스를 미리 계산하여 루프 내부 계산 최소화 + int start_cidx[use_spu]; + int cur_iter = 0; + for (int spu_id = 0; spu_id < use_spu; spu_id++) { + start_cidx[spu_id] = cur_iter; + cur_iter += (spu_id < stm->cout_rem_cnt) ? stm->cout_len : stm->cout_len - 1; + } + + { + const uint64_t bias_fill_pattern = (p->bias>0)? 0x3C003C003C003C00 : 0x0000; + + uint32_t ddr_cin_offset = 0; + uint32_t l2_pad_offset = p->pad_h * p->win_pad_b + p->pad_w * FP16_B; + + __start_plan(nest_id); + __start_shared(); + __fill(l2i->input_addr, p->win_pad_b, p->win_pad_b, p->hin_pad * p->cin, 0x00, 0b0); + for(int cid=0; cidcin; cid++) { + __load(ddri->input_addr + ddr_cin_offset, l2i->input_addr + l2_pad_offset, + p->win*FP16_B, p->win*FP16_B, p->hin, p->win_pad_b); + + ddr_cin_offset += p->hin * p->win * FP16_B; + l2_pad_offset += p->cin_pad_b; + } + __load(ddri->kernel_addr, l2i->kernel_addr, p->knl_ch_b, p->knl_ch_b, p->cin, p->knl_ch_b); + __load(ddri->bias_addr, l2i->bias_addr, p->cin*FP16_B, p->cin*FP16_B, 1, p->cin*FP16_B); + __fill(l2i->bias_ones_addr, BIAS_ONES_LENGTH, BIAS_ONES_LENGTH, 1, bias_fill_pattern, 0b0); + __end_shared(); + __end_plan(nest_id); + } + + + /* --- 2D Tiled Workload Case --- */ + for (int cidx = 0; cidx < t->cout_len; cidx++) { + int last_spu = (cidx < stm->cout_len-1)? use_spu : stm->cout_rem_cnt; + + for (int hid = 0; hid < t->h_div; hid++) { + // H 관련 오프셋 계산 (SPU 공통) + int hout_idx = hid * t->hout_len; + int cur_hout = (hid == t->h_div - 1) ? ceil_rem(stm->hout_len, t->hout_len) : t->hout_len; + int cur_hin = FIND_INPUT_LENGTH(cur_hout, p->k_eff, p->stride); + + uint32_t h_off_out = hout_idx * p->wout_b; + uint32_t h_off_in = hout_idx * p->stride * p->win_pad_b; + + for (int wid = 0; wid < t->w_div; wid++) { + // W 관련 오프셋 계산 (SPU 공통) + int wout_idx = wid * t->wout_len; + int cur_wout = (wid == t->w_div - 1) ? ceil_rem(stm->wout_len, t->wout_len) : t->wout_len; + int cur_win = FIND_INPUT_LENGTH(cur_wout, p->k_eff, p->stride); + + uint32_t w_off_out = wout_idx * FP16_B; + uint32_t w_off_in = wout_idx * p->stride * FP16_B; + + int ohow = cur_hout * cur_wout; + uint32_t ones_height = ceil_div(ohow * FP16_B, BIAS_ONES_LENGTH); + __split(); + __start_plan(nest_id); + for (int spu_id = 0; spu_id < last_spu; spu_id++) { + // SPU별 주소를 하드웨어 명령 직전에 계산 (컴파일러가 레지스터에 바로 올리도록 유도) + int abs_c = start_cidx[spu_id] + cidx; + uint32_t in_addr = l2i->input_addr + (abs_c * p->cin_pad_b) + h_off_in + w_off_in; + uint32_t out_addr = l2i->output_addr + (abs_c * p->cout_b) + h_off_out + w_off_out; + uint32_t knl_addr = l2i->kernel_addr + (abs_c * p->knl_ch_b); + uint32_t bias_ptr = l2i->bias_addr + (abs_c * FP16_B); + + __start_thread(spu_id); + __set_spm_addr(BASE_L1_BANK_R, BASE_L1_BANK_C, BASE_L1_BANK_B, BASE_L1_BANK_A); + + // Bias + __load(l2i->bias_ones_addr, BASE_L1_BANK_A, 0, BIAS_ONES_LENGTH, ones_height, BIAS_ONES_LENGTH); + __load(bias_ptr, BASE_L1_BANK_B, FP16_B, FP16_B, 1, FP16_B); + __mm_s(ohow, 1, 1); + + // Conv + __load(in_addr, BASE_L1_BANK_R, p->win_pad_b, cur_win*FP16_B, cur_hin, cur_win*FP16_B); + __im2col_n(cur_hin, cur_win, p->k_size, p->dilation, p->stride, 1); + __load(knl_addr, BASE_L1_BANK_B, p->knl_ch_b, p->knl_ch_b, 1, p->knl_ch_b); + __mmc_t(ohow, p->k_n, 1); + + __store(BASE_L1_BANK_R, out_addr, p->wout_b, cur_wout*FP16_B, cur_hout, cur_wout*FP16_B); + __end_thread(spu_id); + } + __end_plan(nest_id); + __join(); + } + } + } +} + +static void dwconv_2d_spu_tile(int nest_id, ConvParams* p, SPUTileMap* stm, TileExecPlan* t, L2Info* l2i, DDRInfo* ddri) { + const int use_spu = stm->use_spu; + + // 1. SPU별 시작 좌표 및 채널 정보 선계산 + int start_c[use_spu]; + int hout_assigned[use_spu]; + int wout_assigned[use_spu]; + uint32_t in_base_spu[use_spu]; + uint32_t out_base_spu[use_spu]; + + // 공통 바이트 상수 + { + const uint64_t bias_fill_pattern = (p->bias>0)? 0x3C003C003C003C00 : 0x0000; + + uint32_t ddr_cin_offset = 0; + uint32_t l2_pad_offset = p->pad_h * p->win_pad_b + p->pad_w * FP16_B; + + __start_plan(nest_id); + __start_shared(); + __fill(l2i->input_addr, p->win_pad_b, p->win_pad_b, p->hin_pad * p->cin, 0x00, 0b0); + for(int cid=0; cidcin; cid++) { + __load(ddri->input_addr + ddr_cin_offset, l2i->input_addr + l2_pad_offset, + p->win*FP16_B, p->win*FP16_B, p->hin, p->win_pad_b); + + ddr_cin_offset += p->hin * p->win * FP16_B; + l2_pad_offset += p->cin_pad_b; + } + __load(ddri->kernel_addr, l2i->kernel_addr, p->knl_ch_b, p->knl_ch_b, p->cin, p->knl_ch_b); + __load(ddri->bias_addr, l2i->bias_addr, p->cin*FP16_B, p->cin*FP16_B, 1, p->cin*FP16_B); + __fill(l2i->bias_ones_addr, BIAS_ONES_LENGTH, BIAS_ONES_LENGTH, 1, bias_fill_pattern, 0b0); + __end_shared(); + __end_plan(nest_id); + } + + { + int spu_id = 0; + for (int cid = 0; cid < stm->cout_div; cid++) { + int start_hout = 0; + for (int hid = 0; hid < stm->h_div; hid++) { + int start_wout = 0; + int cur_h = (hid < stm->h_rem_cnt) ? stm->hout_len : stm->hout_len - 1; + for (int wid = 0; wid < stm->w_div; wid++) { + int cur_w = (wid < stm->w_rem_cnt) ? stm->wout_len : stm->wout_len - 1; + + start_c[spu_id] = cid; + hout_assigned[spu_id] = cur_h; + wout_assigned[spu_id] = cur_w; + + // L2 내 SPU별 시작 주소 확정 + in_base_spu[spu_id] = l2i->input_addr + (cid * p->cin_pad_b) + + (start_hout * p->stride * p->win_pad_b) + (start_wout * p->stride * FP16_B); + out_base_spu[spu_id] = l2i->output_addr + (cid * p->cout_b) + + (start_hout * p->wout_b) + (start_wout * FP16_B); + + start_wout += cur_w; + spu_id++; + } + start_hout += cur_h; + } + } + } + + /* --- 2D Tiled Workload Logic --- */ + for (int hid = 0; hid < t->h_div; hid++) { + const bool is_h_rem = (hid == t->h_div - 1); + const int hout_idx = hid * t->hout_len; + const uint32_t hout_off = hout_idx * p->wout_b; + const uint32_t hin_off = hout_idx * p->stride * p->win_pad_b; + + for (int wid = 0; wid < t->w_div; wid++) { + const bool is_w_rem = (wid == t->w_div - 1); + const int wout_idx = wid * t->wout_len; + const uint32_t wout_off = wout_idx * FP16_B; + const uint32_t win_off = wout_idx * p->stride * FP16_B; + __split(); + __start_plan(nest_id); + for (int spu_id = 0; spu_id < use_spu; spu_id++) { + // 현재 타일의 가변 길이 계산 + int cur_hout = is_h_rem ? ceil_rem(hout_assigned[spu_id], t->hout_len) : t->hout_len; + int cur_wout = is_w_rem ? ceil_rem(wout_assigned[spu_id], t->wout_len) : t->wout_len; + int cur_hin = FIND_INPUT_LENGTH(cur_hout, p->k_eff, p->stride); + int cur_win = FIND_INPUT_LENGTH(cur_wout, p->k_eff, p->stride); + int ohow = cur_hout * cur_wout; + + // SPU별 L2 주소 계산 + uint32_t in_addr = in_base_spu[spu_id] + hin_off + win_off; + uint32_t out_addr = out_base_spu[spu_id] + hout_off + wout_off; + uint32_t knl_addr = l2i->kernel_addr + (start_c[spu_id] * p->knl_ch_b); + uint32_t bias_ptr = l2i->bias_addr + (start_c[spu_id] * FP16_B); + uint32_t ones_height = ceil_div(ohow * FP16_B, BIAS_ONES_LENGTH); + + __start_thread(spu_id); + __set_spm_addr(BASE_L1_BANK_R, BASE_L1_BANK_C, BASE_L1_BANK_B, BASE_L1_BANK_A); + + // 1. Bias 처리 (BANK_A: Ones, BANK_B: Bias Value) + __load(l2i->bias_ones_addr, BASE_L1_BANK_A, 0, BIAS_ONES_LENGTH, ones_height, BIAS_ONES_LENGTH); + __load(bias_ptr, BASE_L1_BANK_B, FP16_B, FP16_B, 1, FP16_B); + __mm_s(ohow, 1, 1); + + // 2. Input Load & Im2col (BANK_R: Input, BANK_A: Im2col Result) + __load(in_addr, BASE_L1_BANK_R, p->win_pad_b, cur_win * FP16_B, cur_hin, cur_win * FP16_B); + + // [중요] im2col 결과 주소를 32-byte aligned 된 BANK_A로 명시적 설정 + __set_spm_addr(BASE_L1_BANK_R, BASE_L1_BANK_C, BASE_L1_BANK_B, BASE_L1_BANK_A); + __im2col_n(cur_hin, cur_win, p->k_size, p->dilation, p->stride, 1); + + // 3. Kernel Load (BANK_B) + __load(knl_addr, BASE_L1_BANK_B, p->knl_ch_b, p->knl_ch_b, 1, p->knl_ch_b); + + // 4. MXE Convolution 연산 + __mmc_t(ohow, p->k_n, 1); + + // 5. 결과 저장 (BANK_R -> L2) + __store(BASE_L1_BANK_R, out_addr, p->wout_b, cur_wout * FP16_B, cur_hout, cur_wout * FP16_B); + __end_thread(spu_id); + } + __end_plan(nest_id); + __join(); + } + } +} + + +void print_convParams(ConvParams* p) { + gtx_printf("\n[ ConvParams ]\n"); + gtx_printf(" - Input (C x H x W) : %d x %d x %d\n", p->cin, p->hin_pad, p->win_pad); + gtx_printf(" - Kernel (OC x IC x KH x KW) : %d x %d x %d x %d\n", p->cout, p->cin, p->k_size, p->k_size); + // gtx_printf(" - Effective (Kn, Keff) : %d, %d\n", p->k_n, p->k_eff); + gtx_printf(" - Stride / Dilation : %d / %d\n", p->stride, p->dilation); + gtx_printf(" - Output (H x W) : %d x %d\n", p->hout, p->wout); + gtx_printf(" - Bias / Activation : %s \n", p->bias? "true":"false"); + gtx_printf("--------------------------------------\n"); +} + + +void print_l2Info(L2Info* l2i, const char* name) { + gtx_printf("\n[ L2 Info: %s ]\n", name); + gtx_printf(" Input : 0x%08x\n", l2i->input_addr); + gtx_printf(" Kernel : 0x%08x\n", l2i->kernel_addr); + gtx_printf(" Bias : 0x%08x\n", l2i->bias_addr); + gtx_printf(" Ones : 0x%08x\n", l2i->bias_ones_addr); + gtx_printf(" Output : 0x%08x\n", l2i->output_addr); + gtx_printf(" Total : %d Bytes\n", l2i->total_bytes); + gtx_printf("--------------------------------------\n"); +} + + +void print_sputilemap(SPUTileMap* stm) { + gtx_printf("\n[ SPUTileMap ]\n"); + gtx_printf(" - SPU Div (C x H x W) : %d x %d x %d (Total SPU: %d)\n", + stm->cout_div, stm->h_div, stm->w_div, stm->use_spu); + gtx_printf(" - Base Len (C, H, W) : %d, %d, %d\n", stm->cout_len, stm->hout_len, stm->wout_len); + gtx_printf(" - Rem Cnt (C, H, W) : %d, %d, %d\n", stm->cout_rem_cnt, stm->h_rem_cnt, stm->w_rem_cnt); + gtx_printf("--------------------\n"); +} + +void print_tileinfo(TileExecPlan* t, const char* name) { + gtx_printf("\n[ Tile Info: %s ]\n", name); + gtx_printf(" - Tile Div (Cin, Cout, H, W) : %d, %d, %d, %d\n", + t->cin_div, t->cout_div, t->h_div, t->w_div); + gtx_printf(" - Tile Len (Cin, Cout, H, W) : %d, %d, %d, %d\n", + t->cin_len, t->cout_len, t->hout_len, t->wout_len); + gtx_printf(" - Work per SPU : %d\n", t->work_per_spu); + gtx_printf("--------------------------------------\n"); +} + + + + +// static void dwconv_ch_tile(int nest_id, ConvParams* p, SPUTileMap* stm, TileExecPlan* t, L2Info* l2i, DDRInfo* ddri) { +// const int use_spu = stm->use_spu; + +// int start_cidx[use_spu]; +// int cur_iter = 0; +// for (int spu_id = 0; spu_id < use_spu; spu_id++) { +// start_cidx[spu_id] = cur_iter; +// cur_iter += (spu_id <= stm->cout_rem_cnt) ? stm->cout_len : stm->cout_len - 1; +// } + +// { +// const uint64_t bias_fill_pattern = (p->bias>0)? 0x3C003C003C003C00 : 0x0000; +// uint32_t l2_cin_offset = 0; +// uint32_t ddr_cin_offset = 0; +// uint32_t l2_pad_offset = p->pad_h * p->win_pad_b + p->pad_w * FP16_B; + +// gtx_printf("\nload ddr to l2\n"); +// __start_plan(nest_id); +// __start_shared(); + +// for(int cid=0; cidcin; cid++) { +// __fill(l2i->input_addr + l2_cin_offset, p->win_pad_b, p->win_pad_b, p->hin_pad, 0x00, 0b0); +// __load(ddri->input_addr + ddr_cin_offset, l2i->input_addr + l2_cin_offset + l2_pad_offset, +// p->win*FP16_B, p->win*FP16_B, p->hin, p->win_pad_b); + +// ddr_cin_offset += p->hin*p->win*FP16_B; +// l2_cin_offset += p->cin_pad_b; +// } +// __load(ddri->kernel_addr, l2i->kernel_addr, p->knl_ch_b, p->knl_ch_b, p->cin, p->knl_ch_b); +// __load(ddri->bias_addr, l2i->bias_addr, p->cin*FP16_B, p->cin*FP16_B, 1, p->cin*FP16_B); +// __fill(l2i->bias_addr, BIAS_ONES_LENGTH, BIAS_ONES_LENGTH, 1, bias_fill_pattern, 0b0); +// __end_shared(); +// __end_plan(nest_id); +// } + +// /* --- Channel Chunk Workload Case --- */ +// int ohow = t->hout_len * t->wout_len; +// int ohow_aligned = align_up(ohow, ALIGN_B / FP32_B); +// uint32_t ones_height = ceil_div(ohow_aligned * FP16_B, BIAS_ONES_LENGTH); + +// for (int cid = 0; cid < t->cout_div; cid++) { +// const int is_c_rem = (cid == t->cout_div - 1); +// const int cidx_base = cid * t->cout_len; +// gtx_printf("ch tile id: %d\n", cid); +// __start_plan(nest_id); +// for (int spu_id = 0; spu_id < use_spu; spu_id++) { +// // SPU별 처리량 및 주소 확정 +// int ch_assigned = (spu_id <= stm->cout_rem_cnt) ? stm->cout_len : stm->cout_len - 1; +// int cur_c_len = is_c_rem ? ceil_rem(ch_assigned, t->cout_len) : t->cout_len; +// int abs_c_start = start_cidx[spu_id] + cidx_base; + + +// __start_thread(spu_id); +// __set_spm_addr(BASE_L1_BANK_R, BASE_L1_BANK_C, BASE_L1_BANK_B, BASE_L1_BANK_A); + +// // Batch Load (Input, Bias, Kernel) +// __load(l2i->bias_ones_addr, BASE_L1_BANK_B, 0, BIAS_ONES_LENGTH, ones_height, BIAS_ONES_LENGTH); +// __load(l2i->bias_addr + abs_c_start*FP16_B, BASE_L1_BANK_A, cur_c_len*FP16_B, cur_c_len*FP16_B, 1, cur_c_len*FP16_B); +// __mm_s(cur_c_len, 1, ohow_aligned); + +// // 연산 집약 구간 (Inner Loop) +// for (int co = 0; co < cur_c_len; co++) { +// __load(l2i->input_addr + abs_c_start * p->cin_pad_b, BASE_L1_BANK_R, p->win_pad_b, p->win_pad_b, p->hin_pad, p->win_pad_b); +// __im2col_d(p->hin_pad, p->win_pad, p->k_size, p->dilation, p->stride, 1); + +// // __set_spm_addr(BASE_L1_BANK_R + co * p->cout_b, +// // BASE_L1_BANK_C + co * p->cout_b * 2, +// // BASE_L1_BANK_B + co * p->knl_ch_b, +// // BASE_L1_BANK_A + co * im2col_c_b_aligned); + +// __mmc_t(ohow, p->k_n, 1); +// } + +// __store(BASE_L1_BANK_R, l2i->output_addr + abs_c_start * p->cout_b, p->wout_b, p->wout_b, p->hout * cur_c_len, p->wout_b); +// __end_thread(spu_id); +// } +// __end_plan(nest_id); +// } +// } + + + +// //stm->cout_len >= 1 +// static void dwconv_tile_ch(int nest_id, ConvParams* p, SPUTileMap* stm, TileExecPlan* t, L2Info* l2i) { +// const int use_spu = stm->use_spu; + +// int c_assigned[use_spu]; +// int start_cidx[use_spu]; + +// { +// int spu_id = 0; +// int start_c_iter = 0; + +// for(int cid=0; cidcout_div; cid ++) { +// c_assigned[spu_id] = cid <= stm->cout_rem_cnt? stm->cout_len : stm->cout_len -1; +// start_cidx[spu_id] = start_c_iter; + +// start_c_iter += c_assigned[spu_id]; +// spu_id++; +// } +// } + +// uint32_t input_row_bytes = p->win_pad * FP16_B; +// uint32_t input_cin_bytes = p->hin_pad * input_row_bytes; +// uint32_t output_row_bytes = p->wout * FP16_B; +// uint32_t output_cout_bytes = p->hout * output_row_bytes; +// uint32_t knl_ch_bytes = p->k_n * FP16_B; + +// int t_hout_len_rem = ceil_rem(stm->hout_len, t->hout_len); +// int t_hin_len = FIND_INPUT_LENGTH(t->hout_len, p->k_eff, p->stride); +// int t_hin_len_rem = FIND_INPUT_LENGTH(t_hout_len_rem, p->k_eff, p->stride); + +// int t_wout_len_rem = ceil_rem(stm->wout_len, t->wout_len); +// int t_win_len = FIND_INPUT_LENGTH(t->wout_len, p->k_eff, p->stride); +// int t_win_len_rem = FIND_INPUT_LENGTH(t_wout_len_rem, p->k_eff, p->stride); + + +// uint32_t t_win_bytes = t_win_len * FP16_B; +// uint32_t t_wout_bytes = t->wout_len * FP16_B; + +// if(t->h_div * t->w_div > 1) {//2D tiled workload +// gtx_printf("2D tiled workload \n"); +// for(int cidx=0; cidxcout_div; cidx++) { +// // gtx_printf("cidx: %d\n", cidx); +// uint32_t knl_base[use_spu]; +// uint32_t input_base[use_spu]; +// uint32_t output_base[use_spu]; +// uint32_t bias_base[use_spu]; +// for(int spu_id=0; spu_idkernel_addr + knl_ch_bytes * (start_cidx[spu_id] + cidx); +// input_base[spu_id] = l2i->input_addr + input_cin_bytes * (start_cidx[spu_id] + cidx); +// output_base[spu_id] = l2i->output_addr + output_cout_bytes * (start_cidx[spu_id] + cidx); +// bias_base[spu_id] = l2i->bias_addr + FP16_B * (start_cidx[spu_id] + cidx); +// } + +// for(int hid=0; hidh_div; hid++) { +// int hout_idx = hid * t->hout_len; +// int hout_offset = hout_idx * output_row_bytes; +// int hin_offset = hout_idx * p->stride * input_row_bytes; +// bool is_h_rem = (hid == t->h_div -1); +// int cur_len_hout = is_h_rem? t_hout_len_rem : t->hout_len; +// int cur_len_hin = is_h_rem? t_hin_len_rem : t_hin_len; + +// for(int wid=0; widw_div; wid++) { +// // gtx_printf("hid, wid : %d, %d\n", hid, wid); +// int wout_idx = wid * t->wout_len; +// int wout_offset = wout_idx * FP16_B; +// int win_offset = wout_idx * p->stride * FP16_B; +// bool is_w_rem = (wid == t->w_div -1); +// int cur_len_wout = is_w_rem? t_wout_len_rem : t->wout_len; +// int cur_len_win = is_w_rem? t_win_len_rem : t_win_len; +// uint32_t cur_length_in = cur_len_win * FP16_B; +// uint32_t cur_length_out = cur_len_wout * FP16_B; +// int bias_ones_len = cur_len_wout * FP16_B; +// int ohow = cur_len_hout * cur_len_wout; + +// // __split(); +// __start_plan(nest_id); +// for(int spu_id=0; spu_idbias_ones_addr, BASE_L1_BANK_A, +// bias_ones_len, bias_ones_len, cur_len_hout, bias_ones_len); +// __load(bias_base[spu_id], BASE_L1_BANK_B, FP16_B, FP16_B, 1, FP16_B); +// __mm_s(cur_len_hout * cur_len_wout, 1, 1); +// // accumulate once (dwconv) +// __load(input_addr , BASE_L1_BANK_R, +// input_row_bytes, cur_length_in, cur_len_hin, cur_length_in); +// __im2col_n(cur_len_hin, cur_len_win, p->k_size, p->dilation, p->stride, 1); +// __load(knl_base[spu_id], BASE_L1_BANK_B, +// knl_ch_bytes, knl_ch_bytes, 1, knl_ch_bytes); +// __mmc_t(ohow, p->k_n, 1); +// __store(BASE_L1_BANK_R, output_addr, output_row_bytes, cur_length_out, cur_len_hout, cur_length_out); +// __end_thread(spu_id); +// } +// __end_plan(nest_id); +// // __join(); +// } +// } +// } +// } + +// else {// channel chunk workload +// gtx_printf("channel chunk workload\n"); +// int ohow = t->hout_len * t->wout_len; +// int bias_ones_len = t->wout_len * FP16_B; +// uint32_t im2col_c_bytes = output_cout_bytes * p->k_n; + +// // gtx_printf("cid: "); +// for(int cid=0; cidcout_div; cid++) { +// // gtx_printf("%d, ",cid); +// int is_c_rem = (cid == t->cout_div - 1); +// int cidx = cid * t->cout_len; + +// uint32_t knl_base[use_spu]; +// uint32_t input_base[use_spu]; +// uint32_t output_base[use_spu]; +// uint32_t bias_base[use_spu]; +// int cur_c_len[use_spu]; +// for(int spu_id=0; spu_idkernel_addr + knl_ch_bytes * (start_cidx[spu_id] + cidx); +// input_base[spu_id] = l2i->input_addr + input_cin_bytes * (start_cidx[spu_id] + cidx); +// output_base[spu_id] = l2i->output_addr + output_cout_bytes * (start_cidx[spu_id] + cidx); +// bias_base[spu_id] = l2i->bias_addr + FP16_B * (start_cidx[spu_id] + cidx); +// cur_c_len[spu_id] = is_c_rem? ceil_rem(c_assigned[spu_id], t->cout_len) : t->cout_len; +// } + +// // bias fp32 setting : first GEMM +// // __split(); +// __start_plan(nest_id); +// for(int spu_id=0; spu_idhin_pad * cur_c_len[spu_id]; +// uint32_t out_height = p->hout * cur_c_len[spu_id]; +// __start_thread(spu_id); + +// __set_spm_addr(BASE_L1_BANK_R, BASE_L1_BANK_C, BASE_L1_BANK_B, BASE_L1_BANK_A); +// // bias ones +// __load(l2i->bias_ones_addr, BASE_L1_BANK_B, +// bias_ones_len, bias_ones_len, p->hout, bias_ones_len); +// __load(bias_base[spu_id], BASE_L1_BANK_A, bias_length, bias_length, 1, bias_length); +// __mm_s(cur_c_len[spu_id], 1, ohow); +// __load(input_base[spu_id], BASE_L1_BANK_R, +// input_row_bytes, input_row_bytes, input_height, input_row_bytes); +// __im2col_d(p->hin_pad, p->win_pad, p->k_size, p->dilation, p->stride, cur_c_len[spu_id]); + +// __load(knl_base[spu_id], BASE_L1_BANK_B, +// knl_ch_bytes, knl_ch_bytes, cur_c_len[spu_id], knl_ch_bytes); + +// for(int c_off=0; c_offk_n, 1); +// } + + +// __store(BASE_L1_BANK_R, output_base[spu_id], output_row_bytes, output_row_bytes, out_height, output_row_bytes); + +// __end_thread(spu_id); +// } +// __end_plan(nest_id); +// // __join(); +// } +// } +// } + + +// // stm->h_div * stm->w_div > 1 +// static void dwconv_tile_img(int nest_id, ConvParams* p, SPUTileMap* stm, TileExecPlan* t, L2Info* l2i) { +// int use_spu = stm->use_spu; + +// uint32_t input_row_bytes = p->win_pad * FP16_B; +// uint32_t input_cin_bytes = p->hin_pad * input_row_bytes; +// uint32_t output_row_bytes = p->wout * FP16_B; +// uint32_t output_cout_bytes = p->hout * output_row_bytes; +// uint32_t knl_ch_bytes = p->k_n * FP16_B; +// int bias_length = FP16_B; + +// int hout_assigned[use_spu]; +// int wout_assigned[use_spu]; +// int start_c[use_spu]; +// uint32_t input_base[use_spu]; +// uint32_t knl_base[use_spu]; +// uint32_t out_base[use_spu]; + +// { +// int spu_id = 0; + +// for(int cid=0; cidcout_div; cid++) { +// int start_hout=0; + +// for(int hid=0; hidh_div; hid++) { +// int start_wout=0; +// int cur_len_hout = (hid < stm->h_rem_cnt)? stm->hout_len : stm->hout_len-1; + +// for(int wid=0; widw_div; wid++) { +// int cur_len_wout = wid < stm->w_rem_cnt? stm->wout_len : stm->wout_len-1; +// start_c[spu_id] = cid; +// int start_hin = start_hout * p->stride; +// int start_win = start_wout * p->stride; + +// hout_assigned[spu_id] = cur_len_hout; +// wout_assigned[spu_id] = cur_len_wout; + +// input_base[spu_id] = l2i->input_addr + cid * input_cin_bytes + start_hin * input_row_bytes + start_win * FP16_B; +// knl_base[spu_id] = l2i->kernel_addr + cid * knl_ch_bytes; +// out_base[spu_id] = l2i->output_addr + cid * output_cout_bytes + start_hout * output_row_bytes + start_wout * FP16_B; + +// start_wout += cur_len_wout; +// spu_id++; +// } +// start_hout += cur_len_hout; +// } +// } +// } + +// // tile iteration under SPU work +// for(int hid=0; hidh_div; hid++){ +// bool is_h_rem = (hid == t->h_div-1); +// int hout_idx = hid * t->hout_len; +// uint32_t hout_offset = hout_idx * output_row_bytes; +// uint32_t hin_offset = hout_idx * p->stride * input_row_bytes; +// int cur_hout_len[use_spu]; +// int cur_hin_len[use_spu]; +// for(int spu_id=0; spu_idhout_len) : t->hout_len; +// cur_hin_len[spu_id] = FIND_INPUT_LENGTH(cur_hout_len[spu_id], p->k_eff, p->stride); +// } +// for(int wid=0; widw_div; wid++){ +// gtx_printf("\nhid, wid: %d, %d ||", hid, wid); +// bool is_w_rem = (wid == t->w_div-1); +// int wout_idx = wid * t->wout_len; +// uint32_t wout_offset = wout_idx * FP16_B; +// uint32_t win_offset = wout_idx * p->stride * FP16_B; +// gtx_printf("- bias - "); +// // bias fp32 setting : first GEMM +// // __split(); +// __start_plan(nest_id); +// for(int spu_id=0; spu_idwout_len) : t->wout_len; +// int cur_win_len = FIND_INPUT_LENGTH(cur_wout_len, p->k_eff, p->stride); +// int ohow = cur_wout_len * cur_hout_len[spu_id]; + +// uint32_t bias_addr = l2i->bias_addr + start_c[spu_id] * FP16_B; +// int ones_length = cur_wout_len * FP16_B; +// uint32_t input_addr = input_base[spu_id] + hin_offset + win_offset; +// uint32_t output_addr = out_base[spu_id] + hout_offset + wout_offset; +// // gtx_printf("mm_S : M, K, N : %d, %d, %d \n", cur_hout[spu_id]*cur_wout[spu_id], 1, cur_cout[spu_id]); +// __start_thread(spu_id); +// __set_spm_addr(BASE_L1_BANK_R, BASE_L1_BANK_C, BASE_L1_BANK_B, BASE_L1_BANK_A); +// __load(l2i->bias_ones_addr, BASE_L1_BANK_A, ones_length, ones_length, cur_hout_len[spu_id], ones_length); +// __load(bias_addr, BASE_L1_BANK_B, bias_length, bias_length, 1, bias_length); +// __mm_s(cur_hout_len[spu_id]*cur_wout_len, 1, 1); + +// // accumulate once (dwconv) +// __load(input_addr , BASE_L1_BANK_R, +// input_row_bytes, cur_win_len*FP16_B, cur_hin_len[spu_id], cur_win_len*FP16_B); +// __im2col_n(cur_hin_len[spu_id], cur_win_len, p->k_size, p->dilation, p->stride, 1); +// __load(knl_base[spu_id], BASE_L1_BANK_B, +// knl_ch_bytes, knl_ch_bytes, 1, knl_ch_bytes); +// __mmc_t(ohow, p->k_n, 1); +// __store(BASE_L1_BANK_R, output_addr, output_row_bytes, cur_wout_len*FP16_B, cur_hout_len[spu_id], cur_wout_len*FP16_B); +// __end_thread(spu_id); +// } +// __end_plan(nest_id); +// // __join(); +// } +// } +// } diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-conv-2d-dw.c b/depend/ggml-gtx/src/ggml-gtx-kernel-conv-2d-dw.c new file mode 100644 index 0000000..7f38eac --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-conv-2d-dw.c @@ -0,0 +1,525 @@ +//================================================================== +// GGML GTX kernel: CONV_2D_DW +// +// Descriptor wrapper around the host-owned GTX depthwise-convolution core. +//================================================================== + +#include + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#include "ggml-gtx-kernel-conv-2d-dw-core.h" + +static void ggml_gtx_dwconv_rebase_scratch(L2Info * l2i, uint32_t scratch_base) { + l2i->input_addr += scratch_base; + l2i->kernel_addr += scratch_base; + l2i->bias_addr += scratch_base; + l2i->output_addr += scratch_base; + l2i->bias_ones_addr += scratch_base; +} + +static bool ggml_gtx_dwconv_l2_slice_fits(const ConvParams * p, const L2Info * l2i) { + const uint64_t output_bytes = (uint64_t) p->cout * (uint64_t) p->cout_b; + return (uint64_t) l2i->output_addr + output_bytes <= (uint64_t) l2i->bias_ones_addr; +} + +static bool ggml_gtx_dwconv_planar_layout(const ggml_gtx_kernel_launch_desc_t * desc, + const ConvParams * p) { + const uint32_t input_row_bytes = (uint32_t) p->win * FP16_B; + const uint32_t input_ch_bytes = (uint32_t) p->hin * input_row_bytes; + const uint32_t kernel_row_bytes = (uint32_t) p->k_size * FP16_B; + const uint32_t kernel_ch_bytes = (uint32_t) p->k_size * kernel_row_bytes; + const uint32_t output_row_bytes = (uint32_t) p->wout * FP16_B; + const uint32_t output_ch_bytes = (uint32_t) p->hout * output_row_bytes; + + return desc->src_nb[1][0] == FP16_B && + desc->src_nb[1][1] == input_row_bytes && + desc->src_nb[1][2] == input_ch_bytes && + desc->src_nb[0][0] == FP16_B && + desc->src_nb[0][1] == kernel_row_bytes && + desc->src_nb[0][3] == kernel_ch_bytes && + desc->nb[0] == FP16_B && + desc->nb[1] == output_row_bytes && + desc->nb[2] == output_ch_bytes; +} + +static void ggml_gtx_dwconv_copy_strided_elems(uint32_t src, + uint32_t dst, + uint32_t src_stride, + uint32_t dst_stride, + uint32_t elems) { + while (elems > 0u) { + const uint16_t chunk = (uint16_t) (elems > UINT16_MAX ? UINT16_MAX : elems); + __copy(src, dst, src_stride, FP16_B, chunk, dst_stride); + src += (uint32_t) chunk * src_stride; + dst += (uint32_t) chunk * dst_stride; + elems -= chunk; + } +} + +static void ggml_gtx_dwconv_load_l2_strided( + int nest_id, + const ConvParams * p, + const L2Info * l2i, + uint64_t input_base, + uint64_t kernel_base, + uint64_t input_x_stride, + uint64_t input_y_stride, + uint64_t input_c_stride, + uint64_t kernel_x_stride, + uint64_t kernel_y_stride, + uint64_t kernel_c_stride) { + + const uint32_t l2_pad_offset = (uint32_t) p->pad_h * (uint32_t) p->win_pad_b + + (uint32_t) p->pad_w * FP16_B; + const uint64_t bias_fill_pattern = p->bias ? 0x3C003C003C003C00ull : 0x0000ull; + __start_plan(nest_id); + __start_shared(); + __fill(l2i->input_addr, + (uint32_t) p->win_pad_b, + (uint16_t) p->win_pad_b, + (uint16_t) ((uint32_t) p->hin_pad * (uint32_t) p->cin), + 0x00, + 0); + + for (int cid = 0; cid < p->cin; ++cid) { + const uint64_t input_ch = input_base + (uint64_t) cid * input_c_stride; + const uint32_t l2_ch = l2i->input_addr + (uint32_t) cid * (uint32_t) p->cin_pad_b; + if (p->pad_w == 0 && p->win_pad == p->win && + input_y_stride == (uint64_t) p->win * input_x_stride) { + ggml_gtx_dwconv_copy_strided_elems((uint32_t) input_ch, + l2_ch + l2_pad_offset, + (uint32_t) input_x_stride, + FP16_B, + (uint32_t) p->hin * (uint32_t) p->win); + } else { + for (int y = 0; y < p->hin; ++y) { + ggml_gtx_dwconv_copy_strided_elems( + (uint32_t) input_ch + (uint32_t) y * (uint32_t) input_y_stride, + l2_ch + l2_pad_offset + (uint32_t) y * (uint32_t) p->win_pad_b, + (uint32_t) input_x_stride, + FP16_B, + (uint32_t) p->win); + } + } + + const uint64_t kernel_ch = kernel_base + (uint64_t) cid * kernel_c_stride; + const uint32_t l2_kernel_ch = l2i->kernel_addr + (uint32_t) cid * (uint32_t) p->knl_ch_b; + if (kernel_y_stride == (uint64_t) p->k_size * kernel_x_stride) { + ggml_gtx_dwconv_copy_strided_elems((uint32_t) kernel_ch, + l2_kernel_ch, + (uint32_t) kernel_x_stride, + FP16_B, + (uint32_t) p->k_size * (uint32_t) p->k_size); + } else { + for (int ky = 0; ky < p->k_size; ++ky) { + ggml_gtx_dwconv_copy_strided_elems( + (uint32_t) kernel_ch + (uint32_t) ky * (uint32_t) kernel_y_stride, + l2_kernel_ch + (uint32_t) ky * (uint32_t) p->k_size * FP16_B, + (uint32_t) kernel_x_stride, + FP16_B, + (uint32_t) p->k_size); + } + } + } + + __fill(l2i->bias_addr, + (uint32_t) p->cin * FP16_B, + (uint16_t) ((uint32_t) p->cin * FP16_B), + 1, + 0x00, + 0); + __fill(l2i->bias_ones_addr, + BIAS_ONES_LENGTH, + BIAS_ONES_LENGTH, + 1, + bias_fill_pattern, + 0); + __end_shared(); + __end_plan(nest_id); +} + +static void ggml_gtx_dwconv_store_l2_strided( + int nest_id, + const ConvParams * p, + uint32_t l2_out_addr, + uint64_t ddr_output, + uint64_t output_x_stride, + uint64_t output_y_stride, + uint64_t output_c_stride) { + + __start_plan(nest_id); + __start_shared(); + for (int cid = 0; cid < p->cout; ++cid) { + const uint32_t l2_ch = l2_out_addr + (uint32_t) cid * (uint32_t) p->cout_b; + const uint64_t ddr_ch = ddr_output + (uint64_t) cid * output_c_stride; + if (output_y_stride == (uint64_t) p->wout * output_x_stride) { + ggml_gtx_dwconv_copy_strided_elems(l2_ch, + (uint32_t) ddr_ch, + FP16_B, + (uint32_t) output_x_stride, + (uint32_t) p->hout * (uint32_t) p->wout); + } else { + for (int y = 0; y < p->hout; ++y) { + ggml_gtx_dwconv_copy_strided_elems( + l2_ch + (uint32_t) y * (uint32_t) p->wout_b, + (uint32_t) ddr_ch + (uint32_t) y * (uint32_t) output_y_stride, + FP16_B, + (uint32_t) output_x_stride, + (uint32_t) p->wout); + } + } + } + __end_shared(); + __end_plan(nest_id); +} + +static void ggml_gtx_dwconv_2d_tile_compute_only( + int nest_id, + ConvParams * p, + SPUTileMap * stm, + TileExecPlan * t, + L2Info * l2i) { + + int use_spu = stm->use_spu; + + int start_cidx[use_spu]; + int cur_iter = 0; + for (int spu_id = 0; spu_id < use_spu; spu_id++) { + start_cidx[spu_id] = cur_iter; + cur_iter += (spu_id < stm->cout_rem_cnt) ? stm->cout_len : stm->cout_len - 1; + } + + for (int cidx = 0; cidx < stm->cout_len; cidx++) { + int last_spu = (cidx < stm->cout_len - 1) ? use_spu : stm->cout_rem_cnt; + + for (int hid = 0; hid < t->h_div; hid++) { + int hout_idx = hid * t->hout_len; + int cur_hout = (hid == t->h_div - 1) ? ceil_rem(stm->hout_len, t->hout_len) : t->hout_len; + int cur_hin = FIND_INPUT_LENGTH(cur_hout, p->k_eff, p->stride); + + uint32_t h_off_out = (uint32_t) hout_idx * (uint32_t) p->wout_b; + uint32_t h_off_in = (uint32_t) hout_idx * (uint32_t) p->stride * (uint32_t) p->win_pad_b; + + for (int wid = 0; wid < t->w_div; wid++) { + int wout_idx = wid * t->wout_len; + int cur_wout = (wid == t->w_div - 1) ? ceil_rem(stm->wout_len, t->wout_len) : t->wout_len; + int cur_win = FIND_INPUT_LENGTH(cur_wout, p->k_eff, p->stride); + + uint32_t w_off_out = (uint32_t) wout_idx * FP16_B; + uint32_t w_off_in = (uint32_t) wout_idx * (uint32_t) p->stride * FP16_B; + + int ohow = cur_hout * cur_wout; + uint32_t ones_height = (uint32_t) ceil_div(ohow * FP16_B, BIAS_ONES_LENGTH); + __split(); + __start_plan(nest_id); + for (int spu_id = 0; spu_id < last_spu; spu_id++) { + int abs_c = start_cidx[spu_id] + cidx; + uint32_t in_addr = l2i->input_addr + (uint32_t) abs_c * (uint32_t) p->cin_pad_b + + h_off_in + w_off_in; + uint32_t out_addr = l2i->output_addr + (uint32_t) abs_c * (uint32_t) p->cout_b + + h_off_out + w_off_out; + uint32_t knl_addr = l2i->kernel_addr + (uint32_t) abs_c * (uint32_t) p->knl_ch_b; + uint32_t bias_ptr = l2i->bias_addr + (uint32_t) abs_c * FP16_B; + + __start_thread(spu_id); + __set_spm_addr(BASE_L1_BANK_R, BASE_L1_BANK_C, BASE_L1_BANK_B, BASE_L1_BANK_A); + + __load(l2i->bias_ones_addr, BASE_L1_BANK_A, 0, BIAS_ONES_LENGTH, + (uint16_t) ones_height, BIAS_ONES_LENGTH); + __load(bias_ptr, BASE_L1_BANK_B, FP16_B, FP16_B, 1, FP16_B); + __mm_s(ohow, 1, 1); + + __load(in_addr, BASE_L1_BANK_R, (uint32_t) p->win_pad_b, + (uint16_t) (cur_win * FP16_B), (uint16_t) cur_hin, + (uint32_t) (cur_win * FP16_B)); + __im2col_n(cur_hin, cur_win, p->k_size, p->dilation, p->stride, 1); + __load(knl_addr, BASE_L1_BANK_B, (uint32_t) p->knl_ch_b, + (uint16_t) p->knl_ch_b, 1, (uint32_t) p->knl_ch_b); + __mmc_t(ohow, p->k_n, 1); + + __store(BASE_L1_BANK_R, out_addr, (uint32_t) p->wout_b, + (uint16_t) (cur_wout * FP16_B), (uint16_t) cur_hout, + (uint32_t) (cur_wout * FP16_B)); + __end_thread(spu_id); + } + __end_plan(nest_id); + __join(); + } + } + } +} + +static void ggml_gtx_dwconv_2d_spu_tile_compute_only( + int nest_id, + ConvParams * p, + SPUTileMap * stm, + TileExecPlan * t, + L2Info * l2i) { + + const int use_spu = stm->use_spu; + + int start_c[use_spu]; + int hout_assigned[use_spu]; + int wout_assigned[use_spu]; + uint32_t in_base_spu[use_spu]; + uint32_t out_base_spu[use_spu]; + + { + int spu_id = 0; + for (int cid = 0; cid < stm->cout_div; cid++) { + int start_hout = 0; + for (int hid = 0; hid < stm->h_div; hid++) { + int start_wout = 0; + int cur_h = (hid < stm->h_rem_cnt) ? stm->hout_len : stm->hout_len - 1; + for (int wid = 0; wid < stm->w_div; wid++) { + int cur_w = (wid < stm->w_rem_cnt) ? stm->wout_len : stm->wout_len - 1; + + start_c[spu_id] = cid; + hout_assigned[spu_id] = cur_h; + wout_assigned[spu_id] = cur_w; + + in_base_spu[spu_id] = l2i->input_addr + (uint32_t) cid * (uint32_t) p->cin_pad_b + + (uint32_t) start_hout * (uint32_t) p->stride * (uint32_t) p->win_pad_b + + (uint32_t) start_wout * (uint32_t) p->stride * FP16_B; + out_base_spu[spu_id] = l2i->output_addr + (uint32_t) cid * (uint32_t) p->cout_b + + (uint32_t) start_hout * (uint32_t) p->wout_b + + (uint32_t) start_wout * FP16_B; + + start_wout += cur_w; + spu_id++; + } + start_hout += cur_h; + } + } + } + + for (int hid = 0; hid < t->h_div; hid++) { + const bool is_h_rem = (hid == t->h_div - 1); + const int hout_idx = hid * t->hout_len; + const uint32_t hout_off = (uint32_t) hout_idx * (uint32_t) p->wout_b; + const uint32_t hin_off = (uint32_t) hout_idx * (uint32_t) p->stride * (uint32_t) p->win_pad_b; + + for (int wid = 0; wid < t->w_div; wid++) { + const bool is_w_rem = (wid == t->w_div - 1); + const int wout_idx = wid * t->wout_len; + const uint32_t wout_off = (uint32_t) wout_idx * FP16_B; + const uint32_t win_off = (uint32_t) wout_idx * (uint32_t) p->stride * FP16_B; + __split(); + __start_plan(nest_id); + for (int spu_id = 0; spu_id < use_spu; spu_id++) { + int cur_hout = is_h_rem ? ceil_rem(hout_assigned[spu_id], t->hout_len) : t->hout_len; + int cur_wout = is_w_rem ? ceil_rem(wout_assigned[spu_id], t->wout_len) : t->wout_len; + int cur_hin = FIND_INPUT_LENGTH(cur_hout, p->k_eff, p->stride); + int cur_win = FIND_INPUT_LENGTH(cur_wout, p->k_eff, p->stride); + int ohow = cur_hout * cur_wout; + + uint32_t in_addr = in_base_spu[spu_id] + hin_off + win_off; + uint32_t out_addr = out_base_spu[spu_id] + hout_off + wout_off; + uint32_t knl_addr = l2i->kernel_addr + (uint32_t) start_c[spu_id] * (uint32_t) p->knl_ch_b; + uint32_t bias_ptr = l2i->bias_addr + (uint32_t) start_c[spu_id] * FP16_B; + uint32_t ones_height = (uint32_t) ceil_div(ohow * FP16_B, BIAS_ONES_LENGTH); + + __start_thread(spu_id); + __set_spm_addr(BASE_L1_BANK_R, BASE_L1_BANK_C, BASE_L1_BANK_B, BASE_L1_BANK_A); + + __load(l2i->bias_ones_addr, BASE_L1_BANK_A, 0, BIAS_ONES_LENGTH, + (uint16_t) ones_height, BIAS_ONES_LENGTH); + __load(bias_ptr, BASE_L1_BANK_B, FP16_B, FP16_B, 1, FP16_B); + __mm_s(ohow, 1, 1); + + __load(in_addr, BASE_L1_BANK_R, (uint32_t) p->win_pad_b, + (uint16_t) (cur_win * FP16_B), (uint16_t) cur_hin, + (uint32_t) (cur_win * FP16_B)); + + __set_spm_addr(BASE_L1_BANK_R, BASE_L1_BANK_C, BASE_L1_BANK_B, BASE_L1_BANK_A); + __im2col_n(cur_hin, cur_win, p->k_size, p->dilation, p->stride, 1); + + __load(knl_addr, BASE_L1_BANK_B, (uint32_t) p->knl_ch_b, + (uint16_t) p->knl_ch_b, 1, (uint32_t) p->knl_ch_b); + __mmc_t(ohow, p->k_n, 1); + + __store(BASE_L1_BANK_R, out_addr, (uint32_t) p->wout_b, + (uint16_t) (cur_wout * FP16_B), (uint16_t) cur_hout, + (uint32_t) (cur_wout * FP16_B)); + __end_thread(spu_id); + } + __end_plan(nest_id); + __join(); + } + } +} + +int ggml_gtx_kernel_conv_2d_dw_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_CONV_2D_DW_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_CONV_2D_DW_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_L2_STASHED_N1 || + desc->l2_dst_addr == 0 || + desc->l2_src_addr[0] == 0 || + desc->l2_src_addr[1] == 0 || + desc->l2_scratch_capacity == 0 || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->src_type[1] != GGML_GTX_TYPE_F16) { + return -2; + } + + const bool c_fast = desc->op_params[GGML_GTX_CONV_LAYOUT_PARAM] == GGML_GTX_CONV_LAYOUT_C_FAST; + const uint32_t kw = c_fast ? (uint32_t)desc->op_params[GGML_GTX_CONV_LOGICAL_KW_PARAM] : desc->src_ne[0][0]; + const uint32_t kh = c_fast ? (uint32_t)desc->op_params[GGML_GTX_CONV_LOGICAL_KH_PARAM] : desc->src_ne[0][1]; + const uint32_t c = c_fast ? (uint32_t)desc->op_params[GGML_GTX_CONV_LOGICAL_C_PARAM] : desc->src_ne[1][2]; + const uint32_t iw = c_fast ? (uint32_t)desc->op_params[GGML_GTX_CONV_LOGICAL_IW_PARAM] : desc->src_ne[1][0]; + const uint32_t ih = c_fast ? (uint32_t)desc->op_params[GGML_GTX_CONV_LOGICAL_IH_PARAM] : desc->src_ne[1][1]; + const uint32_t ow = c_fast ? (uint32_t)desc->op_params[GGML_GTX_CONV_LOGICAL_OW_PARAM] : desc->ne[0]; + const uint32_t oh = c_fast ? (uint32_t)desc->op_params[GGML_GTX_CONV_LOGICAL_OH_PARAM] : desc->ne[1]; + const uint32_t n = c_fast ? 1u : desc->src_ne[1][3]; + const int32_t s0 = desc->op_params[0]; + const int32_t s1 = desc->op_params[1]; + const int32_t p0 = desc->op_params[2]; + const int32_t p1 = desc->op_params[3]; + const int32_t d0 = desc->op_params[4]; + const int32_t d1 = desc->op_params[5]; + + const bool raw_shape_ok = c_fast ? + (desc->src_ne[0][0] == c && desc->src_ne[0][1] == kw && desc->src_ne[0][2] == kh && + desc->src_ne[0][3] == 1 && desc->src_ne[1][0] == c && desc->src_ne[1][1] == iw && + desc->src_ne[1][2] == ih && desc->src_ne[1][3] == 1 && desc->ne[0] == c && + desc->ne[1] == ow && desc->ne[2] == oh && desc->ne[3] == 1) : + (desc->src_ne[0][2] == 1 && desc->src_ne[0][3] == c && desc->ne[2] == c && + desc->ne[3] == 1); + if (kw == 0 || kh == 0 || c == 0 || iw == 0 || ih == 0 || ow == 0 || oh == 0 || + kw != kh || !raw_shape_ok || n != 1 || + s0 <= 0 || s1 <= 0 || s0 != s1 || + d0 <= 0 || d1 <= 0 || d0 != d1 || + p0 < 0 || p1 < 0) { + return -3; + } + + ConvParams full = { + .cin = (int) c, + .hin = (int) ih, + .win = (int) iw, + .cout = (int) c, + .k_size = (int) kw, + .pad_h = 0, + .pad_w = (int) p0, + .stride = (int) s0, + .dilation = (int) d0, + .bias = false, + }; + derive_convParam(&full); + + const int64_t required_hin_pad = ((int64_t)oh - 1) * s1 + full.k_eff; + const int64_t pad_total = required_hin_pad - ih; + if (pad_total < p1 || pad_total - p1 > UINT16_MAX) { + return -4; + } + full.pad_h = (int)p1; + full.hin_pad = (int)required_hin_pad; + full.hout = (int)oh; + full.cin_pad_b = full.hin_pad * full.win_pad_b; + full.cout_b = full.hout * full.wout_b; + + if ((uint32_t)full.wout != ow) { + return -4; + } + + const uint32_t nest_count = 1u; + const uint64_t input_x_stride = desc->src_nb[1][c_fast ? 1 : 0]; + const uint64_t input_y_stride = desc->src_nb[1][c_fast ? 2 : 1]; + const uint64_t input_ch_stride = desc->src_nb[1][c_fast ? 0 : 2]; + const uint64_t kernel_x_stride = desc->src_nb[0][c_fast ? 1 : 0]; + const uint64_t kernel_y_stride = desc->src_nb[0][c_fast ? 2 : 1]; + const uint64_t kernel_ch_stride = desc->src_nb[0][c_fast ? 0 : 3]; + const uint64_t output_x_stride = desc->nb[c_fast ? 1 : 0]; + const uint64_t output_y_stride = desc->nb[c_fast ? 2 : 1]; + const uint64_t output_ch_stride = desc->nb[c_fast ? 0 : 2]; + const uint32_t scratch_base = desc->l2_scratch_addr; + uint32_t ch_start = 0; + + while (ch_start < c) { + for (uint32_t nest_id = 0; nest_id < nest_count && ch_start < c; ++nest_id) { + const uint32_t remaining_ch = c - ch_start; + const uint32_t remaining_nests = nest_count - nest_id; + uint32_t ch_count = (remaining_ch + remaining_nests - 1u) / remaining_nests; + + ConvParams p; + L2Info l2i; + bool found_slice = false; + while (ch_count > 0) { + p = full; + p.cin = (int) ch_count; + p.cout = (int) ch_count; + derive_convParam(&p); + p.hin_pad = full.hin_pad; + p.hout = full.hout; + p.cin_pad_b = p.hin_pad * p.win_pad_b; + p.cout_b = p.hout * p.wout_b; + if ((uint64_t) p.hin_pad * (uint64_t) p.cin <= UINT16_MAX && + calculate_l2_usage_bias32_dwconv(&p, &l2i, 0u, desc->l2_scratch_capacity) && + ggml_gtx_dwconv_l2_slice_fits(&p, &l2i)) { + ggml_gtx_dwconv_rebase_scratch(&l2i, scratch_base); + found_slice = true; + break; + } + --ch_count; + } + + if (!found_slice) { + return -5; + } + + DDRInfo ddri = { + .input_addr = desc->l2_src_addr[1] + (uint64_t) ch_start * input_ch_stride, + .kernel_addr = desc->l2_src_addr[0] + (uint64_t) ch_start * kernel_ch_stride, + .bias_addr = desc->l2_src_addr[0] + (uint64_t) ch_start * kernel_ch_stride, + .output_addr = desc->l2_dst_addr + (uint64_t) ch_start * output_ch_stride, + }; + + SPUTileMap stm; + TileExecPlan t; + tile_dwconv_workload(&p, &stm, &t, GGML_GTX_SPU_NUM_PER_NEST); + + ggml_gtx_dwconv_load_l2_strided((int) nest_id, + &p, + &l2i, + ddri.input_addr, + ddri.kernel_addr, + input_x_stride, + input_y_stride, + input_ch_stride, + kernel_x_stride, + kernel_y_stride, + kernel_ch_stride); + if (stm.h_div * stm.w_div > 1) { + ggml_gtx_dwconv_2d_spu_tile_compute_only((int) nest_id, &p, &stm, &t, &l2i); + } else { + ggml_gtx_dwconv_2d_tile_compute_only((int) nest_id, &p, &stm, &t, &l2i); + } + ggml_gtx_dwconv_store_l2_strided((int) nest_id, + &p, + l2i.output_addr, + ddri.output_addr, + output_x_stride, + output_y_stride, + output_ch_stride); + ch_start += ch_count; + } + } + + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_conv_2d_dw_f16(arg); +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-conv-2d.c b/depend/ggml-gtx/src/ggml-gtx-kernel-conv-2d.c new file mode 100644 index 0000000..cde1c3c --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-conv-2d.c @@ -0,0 +1,254 @@ +//================================================================== +// GGML GTX kernel: CONV_2D +// +// Descriptor wrapper around the host-owned GTX convolution core. +//================================================================== + +#include + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#include "ggml-gtx-kernel-conv-2d-core.h" + +static void ggml_gtx_conv_rebase_scratch(L2Info * l2i, uint32_t scratch_base) { + l2i->input_addr += scratch_base; + l2i->kernel_addr += scratch_base; + l2i->bias_addr += scratch_base; + l2i->output_addr += scratch_base; + l2i->bias_ones_addr += scratch_base; +} + +static void set_single_conv_l2_mapped_input( + int nest_id, ConvParams * p, + uint32_t l2_offset, uint32_t l2_cap, L2Info * l2i, + uint32_t l2_input, uint32_t l2_kernel, + uint32_t input_channel_stride, + uint32_t pad_h_top, + uint32_t pad_h_bottom) +{ + (void) l2_offset; + (void) l2_cap; + + const uint32_t input_row_bytes = p->w_in * FP16_B; + const uint32_t padded_row_bytes = p->win_pad * FP16_B; + const uint32_t padded_channel_bytes = p->hin_pad * padded_row_bytes; + const uint32_t knl_read_length = p->c_in * p->k_n * FP16_B; + + __start_plan(nest_id); + __start_shared(); + __fill(l2i->input_addr, padded_row_bytes, (uint16_t)padded_row_bytes, + (uint16_t)((uint32_t)p->hin_pad * p->c_in), 0u, 0); + for (int c = 0; c < p->c_in; ++c) { + __copy(l2_input + (uint32_t)c * input_channel_stride, + l2i->input_addr + (uint32_t)c * padded_channel_bytes + + pad_h_top * padded_row_bytes + (uint32_t)p->pad_w * FP16_B, + input_row_bytes, (uint16_t)input_row_bytes, + (uint16_t)p->h_in, padded_row_bytes); + } + + __copy(l2_kernel, l2i->kernel_addr, + knl_read_length, (uint16_t)knl_read_length, + (uint16_t)p->c_out, knl_read_length); + + const uint64_t bias_fill_pattern = (p->bias > 0) ? 0x3C003C003C003C00ull : 0x0000ull; + const uint32_t bias_length = p->c_out * FP16_B; + __fill(l2i->bias_addr, bias_length, (uint16_t)bias_length, 1, 0u, 0); + __fill(l2i->bias_ones_addr, 0x1000, 0x0fff, 0x10, bias_fill_pattern, 0); + __end_shared(); + __end_plan(nest_id); +} + +static void copy_conv_l2_h_slice( + int nest_id, + const ConvParams * p, + uint32_t l2_out_addr, + uint32_t l2_output, + uint32_t output_oc_stride) +{ + const uint32_t row_bytes = (uint32_t) p->w_out * FP16_B; + const uint32_t channel_bytes = (uint32_t) p->h_out * row_bytes; + + __start_plan(nest_id); + __start_shared(); + for (int oc_idx = 0; oc_idx < p->c_out; ++oc_idx) { + __copy(l2_out_addr + (uint32_t) oc_idx * channel_bytes, + l2_output + (uint32_t) oc_idx * output_oc_stride, + row_bytes, (uint16_t)row_bytes, + (uint16_t)p->h_out, row_bytes); + } + __end_shared(); + __end_plan(nest_id); +} + +int ggml_gtx_kernel_conv_2d_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_CONV_2D_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_CONV_2D_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_L2_STASHED_N1 || + desc->l2_dst_addr == 0 || + desc->l2_src_addr[0] == 0 || + desc->l2_src_addr[1] == 0 || + desc->l2_scratch_capacity == 0 || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->src_type[1] != GGML_GTX_TYPE_F16) { + return -2; + } + + const uint32_t kw = desc->src_ne[0][0]; + const uint32_t kh = desc->src_ne[0][1]; + const uint32_t ic = desc->src_ne[0][2]; + const uint32_t oc = desc->src_ne[0][3]; + const uint32_t iw = desc->src_ne[1][0]; + const uint32_t ih = desc->src_ne[1][1]; + const uint32_t n = desc->src_ne[1][3]; + const int32_t s0 = desc->op_params[0]; + const int32_t s1 = desc->op_params[1]; + const int32_t p0 = desc->op_params[2]; + const int32_t p1 = desc->op_params[3]; + const int32_t d0 = desc->op_params[4]; + const int32_t d1 = desc->op_params[5]; + + if (kw == 0 || kh == 0 || ic == 0 || oc == 0 || iw == 0 || ih == 0 || + kw != kh || desc->src_ne[1][2] != ic || desc->ne[2] != oc || + n != 1 || desc->ne[3] != 1 || + s0 <= 0 || s1 <= 0 || s0 != s1 || + d0 <= 0 || d1 <= 0 || d0 != d1 || + p0 < 0 || p1 < 0) { + return -3; + } + + ConvParams full = { + .c_in = (int) ic, + .h_in = (int) ih, + .w_in = (int) iw, + .c_out = (int) oc, + .k_size = (int) kw, + .pad_h = 0, + .pad_w = (int) p0, + .stride = (int) s0, + .dilation = (int) d0, + .bias = false, + }; + derive_convParam(&full); + + const int64_t required_hin_pad = ((int64_t)desc->ne[1] - 1) * s1 + full.k_eff; + const int64_t pad_total = required_hin_pad - ih; + if (pad_total < p1 || pad_total - p1 > UINT16_MAX) { + return -4; + } + const uint32_t local_pad_bottom = (uint32_t)(pad_total - p1); + full.hin_pad = (int)required_hin_pad; + full.h_out = (int)desc->ne[1]; + + if ((uint32_t)full.w_out != desc->ne[0]) { + return -4; + } + + const uint32_t nest_count = 1u; + const uint32_t dst_oc_stride = desc->nb[2]; + const uint32_t scratch_base = desc->l2_scratch_addr; + L2Info full_l2i; + if (calculate_l2_usage_bias(&full, &full_l2i, 0u, desc->l2_scratch_capacity)) { + ggml_gtx_conv_rebase_scratch(&full_l2i, scratch_base); + SPUTileMap stm; + TileExecPlan t; + tile_nmconv_general(&full, &stm, &t, GGML_GTX_SPU_NUM_PER_NEST); + + set_single_conv_l2_mapped_input(0, &full, scratch_base, desc->l2_scratch_capacity, &full_l2i, + (uint32_t)desc->l2_src_addr[1], + (uint32_t)desc->l2_src_addr[0], + desc->src_nb[1][2], (uint32_t)p1, local_pad_bottom); + conv_tile_dim4_with_bias(0, &full, &stm, &t, &full_l2i); + copy_conv_l2_h_slice(0, &full, full_l2i.output_addr, + (uint32_t)desc->l2_dst_addr, dst_oc_stride); + return 0; + } + + uint32_t out_h_start = 0; + + while (out_h_start < desc->ne[1]) { + for (uint32_t nest_id = 0; nest_id < nest_count && out_h_start < desc->ne[1]; ++nest_id) { + const uint32_t remaining_h = desc->ne[1] - out_h_start; + const uint32_t remaining_nests = nest_count - nest_id; + uint32_t out_h_count = (remaining_h + remaining_nests - 1u) / remaining_nests; + + ConvParams p; + L2Info l2i; + int32_t clip_h_start = 0; + uint32_t local_pad_top = 0; + uint32_t local_pad_bottom = 0; + bool found_slice = false; + + while (out_h_count > 0) { + const int32_t raw_h_start = (int32_t) out_h_start * s1 - p1; + const int32_t raw_h_end = + (int32_t) (out_h_start + out_h_count - 1u) * s1 - p1 + full.k_eff; + clip_h_start = raw_h_start < 0 ? 0 : raw_h_start; + const int32_t clip_h_end = raw_h_end > (int32_t) ih ? (int32_t) ih : raw_h_end; + local_pad_top = (uint32_t) (raw_h_start < 0 ? -raw_h_start : 0); + local_pad_bottom = (uint32_t) (raw_h_end > (int32_t) ih ? raw_h_end - (int32_t) ih : 0); + const uint32_t local_h_in = + clip_h_end > clip_h_start ? (uint32_t) (clip_h_end - clip_h_start) : 0u; + if (local_h_in == 0) { + --out_h_count; + continue; + } + + p = full; + p.h_in = (int) local_h_in; + p.pad_h = 0; + derive_convParam(&p); + p.hin_pad = (int) (local_h_in + local_pad_top + local_pad_bottom); + p.h_out = (p.hin_pad - p.k_eff) / p.stride + 1; + if ((uint32_t) p.h_out != out_h_count) { + --out_h_count; + continue; + } + if (!calculate_l2_usage_bias(&p, &l2i, 0u, desc->l2_scratch_capacity)) { + --out_h_count; + continue; + } + ggml_gtx_conv_rebase_scratch(&l2i, scratch_base); + found_slice = true; + break; + } + + if (!found_slice) { + return -7; + } + + SPUTileMap stm; + TileExecPlan t; + tile_nmconv_general(&p, &stm, &t, GGML_GTX_SPU_NUM_PER_NEST); + + const uint32_t input_addr = (uint32_t)desc->l2_src_addr[1] + + (uint32_t)clip_h_start * desc->src_nb[1][1]; + const uint32_t dst_addr = (uint32_t)desc->l2_dst_addr + + out_h_start * desc->nb[1]; + + set_single_conv_l2_mapped_input((int)nest_id, &p, scratch_base, + desc->l2_scratch_capacity, &l2i, + input_addr, (uint32_t)desc->l2_src_addr[0], + desc->src_nb[1][2], local_pad_top, local_pad_bottom); + conv_tile_dim4_with_bias((int) nest_id, &p, &stm, &t, &l2i); + copy_conv_l2_h_slice((int)nest_id, &p, l2i.output_addr, dst_addr, dst_oc_stride); + out_h_start += out_h_count; + } + } + + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_conv_2d_f16(arg); +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-conv-transpose-2d.c b/depend/ggml-gtx/src/ggml-gtx-kernel-conv-transpose-2d.c new file mode 100644 index 0000000..1412472 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-conv-transpose-2d.c @@ -0,0 +1,292 @@ +//================================================================== +// GGML GTX kernel: CONV_TRANSPOSE_2D +// +// Generalized descriptor wrapper around reference/conv_transpose_2d.c. +// The reference structure is preserved: each output element packs its +// contributing input/kernel values and computes one dot product. +//================================================================== + +#include + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#define GGML_GTX_CT2D_BANK_A 0x00000u +#define GGML_GTX_CT2D_BANK_B 0x20000u +#define GGML_GTX_CT2D_BANK_C 0x30000u +#define GGML_GTX_CT2D_BANK_R 0x50000u + +#define GGML_GTX_CT2D_PACK_INPUT GGML_GTX_CT2D_BANK_C +#define GGML_GTX_CT2D_PACK_KERNEL (GGML_GTX_CT2D_BANK_C + 0x10000u) +#define GGML_GTX_CT2D_SLOT_ADDR (GGML_GTX_CT2D_BANK_C + 0x1f000u) +#define GGML_GTX_CT2D_FP16_B 2u +#define GGML_GTX_CT2D_SPUS GGML_GTX_SPU_NUM_PER_NEST +#define GGML_GTX_CT2D_MAX_CONTRIB 32768u +#define GGML_GTX_CT2D_L1_TILE_BYTES 0x10000u +#define GGML_GTX_CT2D_MM_T_MAX_M 256u + +static inline uint32_t ggml_gtx_ct2d_min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t ggml_gtx_ct2d_outputs_for_tid(uint32_t total, uint8_t tid) { + const uint32_t q = total / GGML_GTX_CT2D_SPUS; + const uint32_t r = total % GGML_GTX_CT2D_SPUS; + return tid < r ? q + 1u : q; +} + +static inline uint32_t ggml_gtx_ct2d_start_for_tid(uint32_t total, uint8_t tid) { + const uint32_t q = total / GGML_GTX_CT2D_SPUS; + const uint32_t r = total % GGML_GTX_CT2D_SPUS; + if (tid < r) { + return (uint32_t) tid * (q + 1u); + } + return r * (q + 1u) + ((uint32_t) tid - r) * q; +} + +/* Reference-style preload/pack path for the common upsampling case. With a + * 2x2 kernel and stride 2, each (ky,kx) maps an input pixel to a disjoint + * output sub-lattice. Pack spatial rows as MxIC and the corresponding + * weights as OCxIC, then use the same __mm_t contract as MUL_MAT. */ +static int ggml_gtx_ct2d_stride2_k2_block(const ggml_gtx_kernel_launch_desc_t * desc, + uint32_t iw, uint32_t ih, + uint32_t ic, uint32_t oc, + uint32_t ow, uint32_t oh) { + const uint32_t input_plane_bytes = iw * ih * GGML_GTX_CT2D_FP16_B; + const uint32_t kernel_ic_stride = 2u * 2u * oc * GGML_GTX_CT2D_FP16_B; + const uint32_t k_bytes = ic * GGML_GTX_CT2D_FP16_B; + if (k_bytes == 0u || k_bytes > UINT16_MAX || iw == 0u || iw > UINT16_MAX) { + return -1; + } + + uint32_t rows_per_block = GGML_GTX_CT2D_L1_TILE_BYTES / (iw * k_bytes); + const uint32_t rows_by_result = GGML_GTX_CT2D_L1_TILE_BYTES / + (oc * iw * GGML_GTX_CT2D_FP16_B); + if (rows_per_block > rows_by_result) rows_per_block = rows_by_result; + /* The FPGA mm.t path rejects the otherwise L1-valid M=512 tile used by + * the 128x128 -> 256x256 MobileSAM upsample. Keep spatial M within the + * verified hardware range and split additional input rows instead. */ + const uint32_t rows_by_mm_m = GGML_GTX_CT2D_MM_T_MAX_M / iw; + if (rows_per_block > rows_by_mm_m) rows_per_block = rows_by_mm_m; + if (rows_per_block > ih) rows_per_block = ih; + if (rows_per_block == 0u) return -1; + + for (uint32_t ky = 0u; ky < 2u; ++ky) { + for (uint32_t kx = 0u; kx < 2u; ++kx) { + for (uint32_t y0 = 0u; y0 < ih; y0 += rows_per_block) { + const uint32_t cur_rows = (ih - y0) < rows_per_block ? ih - y0 : rows_per_block; + const uint32_t m = cur_rows * iw; + if (m == 0u || m > UINT16_MAX) return -1; + + __split(); + __start_plan(0u); + __start_thread(0u); + __set_spm_addr(GGML_GTX_CT2D_BANK_R, GGML_GTX_CT2D_BANK_C, + GGML_GTX_CT2D_BANK_B, GGML_GTX_CT2D_BANK_A); + + for (uint32_t p = 0u; p < m; ++p) { + const uint32_t input_pixel = y0 * iw + p; + __load((uint32_t) desc->l2_src_addr[1] + + input_pixel * GGML_GTX_CT2D_FP16_B, + GGML_GTX_CT2D_BANK_A + p * k_bytes, + input_plane_bytes, + GGML_GTX_CT2D_FP16_B, + (uint16_t) ic, + GGML_GTX_CT2D_FP16_B); + } + + for (uint32_t out_ch = 0u; out_ch < oc; ++out_ch) { + const uint32_t kernel_elem = + ((out_ch * 2u + ky) * 2u + kx); + __load((uint32_t) desc->l2_src_addr[0] + + kernel_elem * GGML_GTX_CT2D_FP16_B, + GGML_GTX_CT2D_BANK_B + out_ch * k_bytes, + kernel_ic_stride, + GGML_GTX_CT2D_FP16_B, + (uint16_t) ic, + GGML_GTX_CT2D_FP16_B); + } + + __mm_t((uint16_t) m, (uint16_t) ic, (uint16_t) oc); + + for (uint32_t out_ch = 0u; out_ch < oc; ++out_ch) { + for (uint32_t ry = 0u; ry < cur_rows; ++ry) { + const uint32_t out_y = (y0 + ry) * 2u + ky; + const uint32_t dst_elem = + (out_ch * oh + out_y) * ow + kx; + __store(GGML_GTX_CT2D_BANK_R + + (out_ch * m + ry * iw) * GGML_GTX_CT2D_FP16_B, + (uint32_t) desc->l2_dst_addr + + dst_elem * GGML_GTX_CT2D_FP16_B, + 2u * GGML_GTX_CT2D_FP16_B, + GGML_GTX_CT2D_FP16_B, + (uint16_t) iw, + GGML_GTX_CT2D_FP16_B); + } + } + __end_thread(0u); + __end_plan(0u); + __join(); + } + } + } + return 0; +} + +int ggml_gtx_kernel_conv_transpose_2d_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_CONV_TRANSPOSE_2D_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_CONV_TRANSPOSE_2D_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_L2_STASHED_N1 || + desc->l2_dst_addr == 0 || + desc->l2_src_addr[0] == 0 || + desc->l2_src_addr[1] == 0 || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->src_type[1] != GGML_GTX_TYPE_F16) { + return -2; + } + + const uint32_t kw = desc->src_ne[0][0]; + const uint32_t kh = desc->src_ne[0][1]; + const uint32_t oc = desc->src_ne[0][2]; + const uint32_t ic = desc->src_ne[0][3]; + const uint32_t iw = desc->src_ne[1][0]; + const uint32_t ih = desc->src_ne[1][1]; + const uint32_t in_c = desc->src_ne[1][2]; + const uint32_t batch = desc->src_ne[1][3]; + const uint32_t ow = desc->ne[0]; + const uint32_t oh = desc->ne[1]; + const uint32_t out_c = desc->ne[2]; + const uint32_t out_b = desc->ne[3]; + const int32_t stride = desc->op_params[0]; + + if (kw == 0 || kh == 0 || oc == 0 || ic == 0 || + iw == 0 || ih == 0 || batch == 0 || + in_c != ic || out_c != oc || out_b != batch || + stride <= 0 || + ow != (iw - 1u) * (uint32_t) stride + kw || + oh != (ih - 1u) * (uint32_t) stride + kh || + kw * kh * ic > GGML_GTX_CT2D_MAX_CONTRIB) { + return -3; + } + + const uint64_t input_bytes = (uint64_t) iw * ih * ic * batch * GGML_GTX_CT2D_FP16_B; + const uint64_t kernel_bytes = (uint64_t) kw * kh * oc * ic * GGML_GTX_CT2D_FP16_B; + const uint64_t output_bytes = (uint64_t) ow * oh * oc * batch * GGML_GTX_CT2D_FP16_B; + if (input_bytes == 0 || kernel_bytes == 0 || output_bytes == 0 || + iw * GGML_GTX_CT2D_FP16_B > UINT16_MAX || + kw * GGML_GTX_CT2D_FP16_B > UINT16_MAX || + ow * GGML_GTX_CT2D_FP16_B > UINT16_MAX || + ih * ic * batch > UINT16_MAX || + kh * oc * ic > UINT16_MAX || + oh * oc * batch > UINT16_MAX) { + return -4; + } + + if (kw == 2u && kh == 2u && stride == 2 && batch == 1u) { + return ggml_gtx_ct2d_stride2_k2_block(desc, iw, ih, ic, oc, ow, oh); + } + + const uint32_t total_outputs = ow * oh * oc * batch; + __split(); + __start_plan(0); + for (uint8_t tid = 0; tid < GGML_GTX_CT2D_SPUS; ++tid) { + const uint32_t thread_outputs = ggml_gtx_ct2d_outputs_for_tid(total_outputs, tid); + const uint32_t base_output = ggml_gtx_ct2d_start_for_tid(total_outputs, tid); + __start_thread(tid); + if (thread_outputs > 0u) { + __set_spm_addr(GGML_GTX_CT2D_BANK_R, GGML_GTX_CT2D_BANK_C, + GGML_GTX_CT2D_BANK_B, GGML_GTX_CT2D_BANK_A); + for (uint32_t out_local = 0; out_local < thread_outputs; ++out_local) { + const uint32_t out_idx = base_output + out_local; + const uint32_t out_x = out_idx % ow; + const uint32_t out_y = (out_idx / ow) % oh; + const uint32_t out_ch = (out_idx / (ow * oh)) % oc; + const uint32_t out_batch = out_idx / (ow * oh * oc); + uint32_t contrib_count = 0; + + for (uint32_t ky = 0; ky < kh; ++ky) { + if (out_y < ky) { + continue; + } + const uint32_t dy = out_y - ky; + if ((dy % (uint32_t) stride) != 0u) { + continue; + } + const uint32_t in_y = dy / (uint32_t) stride; + if (in_y >= ih) { + continue; + } + + for (uint32_t kx = 0; kx < kw; ++kx) { + if (out_x < kx) { + continue; + } + const uint32_t dx = out_x - kx; + if ((dx % (uint32_t) stride) != 0u) { + continue; + } + const uint32_t in_x = dx / (uint32_t) stride; + if (in_x >= iw) { + continue; + } + + for (uint32_t in_ch = 0; in_ch < ic; ++in_ch) { + const uint32_t input_off = + (((out_batch * ic + in_ch) * ih + in_y) * iw + in_x) * + GGML_GTX_CT2D_FP16_B; + const uint32_t kernel_off = + (((in_ch * oc + out_ch) * kh + ky) * kw + kx) * + GGML_GTX_CT2D_FP16_B; + const uint32_t pack_off = contrib_count * GGML_GTX_CT2D_FP16_B; + + __load((uint32_t)desc->l2_src_addr[1] + input_off, + GGML_GTX_CT2D_PACK_INPUT + pack_off, + GGML_GTX_CT2D_FP16_B, + GGML_GTX_CT2D_FP16_B, + 1, GGML_GTX_CT2D_FP16_B); + __load((uint32_t)desc->l2_src_addr[0] + kernel_off, + GGML_GTX_CT2D_PACK_KERNEL + pack_off, + GGML_GTX_CT2D_FP16_B, + GGML_GTX_CT2D_FP16_B, + 1, GGML_GTX_CT2D_FP16_B); + ++contrib_count; + } + } + } + + __set_spm_addr_A(GGML_GTX_CT2D_PACK_INPUT); + __set_spm_addr_B(GGML_GTX_CT2D_PACK_KERNEL); + __dot_product(contrib_count, 0); + __set_spm_addr_A(GGML_GTX_CT2D_BANK_A); + __set_spm_addr_B(GGML_GTX_CT2D_BANK_B); + __store_svr(GGML_GTX_CT2D_SLOT_ADDR, 0); + + __store(GGML_GTX_CT2D_SLOT_ADDR, + (uint32_t)desc->l2_dst_addr + + out_idx * GGML_GTX_CT2D_FP16_B, + GGML_GTX_CT2D_FP16_B, + GGML_GTX_CT2D_FP16_B, + 1, GGML_GTX_CT2D_FP16_B); + } + } + __end_thread(tid); + } + __end_plan(0); + __join(); + + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_conv_transpose_2d_f16(arg); +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-cos.c b/depend/ggml-gtx/src/ggml-gtx-kernel-cos.c new file mode 100644 index 0000000..8f24fe0 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-cos.c @@ -0,0 +1,123 @@ +//================================================================== +// GGML GTX kernel: COS +// +// Descriptor wrapper around the verified n1s16 COS arithmetic sequence. +// +// The previous wrapper selected the COS branch in reference/sincos.c. That +// branch is not numerically correct for part of the MobileSAM positional +// encoding range. Keep the range-reduction and Horner sequence from +// gtx-firmware-stash/kernels/n1s16/n1s16_cos.c, with only the memory movement +// changed from fixed DDR buffers to descriptor-provided L2 addresses. +//================================================================== + +#include + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#include "gtx/intrin.h" + +#define COS_DTYPE 2u +#define COS_VEC_ELEMS 16u +#define COS_VEC_BYTES (COS_VEC_ELEMS * COS_DTYPE) + +#define COS_BANK_A 0x00000u +#define COS_BANK_R 0x50000u + +#define COS_FP16_TWO_PI 0x4648u +#define COS_FP16_INV_2PI 0x3118u +#define COS_FP16_C0 0x3C00u +#define COS_FP16_C1 0xB800u +#define COS_FP16_C2 0x2955u +#define COS_FP16_C3 0x95B4u +#define COS_FP16_C4 0x01A0u + +static void ggml_gtx_cos_vec_l2(uint32_t src_addr, uint32_t dst_addr) { + __start_plan(0); + __start_thread(0); + __load(src_addr, COS_BANK_A, + COS_VEC_BYTES, COS_VEC_BYTES, 1, COS_VEC_BYTES); + __load_svr(COS_BANK_A, 0); + + // n = round(x / (2*pi)); x_red = x - n * 2*pi + __mul_is(0, COS_FP16_INV_2PI, 1, 0); + __rne_i(1, 2); + __mul_is(2, COS_FP16_TWO_PI, 3, 0); + __sub_ii(0, 3, 4); + + // Match the x^8 coefficient already used by reference/sincos.c: + // cos(x_red) = 1 + x^2*(-1/2 + x^2*(1/24 + + // x^2*(-1/720 + x^2/40320))) + __mul_ii(4, 4, 5); + __mul_is(5, COS_FP16_C4, 6, 0); + __add_is(6, COS_FP16_C3, 7, 0); + __mul_ii(5, 7, 8); + __add_is(8, COS_FP16_C2, 9, 0); + __mul_ii(5, 9, 10); + __add_is(10, COS_FP16_C1, 11, 0); + __mul_ii(5, 11, 12); + __add_is(12, COS_FP16_C0, 13, 0); + + __store_svr(COS_BANK_R, 13); + __store(COS_BANK_R, dst_addr, + COS_VEC_BYTES, COS_VEC_BYTES, 1, COS_VEC_BYTES); + __end_thread(0); + __end_plan(0); +} + +int ggml_gtx_kernel_cos_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_COS_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_COS_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_L2_STASHED_N1 || + desc->l2_dst_addr == 0 || + desc->l2_src_addr[0] == 0 || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->nb[0] != COS_DTYPE || + desc->src_nb[0][0] != COS_DTYPE) { + return -2; + } + + const uint64_t elems64 = (uint64_t) desc->ne[0] * + (uint64_t) desc->ne[1] * + (uint64_t) desc->ne[2] * + (uint64_t) desc->ne[3]; + if (elems64 == 0 || (elems64 % 16u) != 0u) { + return -3; + } + + for (int i = 0; i < 4; ++i) { + if (desc->src_ne[0][i] != desc->ne[i]) { + return -4; + } + } + if (desc->nb[1] != desc->ne[0] * COS_DTYPE || + desc->src_nb[0][1] != desc->src_ne[0][0] * COS_DTYPE || + (desc->ne[2] > 1 && desc->nb[2] != desc->nb[1] * desc->ne[1]) || + (desc->ne[3] > 1 && desc->nb[3] != desc->nb[2] * desc->ne[2]) || + (desc->src_ne[0][2] > 1 && desc->src_nb[0][2] != desc->src_nb[0][1] * desc->src_ne[0][1]) || + (desc->src_ne[0][3] > 1 && desc->src_nb[0][3] != desc->src_nb[0][2] * desc->src_ne[0][2])) { + return -5; + } + + for (uint64_t elem = 0; elem < elems64; elem += COS_VEC_ELEMS) { + const uint64_t off = elem * COS_DTYPE; + ggml_gtx_cos_vec_l2(desc->l2_src_addr[0] + off, + desc->l2_dst_addr + off); + } + + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_cos_f16(arg); +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-cpy.c b/depend/ggml-gtx/src/ggml-gtx-kernel-cpy.c new file mode 100644 index 0000000..28e38dd --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-cpy.c @@ -0,0 +1,178 @@ +//================================================================== +// GGML GTX kernel: CPY +//================================================================== + +#include +#include +#include + +#include "ggml-gtx-baremetal.h" +#include "ggml-gtx-compute.h" +#include "ggml-gtx-port.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +int ggml_gtx_kernel_cpy_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_CPY_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_CPY_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + if (desc->dst_addr == 0 || + desc->src_addr[0] == 0) { + return -2; + } + + const uint64_t dst_elements = (uint64_t) desc->ne[0] * + (uint64_t) desc->ne[1] * + (uint64_t) desc->ne[2] * + (uint64_t) desc->ne[3]; + const uint64_t src_elements = (uint64_t) desc->src_ne[0][0] * + (uint64_t) desc->src_ne[0][1] * + (uint64_t) desc->src_ne[0][2] * + (uint64_t) desc->src_ne[0][3]; + if (dst_elements != src_elements) { + return -3; + } + + const uint16_t elem_size = + desc->src_type[0] == GGML_GTX_TYPE_F32 ? 4 : 2; + ggml_gtx_compute_cpy(desc->src_addr[0], + desc->dst_addr, + desc->ne[0], + desc->ne[1], + desc->ne[2], + desc->ne[3], + desc->src_ne[0][0], + desc->src_ne[0][1], + desc->src_ne[0][2], + desc->src_ne[0][3], + desc->src_nb[0][0], + desc->src_nb[0][1], + desc->src_nb[0][2], + desc->src_nb[0][3], + desc->nb[0], + desc->nb[1], + desc->nb[2], + desc->nb[3], + elem_size); + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_cpy_f16(arg); +} + +void ggml_gtx_compute_cpy( + uint64_t ddr_src, + uint64_t ddr_dst, + uint32_t ne0, + uint32_t ne1, + uint32_t ne2, + uint32_t ne3, + uint32_t src_ne0, + uint32_t src_ne1, + uint32_t src_ne2, + uint32_t src_ne3, + uint32_t src_nb0, + uint32_t src_nb1, + uint32_t src_nb2, + uint32_t src_nb3, + uint32_t dst_nb0, + uint32_t dst_nb1, + uint32_t dst_nb2, + uint32_t dst_nb3, + uint16_t elem_size) +{ + volatile uint8_t * src = (volatile uint8_t *)(uintptr_t)ddr_src; + volatile uint8_t * dst = (volatile uint8_t *)(uintptr_t)ddr_dst; + + const uint32_t row_bytes = (uint32_t) ne0 * elem_size; + const uint32_t plane_bytes = (uint32_t) ne1 * row_bytes; + const uint32_t block_bytes = (uint32_t) ne2 * plane_bytes; + const bool same_shape = + src_ne0 == ne0 && + src_ne1 == ne1 && + src_ne2 == ne2 && + src_ne3 == ne3; + + const bool src_contiguous = + src_nb0 == elem_size && + src_nb1 == row_bytes && + src_nb2 == plane_bytes && + src_nb3 == block_bytes; + const bool dst_contiguous = + dst_nb0 == elem_size && + dst_nb1 == row_bytes && + dst_nb2 == plane_bytes && + dst_nb3 == block_bytes; + + if (src_contiguous && dst_contiguous) { + const uint64_t total_bytes = (uint64_t) ne3 * block_bytes; + for (uint64_t i = 0; i < total_bytes; ++i) { + dst[i] = src[i]; + } + return; + } + + if (same_shape) { + for (uint32_t i3 = 0; i3 < ne3; ++i3) { + for (uint32_t i2 = 0; i2 < ne2; ++i2) { + for (uint32_t i1 = 0; i1 < ne1; ++i1) { + const uint64_t src_off = (uint64_t) i3 * src_nb3 + + (uint64_t) i2 * src_nb2 + + (uint64_t) i1 * src_nb1; + const uint64_t dst_off = (uint64_t) i3 * dst_nb3 + + (uint64_t) i2 * dst_nb2 + + (uint64_t) i1 * dst_nb1; + if (src_nb0 == elem_size && dst_nb0 == elem_size) { + for (uint32_t b = 0; b < row_bytes; ++b) { + dst[dst_off + b] = src[src_off + b]; + } + } else { + for (uint32_t i0 = 0; i0 < ne0; ++i0) { + const uint64_t dst_col_off = dst_off + (uint64_t) i0 * dst_nb0; + const uint64_t src_col_off = src_off + (uint64_t) i0 * src_nb0; + for (uint16_t eb = 0; eb < elem_size; ++eb) { + dst[dst_col_off + eb] = src[src_col_off + eb]; + } + } + } + } + } + } + return; + } + + const uint64_t total = (uint64_t) ne0 * ne1 * ne2 * ne3; + for (uint64_t linear = 0; linear < total; ++linear) { + uint64_t src_linear = linear; + const uint64_t s0 = src_ne0 ? src_linear % src_ne0 : 0; + src_linear = src_ne0 ? src_linear / src_ne0 : 0; + const uint64_t s1 = src_ne1 ? src_linear % src_ne1 : 0; + src_linear = src_ne1 ? src_linear / src_ne1 : 0; + const uint64_t s2 = src_ne2 ? src_linear % src_ne2 : 0; + src_linear = src_ne2 ? src_linear / src_ne2 : 0; + const uint64_t s3 = src_ne3 ? src_linear % src_ne3 : 0; + + uint64_t dst_linear = linear; + const uint64_t d0 = ne0 ? dst_linear % ne0 : 0; + dst_linear = ne0 ? dst_linear / ne0 : 0; + const uint64_t d1 = ne1 ? dst_linear % ne1 : 0; + dst_linear = ne1 ? dst_linear / ne1 : 0; + const uint64_t d2 = ne2 ? dst_linear % ne2 : 0; + dst_linear = ne2 ? dst_linear / ne2 : 0; + const uint64_t d3 = ne3 ? dst_linear % ne3 : 0; + + const uint64_t src_off = s3 * src_nb3 + s2 * src_nb2 + s1 * src_nb1 + s0 * src_nb0; + const uint64_t dst_off = d3 * dst_nb3 + d2 * dst_nb2 + d1 * dst_nb1 + d0 * dst_nb0; + for (uint16_t eb = 0; eb < elem_size; ++eb) { + dst[dst_off + eb] = src[src_off + eb]; + } + } +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-floor.c b/depend/ggml-gtx/src/ggml-gtx-kernel-floor.c new file mode 100644 index 0000000..51ea700 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-floor.c @@ -0,0 +1,6 @@ +#define GGML_GTX_UNARY_KERNEL_ID GGML_GTX_KERNEL_FLOOR_F16 +#define GGML_GTX_UNARY_KERNEL_FN ggml_gtx_kernel_floor_f16 +#define GGML_GTX_UNARY_INPUT_BANK GGML_GTX_UNARY_BANK_A +#define GGML_GTX_UNARY_OUTPUT_BANK GGML_GTX_UNARY_BANK_R +#define GGML_GTX_UNARY_APPLY(elems) __floor_v(elems) +#include "ggml-gtx-kernel-unary-ref.h" diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-gelu.c b/depend/ggml-gtx/src/ggml-gtx-kernel-gelu.c new file mode 100644 index 0000000..6d96955 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-gelu.c @@ -0,0 +1,245 @@ +//================================================================== +// GGML GTX kernel: GELU +//================================================================== +// Parameterized wrapper around reference/gelu.c. The reference kernel +// structure is kept intact: load per-SPU vector tiles, run __gelu on each +// tile, and store the result. Only fixed DDR addresses and compile-time +// tensor sizes are replaced by launch-descriptor parameters. + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#define GGML_GTX_GELU_L2_INPUT 0x000000u +#define GGML_GTX_GELU_L2_RES 0x080000u +#define GGML_GTX_GELU_L2_BANK_OFFSET 0x100000u +#define GGML_GTX_GELU_L1_AR 0x00000u +#define GGML_GTX_GELU_NOT_USE 0xBEEFu + +static inline uint32_t ggml_gtx_gelu_elems_for_tid(uint32_t total_elems, uint8_t tid) { + const uint32_t q = total_elems / GGML_GTX_SPU_NUM_PER_NEST; + const uint32_t r = total_elems % GGML_GTX_SPU_NUM_PER_NEST; + return tid < r ? q + 1u : q; +} + +static inline uint32_t ggml_gtx_gelu_start_for_tid(uint32_t total_elems, uint8_t tid) { + const uint32_t q = total_elems / GGML_GTX_SPU_NUM_PER_NEST; + const uint32_t r = total_elems % GGML_GTX_SPU_NUM_PER_NEST; + if (tid < r) { + return (uint32_t) tid * (q + 1u); + } + return r * (q + 1u) + ((uint32_t) tid - r) * q; +} + +static uint16_t ggml_gtx_gelu_min_u16(uint64_t a, uint64_t b) { + return (uint16_t) (a < b ? a : b); +} + +void ggml_gtx_compute_gelu( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest) +{ +#ifdef GGML_GTX_USE_GTX + const uint32_t elem_count = (uint32_t) i_row * (uint32_t) i_col; + if (elem_count == 0 || spu_num_per_nest == 0) { + return; + } + + const uint8_t nest_id = 0; + const uint32_t dtype_size = GGML_GTX_DTYPE_SIZE; + const uint32_t max_chunk_elems = 65535u; + + (void) nest_num; + + for (uint32_t base_elem = 0; base_elem < elem_count; base_elem += max_chunk_elems) { + const uint32_t chunk_elems = (elem_count - base_elem) < max_chunk_elems ? + (elem_count - base_elem) : max_chunk_elems; + const uint32_t cal_quot = chunk_elems / spu_num_per_nest; + const uint32_t cal_mod = chunk_elems % spu_num_per_nest; + uint32_t bank_offset_shared = 0; + uint32_t bank_offset_thread = 0; + + __split(); + + __start_plani(nest_id); + + __set_spm_addr_A(GGML_GTX_GELU_L1_AR); + __set_spm_addr_R(GGML_GTX_GELU_L1_AR); + + __start_shared(); + + for (uint8_t thread_id = 0; thread_id < spu_num_per_nest; thread_id++) { + const uint16_t cal_num_per_thread = (uint16_t) (thread_id < cal_mod ? (cal_quot + 1u) : cal_quot); + const uint32_t offset_thread = + (((thread_id < cal_mod) ? + (uint32_t) thread_id * (cal_quot + 1u) : + (uint32_t) cal_mod * (cal_quot + 1u) + (uint32_t) (thread_id - cal_mod) * cal_quot) * + dtype_size); + + if (cal_num_per_thread > 0) { + __load(ddr_input + (uint64_t) base_elem * dtype_size + offset_thread, + GGML_GTX_GELU_L2_INPUT + bank_offset_shared, + cal_num_per_thread, + cal_num_per_thread, + dtype_size, + cal_num_per_thread); + __credit_ld((uint16_t)(0x1u << thread_id), GGML_GTX_GELU_NOT_USE); + } + + bank_offset_shared += GGML_GTX_GELU_L2_BANK_OFFSET; + } + + bank_offset_shared = 0; + + for (uint8_t thread_id = 0; thread_id < spu_num_per_nest; thread_id++) { + const uint16_t cal_num_per_thread = (uint16_t) (thread_id < cal_mod ? (cal_quot + 1u) : cal_quot); + const uint16_t target = (uint16_t)(0x1u << thread_id); + const uint32_t offset_thread = + (((thread_id < cal_mod) ? + (uint32_t) thread_id * (cal_quot + 1u) : + (uint32_t) cal_mod * (cal_quot + 1u) + (uint32_t) (thread_id - cal_mod) * cal_quot) * + dtype_size); + + if (cal_num_per_thread > 0) { + __credit_chk(target); + __store(GGML_GTX_GELU_L2_RES + bank_offset_shared, + ddr_result + (uint64_t) base_elem * dtype_size + offset_thread, + cal_num_per_thread, + cal_num_per_thread, + dtype_size, + cal_num_per_thread); + __credit_st(target); + } + + bank_offset_shared += GGML_GTX_GELU_L2_BANK_OFFSET; + } + + __end_shared(); + + for (uint8_t thread_id = 0; thread_id < spu_num_per_nest; thread_id++) { + const uint16_t cal_num_per_thread = (uint16_t) (thread_id < cal_mod ? (cal_quot + 1u) : cal_quot); + + __start_thread(thread_id); + + if (cal_num_per_thread > 0) { + const uint32_t l2_input_thread = GGML_GTX_GELU_L2_INPUT + bank_offset_thread; + const uint32_t l2_res_thread = GGML_GTX_GELU_L2_RES + bank_offset_thread; + + __credit_chk(GGML_GTX_GELU_NOT_USE); + __load(l2_input_thread, + GGML_GTX_GELU_L1_AR, + cal_num_per_thread, + cal_num_per_thread, + dtype_size, + GGML_GTX_GELU_NOT_USE); + __credit_ld(GGML_GTX_GELU_NOT_USE, GGML_GTX_GELU_NOT_USE); + + __gelu(cal_num_per_thread); + + __store(GGML_GTX_GELU_L1_AR, + l2_res_thread, + cal_num_per_thread, + cal_num_per_thread, + dtype_size, + GGML_GTX_GELU_NOT_USE); + __credit_st(GGML_GTX_GELU_NOT_USE); + } + + bank_offset_thread += GGML_GTX_GELU_L2_BANK_OFFSET; + + __end_thread(thread_id); + } + + __end_plani(nest_id); + + __join(); + } +#else + (void) ddr_input; + (void) ddr_result; + (void) i_row; + (void) i_col; + (void) nest_num; + (void) spu_num_per_nest; +#endif +} + +int ggml_gtx_kernel_gelu_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_GELU_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_GELU_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + const uint32_t elems = desc->ne[0]; + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_L2_STASHED_N1 || + desc->nest_id >= GGML_GTX_NEST_NUM || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->elem_size != GGML_GTX_DTYPE_SIZE || + elems == 0 || + elems > UINT16_MAX) { + return -2; + } + + __split(); + __start_plan((uint8_t) desc->nest_id); + __set_spm_addr_A(GGML_GTX_GELU_L1_AR); + __set_spm_addr_R(GGML_GTX_GELU_L1_AR); + + for (uint8_t thread_id = 0; thread_id < GGML_GTX_SPU_NUM_PER_NEST; thread_id++) { + const uint32_t elems_this_tid = ggml_gtx_gelu_elems_for_tid(elems, thread_id); + + __start_thread(thread_id); + if (elems_this_tid > 0u) { + const uint32_t elem_start = ggml_gtx_gelu_start_for_tid(elems, thread_id); + const uint32_t max_l1_elems = (UINT16_MAX / GGML_GTX_DTYPE_SIZE) & ~15u; + uint32_t local = 0u; + + while (local < elems_this_tid) { + const uint32_t remaining = elems_this_tid - local; + const uint32_t chunk_elems = remaining < max_l1_elems ? remaining : max_l1_elems; + const uint32_t chunk_off = (elem_start + local) * GGML_GTX_DTYPE_SIZE; + const uint16_t chunk_bytes = (uint16_t) (chunk_elems * GGML_GTX_DTYPE_SIZE); + + __load((uint32_t) desc->l2_src_addr[0] + chunk_off, + GGML_GTX_GELU_L1_AR, + chunk_bytes, + chunk_bytes, + 1u, + chunk_bytes); + + __gelu(chunk_elems); + + __store(GGML_GTX_GELU_L1_AR, + (uint32_t) desc->l2_dst_addr + chunk_off, + chunk_bytes, + chunk_bytes, + 1u, + chunk_bytes); + local += chunk_elems; + } + } + __end_thread(thread_id); + } + __end_plan((uint8_t) desc->nest_id); + __join(); + + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_gelu_f16(arg); +} + +void ggml_gtx_printf(const char * fmt, ...) { + (void) fmt; +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-get-rows.c b/depend/ggml-gtx/src/ggml-gtx-kernel-get-rows.c new file mode 100644 index 0000000..d93deda --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-get-rows.c @@ -0,0 +1,133 @@ +//================================================================== +// GGML GTX kernel: GET_ROWS +//================================================================== + +#include + +#include "ggml-gtx-baremetal.h" +#include "ggml-gtx-compute.h" +#include "ggml-gtx-port.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#ifdef GGML_GTX_USE_GTX +#include "ggml-gtx/intrinsics/intrin.h" +#endif + +int ggml_gtx_kernel_get_rows_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_GET_ROWS_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_GET_ROWS_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + if (desc->dst_addr == 0 || + desc->src_addr[0] == 0 || + desc->src_addr[1] == 0) { + return -2; + } + + ggml_gtx_compute_get_rows(desc->src_addr[0], + desc->src_addr[1], + desc->dst_addr, + (uint16_t) desc->src_ne[0][0], + (uint16_t) desc->src_ne[0][1], + (uint16_t) desc->src_ne[1][0], + (uint16_t) desc->src_ne[1][1], + (uint16_t) desc->src_ne[1][2], + desc->src_type[1] == GGML_GTX_TYPE_I64 ? 8 : 4, + GGML_GTX_NEST_NUM, + GGML_GTX_SPU_NUM_PER_NEST); + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_get_rows_f16(arg); +} + +void ggml_gtx_printf(const char * fmt, ...) { + (void) fmt; +} + +static inline uint64_t ggml_gtx_read_rows_index(uint64_t ddr_indices, uint32_t idx_pos, uint16_t idx_size) { + if (idx_size == 8) { + volatile uint64_t * idx64 = (volatile uint64_t *)(uintptr_t)ddr_indices; + return idx64[idx_pos]; + } + + volatile uint32_t * idx32 = (volatile uint32_t *)(uintptr_t)ddr_indices; + return idx32[idx_pos]; +} + +void ggml_gtx_compute_get_rows( + uint64_t ddr_data, + uint64_t ddr_indices, + uint64_t ddr_dst, + uint16_t n_embd, + uint16_t n_vocab, + uint16_t n_rows, + uint16_t n_channels, + uint16_t n_batches, + uint16_t idx_size, + uint8_t nest_num, + uint8_t spu_num_per_nest) +{ + GGML_GTX_UNUSED(nest_num); + GGML_GTX_UNUSED(spu_num_per_nest); + + const uint32_t row_bytes = (uint32_t)n_embd * GGML_GTX_DTYPE_SIZE; + + for (uint16_t b = 0; b < n_batches; ++b) { + for (uint16_t ch = 0; ch < n_channels; ++ch) { + for (uint16_t r = 0; r < n_rows; ++r) { + const uint32_t idx_pos = ((uint32_t)b * n_channels + ch) * n_rows + r; + const uint32_t raw_word = *(const volatile uint32_t *)(uintptr_t)(ddr_indices + (uint64_t) idx_pos * idx_size); + const uint64_t row_idx = ggml_gtx_read_rows_index(ddr_indices, idx_pos, idx_size); + + if (idx_pos == 0) { + ggml_gtx_printf("[GET_ROWS] data=0x%08llx idx=0x%08llx dst=0x%08llx n_embd=%u n_vocab=%u n_rows=%u n_channels=%u n_batches=%u idx_size=%u raw=0x%08x row_idx=%llu\n", + (unsigned long long) ddr_data, + (unsigned long long) ddr_indices, + (unsigned long long) ddr_dst, + n_embd, n_vocab, n_rows, n_channels, n_batches, idx_size, + raw_word, + (unsigned long long) row_idx); + } + + if (row_idx >= n_vocab) { + ggml_gtx_printf("[GET_ROWS ERROR] row_idx=%llu out of range n_vocab=%u idx_pos=%u idx_addr=0x%08llx\n", + (unsigned long long) row_idx, + n_vocab, + idx_pos, + (unsigned long long) (ddr_indices + (uint64_t) idx_pos * idx_size)); + return; + } + + const uint64_t src_addr = ddr_data + + ((((uint64_t)b * n_channels + ch) * n_vocab + row_idx) * row_bytes); + const uint64_t dst_addr = ddr_dst + + ((((uint64_t)b * n_channels + ch) * n_rows + r) * row_bytes); + +#ifdef GGML_GTX_USE_GTX + __copy_mem(src_addr, + dst_addr, + row_bytes, + row_bytes, + 1, + row_bytes, + row_bytes >> 16); +#else + volatile uint16_t * src = (volatile uint16_t *)(uintptr_t)src_addr; + volatile uint16_t * dst = (volatile uint16_t *)(uintptr_t)dst_addr; + for (uint16_t c = 0; c < n_embd; ++c) { + dst[c] = src[c]; + } +#endif + } + } + } +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-im2col.c b/depend/ggml-gtx/src/ggml-gtx-kernel-im2col.c new file mode 100644 index 0000000..aab991e --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-im2col.c @@ -0,0 +1,643 @@ +//================================================================== +// GGML GTX kernel: IM2COL +//================================================================== + +#include +#include +#include + +#include "ggml-gtx-baremetal.h" +#include "ggml-gtx-compute.h" +#include "ggml-gtx-port.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#ifdef GGML_GTX_USE_GTX +#include "ggml-gtx-plan.h" +#include "ggml-gtx-config.h" +#include "ggml-gtx/intrinsics/intrin.h" +#endif + +int ggml_gtx_kernel_im2col_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_IM2COL_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_IM2COL_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + if (desc->dst_addr == 0 || + desc->src_addr[0] == 0 || + desc->src_addr[1] == 0) { + return -2; + } + + const bool is_2d = desc->op_params[6] == 1; + const uint16_t iw = (uint16_t) desc->src_ne[1][0]; + const uint16_t ih = is_2d ? (uint16_t) desc->src_ne[1][1] : 1; + const uint16_t ic = is_2d ? (uint16_t) desc->src_ne[1][2] : (uint16_t) desc->src_ne[1][1]; + const uint16_t n = is_2d ? (uint16_t) desc->src_ne[1][3] : (uint16_t) desc->src_ne[1][2]; + const uint16_t kw = (uint16_t) desc->src_ne[0][0]; + const uint16_t kh = is_2d ? (uint16_t) desc->src_ne[0][1] : 1; + + ggml_gtx_compute_im2col(desc->src_addr[1], + desc->dst_addr, + iw, + ih, + ic, + n, + kw, + kh, + (int16_t) desc->op_params[0], + (int16_t) desc->op_params[1], + (int16_t) desc->op_params[2], + (int16_t) desc->op_params[3], + (int16_t) desc->op_params[4], + (int16_t) desc->op_params[5], + is_2d); + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_im2col_f16(arg); +} + +//================================================================== + +#ifdef GGML_GTX_USE_GTX +#define GTX_IM2COL_FP16_B 2 +#define GTX_IM2COL_L1_4K 0x01000 +#define GTX_IM2COL_L1_END 0x60000 +#define GTX_IM2COL_SPU_PER_NEST GGML_GTX_SPU_NUM_PER_NEST +#define GTX_IM2COL_HALO_RATIO 4 + +#define GTX_IM2COL_MAX(a, b) (((a) > (b)) ? (a) : (b)) +#define GTX_IM2COL_MIN(a, b) (((a) < (b)) ? (a) : (b)) +#define GTX_IM2COL_FIND_INPUT_LENGTH(l_out, k_eff, stride) (((l_out) - 1) * (stride) + (k_eff)) + +typedef struct { + int c_in; + int h_in; + int w_in; + int h_out; + int w_out; + int h_in_pad; + int w_in_pad; + int k_size; + int k_n; + int k_eff; + int pad_h; + int pad_w; + int stride; + int dilation; +} gtx_im2col_params_t; + +typedef struct { + int num_w; + int num_h; + int num_c_in; + int len_w_out; + int len_h_out; + int len_w_in; + int len_h_in; + int len_c_in; +} gtx_im2col_tile_plan_t; + +typedef struct { + uint32_t input; + uint32_t result; +} gtx_im2col_l1_banks_t; + +static inline int gtx_im2col_ceil_div(int a, int b) { + return (a + b - 1) / b; +} + +static inline int gtx_im2col_ceil_rem(int a, int b) { + return (a + b - 1) % b + 1; +} + +static void gtx_im2col_derive_params(gtx_im2col_params_t * p) { + p->h_in_pad = p->h_in + p->pad_h * 2; + p->w_in_pad = p->w_in + p->pad_w * 2; + p->k_n = p->k_size * p->k_size; + p->k_eff = (p->k_size - 1) * p->dilation + 1; + p->h_out = (p->h_in_pad - p->k_eff) / p->stride + 1; + p->w_out = (p->w_in_pad - p->k_eff) / p->stride + 1; +} + +static void gtx_im2col_init_tile(gtx_im2col_tile_plan_t * t) { + t->num_c_in = 1; + t->num_w = 1; + t->num_h = 1; +} + +static void gtx_im2col_update_w_tile(int num_w, const gtx_im2col_params_t * p, gtx_im2col_tile_plan_t * t) { + t->num_w = num_w; + t->len_w_out = gtx_im2col_ceil_div(p->w_out, t->num_w); + t->len_w_in = GTX_IM2COL_FIND_INPUT_LENGTH(t->len_w_out, p->k_eff, p->stride); +} + +static void gtx_im2col_update_h_tile(int num_h, const gtx_im2col_params_t * p, gtx_im2col_tile_plan_t * t) { + t->num_h = num_h; + t->len_h_out = gtx_im2col_ceil_div(p->h_out, num_h); + t->len_h_in = GTX_IM2COL_FIND_INPUT_LENGTH(t->len_h_out, p->k_eff, p->stride); +} + +static void gtx_im2col_update_cin_tile(int num_cin, const gtx_im2col_params_t * p, gtx_im2col_tile_plan_t * t) { + t->num_c_in = num_cin; + t->len_c_in = gtx_im2col_ceil_div(p->c_in, t->num_c_in); +} + +static void gtx_im2col_update_tile(const gtx_im2col_params_t * p, gtx_im2col_tile_plan_t * t) { + t->len_w_out = gtx_im2col_ceil_div(p->w_out, t->num_w); + t->len_h_out = gtx_im2col_ceil_div(p->h_out, t->num_h); + t->len_c_in = gtx_im2col_ceil_div(p->c_in, t->num_c_in); + t->len_w_in = GTX_IM2COL_FIND_INPUT_LENGTH(t->len_w_out, p->k_eff, p->stride); + t->len_h_in = GTX_IM2COL_FIND_INPUT_LENGTH(t->len_h_out, p->k_eff, p->stride); +} + +static bool gtx_im2col_calculate_l1_usage(const gtx_im2col_params_t * p, const gtx_im2col_tile_plan_t * t, gtx_im2col_l1_banks_t * b) { + uint32_t input_bytes = (uint32_t)t->len_c_in * t->len_w_in * t->len_h_in * GTX_IM2COL_FP16_B; + uint64_t im2col_bytes = (uint64_t)t->len_c_in * p->k_n * t->len_h_out * t->len_w_out * GTX_IM2COL_FP16_B; + uint32_t ceil_input_bytes = (uint32_t)gtx_im2col_ceil_div((int)input_bytes, GTX_IM2COL_L1_4K) * GTX_IM2COL_L1_4K; + uint64_t ceil_im2col_bytes = (uint64_t)gtx_im2col_ceil_div((int)im2col_bytes, GTX_IM2COL_L1_4K) * GTX_IM2COL_L1_4K; + b->input = ceil_input_bytes; + b->result = (uint32_t)ceil_im2col_bytes; + return ceil_input_bytes + ceil_im2col_bytes < GTX_IM2COL_L1_END; +} + +static bool gtx_im2col_tile_cin(const gtx_im2col_params_t * p, gtx_im2col_tile_plan_t * t, gtx_im2col_l1_banks_t * b) { + gtx_im2col_init_tile(t); + gtx_im2col_update_tile(p, t); + + if (p->c_in < GTX_IM2COL_SPU_PER_NEST) { + gtx_im2col_update_cin_tile(p->c_in, p, t); + return gtx_im2col_calculate_l1_usage(p, t, b); + } + + int div_cin = GTX_IM2COL_SPU_PER_NEST; + while (t->len_c_in > 1) { + gtx_im2col_update_cin_tile(div_cin, p, t); + if (gtx_im2col_calculate_l1_usage(p, t, b)) return true; + div_cin = GTX_IM2COL_MIN(div_cin + GTX_IM2COL_SPU_PER_NEST, p->c_in); + } + + return false; +} + +static bool gtx_im2col_tile_wh(const gtx_im2col_params_t * p, gtx_im2col_tile_plan_t * t, gtx_im2col_l1_banks_t * b) { + uint64_t block_bytes = (uint64_t)b->input + b->result; + int div_estimate = gtx_im2col_ceil_div((int)block_bytes, GTX_IM2COL_L1_END); + int div_h = div_estimate > 8 ? 8 : div_estimate; + int halo = p->k_eff - p->stride; + if (halo <= 0) halo = 1; + + int max_h_div = GTX_IM2COL_MAX(p->h_in / p->k_eff, p->h_in / (halo * GTX_IM2COL_HALO_RATIO)); + while (div_h < max_h_div) { + gtx_im2col_update_h_tile(div_h, p, t); + if (gtx_im2col_calculate_l1_usage(p, t, b)) return true; + div_h++; + } + + int div_w = 2; + int max_w_div = GTX_IM2COL_MAX(p->w_in / p->k_eff, p->w_in / (halo * GTX_IM2COL_HALO_RATIO)); + while (div_w < max_w_div) { + gtx_im2col_update_w_tile(div_w, p, t); + if (gtx_im2col_calculate_l1_usage(p, t, b)) return true; + div_w++; + } + + return false; +} + +static uint64_t gtx_im2col_kernel_tile_cin(uint8_t nest_id, uint64_t ddr_input, uint64_t ddr_result, + const gtx_im2col_params_t * p, + const gtx_im2col_tile_plan_t * t, + const gtx_im2col_l1_banks_t * b) { + uint32_t l2_input = 0x0c00000; + uint32_t l2_result = 0x0; + int l2_input_cap = 0x500000; + int l2_result_cap = 0x1100000; + + uint32_t bank_a = GGML_GTX_L1_BANK_A_OFFSET; + uint32_t bank_r = GTX_IM2COL_L1_END - b->input; + + __start_plan(nest_id); + for (int spu_id = 0; spu_id < GTX_IM2COL_SPU_PER_NEST; spu_id++) { + __start_thread(spu_id); + __set_spm_addr(bank_r, 0x0, 0x0, bank_a); + __end_thread(spu_id); + } + __end_plan(nest_id); + + int len_c_in_extra = gtx_im2col_ceil_rem(p->c_in, t->len_c_in); + + uint32_t input_bytes_row = (uint32_t)p->w_in * GTX_IM2COL_FP16_B; + uint32_t input_bytes_channel = (uint32_t)p->h_in * input_bytes_row; + uint32_t input_ddr_stride = (uint32_t)t->len_c_in * input_bytes_channel; + uint32_t input_ddr_offset = 0; + + uint32_t input_bytes_row_pad = (uint32_t)p->w_in_pad * GTX_IM2COL_FP16_B; + uint32_t input_l2_stride = (uint32_t)t->len_c_in * p->h_in_pad * p->w_in_pad * GTX_IM2COL_FP16_B; + uint32_t input_l2_offset = 0; + + uint32_t output_height = (uint32_t)p->h_out * p->w_out; + uint32_t output_length = (uint32_t)t->len_c_in * p->k_n * GTX_IM2COL_FP16_B; + uint32_t output_stride = output_height * output_length; + uint32_t output_l2_offset = 0; + uint32_t output_ddr_offset = 0; + + for (int tile_id = 0; tile_id < t->num_c_in; tile_id++) { + int spu_id = tile_id % GTX_IM2COL_SPU_PER_NEST; + uint64_t spu_mask = (uint64_t)0x1 << spu_id; + + int len_c_in = (tile_id == t->num_c_in - 1) ? len_c_in_extra : t->len_c_in; + uint32_t input_l1_read_height = (uint32_t)len_c_in * p->h_in_pad; + uint32_t output_row = (uint32_t)len_c_in * p->k_n; + uint32_t output_bytes_row = output_row * GTX_IM2COL_FP16_B; + + __start_plan(nest_id); + __start_shared(); + __pad(ddr_input + input_ddr_offset, l2_input + input_l2_offset, + input_bytes_row, input_bytes_row, p->h_in, + (uint8_t)p->pad_h, (uint8_t)p->pad_h, (uint8_t)p->pad_w, (uint8_t)p->pad_w, + 0x0, input_bytes_channel, len_c_in, 0b10, 0x1); + __credit_ld(spu_mask, 0); + + __credit_chk(spu_mask); + __tpose(l2_result + output_l2_offset, ddr_result + output_ddr_offset, output_row, + p->w_out, p->h_out, 1, 0, 2, 0b01, 1); + __credit_st(spu_mask); + __end_shared(); + + __start_thread(spu_id); + __credit_chk(0xDEAD); + __load(l2_input + input_l2_offset, bank_r, + input_bytes_row_pad, input_bytes_row_pad, input_l1_read_height, input_bytes_row_pad); + __credit_ld(spu_mask, 0); + __im2col_n(p->h_in_pad, p->w_in_pad, p->k_size, p->dilation, p->stride, len_c_in); + __store(bank_a, l2_result + output_l2_offset, + output_bytes_row, output_bytes_row, output_height, output_bytes_row); + __credit_st(spu_mask); + __end_thread(spu_id); + __end_plan(nest_id); + + input_ddr_offset += input_ddr_stride; + output_ddr_offset += output_stride; + input_l2_offset = input_l2_offset < (uint32_t)l2_input_cap ? input_l2_offset + input_l2_stride : 0; + output_l2_offset = output_l2_offset < (uint32_t)l2_result_cap ? output_l2_offset + output_stride : 0; + } + + return 0; +} + +static uint64_t gtx_im2col_kernel_tile_h(uint8_t nest_id, uint64_t ddr_input, uint64_t ddr_result, + const gtx_im2col_params_t * p, + const gtx_im2col_tile_plan_t * t, + const gtx_im2col_l1_banks_t * b) { + uint32_t l2_result = 0x0; + uint32_t l2_tpose = 0x0600000; + uint32_t l2_input = 0x0c00000; + uint32_t l2_input_cap = 0x04A0000; + uint32_t l2_result_cap = 0x08A0000; + + uint32_t bank_a = GGML_GTX_L1_BANK_A_OFFSET; + uint32_t bank_r = GTX_IM2COL_L1_END - b->input; + + int len_c_in_extra = gtx_im2col_ceil_rem(p->c_in, t->len_c_in); + int len_h_out_extra = gtx_im2col_ceil_rem(p->h_out, t->len_h_out); + int len_h_in_extra = GTX_IM2COL_FIND_INPUT_LENGTH(len_h_out_extra, p->k_eff, p->stride); + + uint32_t input_ddr_row = (uint32_t)p->w_in * GTX_IM2COL_FP16_B; + uint32_t input_ddr_height_stride = (uint32_t)t->len_h_out * p->stride * input_ddr_row; + uint32_t input_ddr_channel = (uint32_t)p->h_in * input_ddr_row; + uint32_t input_ddr_channel_stride = (uint32_t)t->len_c_in * input_ddr_channel; + + uint32_t input_l2_row = (uint32_t)p->w_in_pad * GTX_IM2COL_FP16_B; + uint32_t input_l2_height_stride = (uint32_t)t->len_h_in * input_l2_row; + uint32_t input_l2_channel_stride = (uint32_t)t->len_c_in * p->h_in_pad * p->w_in_pad * GTX_IM2COL_FP16_B; + + uint32_t output_l2_width_stride = (uint32_t)t->len_c_in * p->k_n * GTX_IM2COL_FP16_B; + uint32_t output_l2_height_stride = (uint32_t)t->len_h_out * p->w_out * output_l2_width_stride; + + uint32_t output_ddr_width = (uint32_t)p->w_out * GTX_IM2COL_FP16_B; + uint32_t output_ddr_row = (uint32_t)p->h_out * output_ddr_width; + uint32_t output_ddr_height_stride = (uint32_t)t->len_h_out * output_ddr_width; + uint64_t output_ddr_channel_stride = (uint64_t)t->len_c_in * p->k_n * output_ddr_row; + + for (int c_id = 0; c_id < t->num_c_in; c_id++) { + int len_c_in = (c_id < t->num_c_in - 1) ? t->len_c_in : len_c_in_extra; + int t_cin_kn = len_c_in * p->k_n; + + for (int h_id = 0; h_id < t->num_h; h_id++) { + int tile_id = c_id * t->num_h + h_id; + int spu_id = tile_id % GTX_IM2COL_SPU_PER_NEST; + uint16_t spu_mask = (uint16_t)(0x1 << spu_id); + + int len_h_in = t->len_h_in; + int len_h_out = t->len_h_out; + int len_h_l2_load = len_h_in; + int pad_h_ddr_offset = p->pad_h * (int)input_ddr_row; + int pad_u = 0; + int pad_d = 0; + + if (h_id == 0) { + pad_u = p->pad_h; + len_h_l2_load = len_h_in - pad_u; + pad_h_ddr_offset = 0; + } else if (h_id == t->num_h - 1) { + len_h_out = len_h_out_extra; + len_h_in = len_h_in_extra; + pad_d = p->pad_h; + len_h_l2_load = len_h_in - pad_d; + } + + uint64_t input_ddr_offset = (uint64_t)c_id * input_ddr_channel_stride + + (uint64_t)h_id * input_ddr_height_stride - + (uint64_t)pad_h_ddr_offset; + uint32_t input_l2_offset = (c_id * input_l2_channel_stride + h_id * input_l2_height_stride) % l2_input_cap; + uint32_t output_l2_offset = (h_id * output_l2_height_stride + c_id * output_l2_width_stride) % l2_result_cap; + uint64_t output_ddr_offset = (uint64_t)c_id * output_ddr_channel_stride + (uint64_t)h_id * output_ddr_height_stride; + uint32_t output_ddr_length = output_ddr_width * (uint32_t)len_h_out; + + uint32_t output_l2_width = (uint32_t)t_cin_kn * GTX_IM2COL_FP16_B; + uint32_t output_l2_height = (uint32_t)len_h_out * p->w_out; + uint32_t input_l1_height = (uint32_t)len_c_in * len_h_in; + + __split(); + __start_plan(nest_id); + __start_shared(); + __pad(ddr_input + input_ddr_offset, l2_input + input_l2_offset, + input_ddr_row, input_ddr_row, len_h_l2_load, + (uint8_t)pad_u, (uint8_t)pad_d, (uint8_t)p->pad_w, (uint8_t)p->pad_w, + 0x0, input_ddr_channel, len_c_in, 0b10, 0x1); + __credit_ld(spu_mask, 0); + + __credit_chk(spu_mask); + __tpose(l2_result + output_l2_offset, l2_tpose + output_l2_offset, + t_cin_kn, p->w_out, len_h_out, 1, 0, 2, 0b00, 1); + __store(l2_tpose + output_l2_offset, ddr_result + output_ddr_offset, + output_ddr_row, output_ddr_length, t_cin_kn, output_ddr_length); + __credit_st(spu_mask); + __end_shared(); + + __start_thread(spu_id); + __set_spm_addr(bank_r, 0x0, 0x0, bank_a); + __credit_chk(0xDEAD); + __load(l2_input + input_l2_offset, bank_r, + input_l2_row, input_l2_row, input_l1_height, input_l2_row); + __credit_ld(spu_mask, 0); + __im2col_n(len_h_in, p->w_in_pad, p->k_size, p->dilation, p->stride, len_c_in); + __store(bank_a, l2_result + output_l2_offset, + output_l2_width, output_l2_width, output_l2_height, output_l2_width); + __credit_st(spu_mask); + __end_thread(spu_id); + __end_plan(nest_id); + __join(); + } + } + + return 0; +} + +static void gtx_im2col_kernel_1d(uint8_t nest_id, uint64_t ddr_input, uint64_t ddr_result, + uint16_t ch, uint16_t w_in, uint16_t pad_w, + uint16_t k_size, uint16_t stride, + uint16_t dilation, uint16_t n_seq) { + const uint32_t l2_result = 0x0; + const uint32_t l2_input = 0x800000; + + const int w_in_pad = (int)w_in + (int)pad_w * 2; + const int k_eff = ((int)k_size - 1) * (int)dilation + 1; + if (w_in_pad < k_eff || stride == 0) { + return; + } + + const uint32_t pad_offset = (uint32_t)pad_w * GTX_IM2COL_FP16_B; + const uint32_t in_length = (uint32_t)w_in * GTX_IM2COL_FP16_B; + const uint32_t padded_length = (uint32_t)w_in_pad * GTX_IM2COL_FP16_B; + const uint32_t w_out = (uint32_t)((w_in_pad - k_eff) / (int)stride + 1); + + const uint32_t stride_bytes = (uint32_t)stride * GTX_IM2COL_FP16_B; + const uint32_t dilation_bytes = (uint32_t)dilation * GTX_IM2COL_FP16_B; + const uint32_t kernel_bytes = (uint32_t)k_size * GTX_IM2COL_FP16_B; + const uint32_t channel_kernel_bytes = (uint32_t)ch * kernel_bytes; + const uint32_t input_seq_bytes = in_length * (uint32_t)ch; + const uint32_t padded_seq_bytes = padded_length * (uint32_t)ch; + const uint32_t output_seq_bytes = w_out * channel_kernel_bytes; + + __split(); + __start_plan(nest_id); + __start_shared(); + __fill(l2_input, padded_length, padded_length, (uint32_t)ch * n_seq, 0x0, 0b0); + __load(ddr_input, l2_input + pad_offset, + in_length, in_length, (uint32_t)ch * n_seq, padded_length); + + for (uint16_t sid = 0; sid < n_seq; sid++) { + uint32_t base_src = l2_input + (uint32_t)sid * padded_seq_bytes; + uint32_t base_dst = l2_result + (uint32_t)sid * output_seq_bytes; + for (uint32_t ow = 0; ow < w_out; ow++) { + uint32_t base_src_w = base_src + ow * stride_bytes; + uint32_t base_dst_w = base_dst + ow * channel_kernel_bytes; + for (uint16_t cid = 0; cid < ch; cid++) { + uint32_t addr_src = base_src_w + (uint32_t)cid * padded_length; + uint32_t addr_dst = base_dst_w + (uint32_t)cid * kernel_bytes; + __copy(addr_src, addr_dst, + dilation_bytes, GTX_IM2COL_FP16_B, k_size, GTX_IM2COL_FP16_B); + } + } + } + + __store(l2_result, ddr_result, + channel_kernel_bytes, channel_kernel_bytes, + w_out * (uint32_t)n_seq, channel_kernel_bytes); + __end_shared(); + __end_plan(nest_id); + __join(); +} +#endif + +void ggml_gtx_compute_im2col( + uint64_t ddr_src, // input data (F32) + uint64_t ddr_dst, // output (FP16) + uint16_t IW, + uint16_t IH, + uint16_t IC, + uint16_t N, // batch size + uint16_t KW, + uint16_t KH, + int16_t s0, // stride W + int16_t s1, // stride H + int16_t p0, // padding W + int16_t p1, // padding H + int16_t d0, // dilation W + int16_t d1, // dilation H + bool is_2D) +{ +#ifdef GGML_GTX_USE_GTX + if (s0 <= 0 || p0 < 0 || d0 <= 0) { + return; + } + + if (is_2D && (KH != KW || s1 != s0 || p1 != p0 || d1 != d0)) { + return; + } + + if (!is_2D) { + gtx_im2col_kernel_1d(0, ddr_src, ddr_dst, + IC, IW, (uint16_t)p0, KW, (uint16_t)s0, (uint16_t)d0, N); + return; + } + + const uint16_t OW = (uint16_t)((IW + 2 * p0 - d0 * (KW - 1) - 1) / s0 + 1); + const uint16_t OH = (uint16_t)((IH + 2 * p1 - d1 * (KH - 1) - 1) / s1 + 1); + volatile const uint16_t * src = (volatile const uint16_t *)(uintptr_t)ddr_src; + volatile uint16_t * dst = (volatile uint16_t *)(uintptr_t)ddr_dst; + const uint32_t dst_ne0 = (uint32_t)IC * KH * KW; + + for (uint16_t in = 0; in < N; in++) { + for (uint16_t ioh = 0; ioh < OH; ioh++) { + for (uint16_t iow = 0; iow < OW; iow++) { + const uint32_t dst_col = (uint32_t)in * OH * OW + (uint32_t)ioh * OW + iow; + + for (uint16_t iic = 0; iic < IC; iic++) { + for (uint16_t ikh = 0; ikh < KH; ikh++) { + for (uint16_t ikw = 0; ikw < KW; ikw++) { + const int32_t iiw = (int32_t)iow * s0 + (int32_t)ikw * d0 - p0; + const int32_t iih = (int32_t)ioh * s1 + (int32_t)ikh * d1 - p1; + + uint16_t fp16_val = 0; + if (iiw >= 0 && iiw < IW && iih >= 0 && iih < IH) { + const uint64_t src_idx = (uint64_t)in * IC * IH * IW + + (uint64_t)iic * IH * IW + + (uint64_t)iih * IW + + (uint64_t)iiw; + fp16_val = src[src_idx]; + } + + const uint32_t dst_row = (uint32_t)iic * KH * KW + (uint32_t)ikh * KW + ikw; + dst[dst_col * dst_ne0 + dst_row] = fp16_val; + } + } + } + } + } + } + return; + + gtx_im2col_params_t p = { + .c_in = IC, + .h_in = IH, + .w_in = IW, + .pad_h = p1, + .pad_w = p0, + .k_size = KW, + .stride = s0, + .dilation = d0, + }; + gtx_im2col_derive_params(&p); + if (p.h_out <= 0 || p.w_out <= 0) { + return; + } + + gtx_im2col_tile_plan_t t; + gtx_im2col_l1_banks_t b; + bool use_tile_cin = gtx_im2col_tile_cin(&p, &t, &b); + bool use_tile_h = false; + + if (!use_tile_cin) { + use_tile_h = gtx_im2col_tile_wh(&p, &t, &b); + } + + if (!use_tile_cin && !use_tile_h) { + return; + } + + uint64_t src_batch_bytes = (uint64_t)p.c_in * p.h_in * p.w_in * GTX_IM2COL_FP16_B; + uint64_t dst_batch_bytes = (uint64_t)p.c_in * p.k_n * p.h_out * p.w_out * GTX_IM2COL_FP16_B; + + if (use_tile_cin) { + __split(); + } + + for (uint16_t batch = 0; batch < N; ++batch) { + uint64_t src_batch = ddr_src + (uint64_t)batch * src_batch_bytes; + uint64_t dst_batch = ddr_dst + (uint64_t)batch * dst_batch_bytes; + + if (use_tile_cin) { + gtx_im2col_kernel_tile_cin(0, src_batch, dst_batch, &p, &t, &b); + } else { + gtx_im2col_kernel_tile_h(0, src_batch, dst_batch, &p, &t, &b); + } + } + + if (use_tile_cin) { + __join(); + } +#else + // Output dimensions + uint16_t OW = (IW + 2 * p0 - d0 * (KW - 1) - 1) / s0 + 1; + uint16_t OH = is_2D ? (IH + 2 * p1 - d1 * (KH - 1) - 1) / s1 + 1 : 1; + + volatile uint32_t * src = (volatile uint32_t *)(uintptr_t)ddr_src; + volatile uint16_t * dst = (volatile uint16_t *)(uintptr_t)ddr_dst; + + // Output layout: [IC*KH*KW, OW*OH, N] + // dst index: [batch_in * OH * OW + ioh * OW + iow] * (IC*KH*KW) + (iic*KH*KW + ikh*KW + ikw) + uint32_t dst_ne0 = (uint32_t)IC * KH * KW; + + for (uint16_t in = 0; in < N; in++) { + for (uint16_t ioh = 0; ioh < OH; ioh++) { + for (uint16_t iow = 0; iow < OW; iow++) { + uint32_t dst_col = (uint32_t)in * OH * OW + ioh * OW + iow; + + for (uint16_t iic = 0; iic < IC; iic++) { + for (uint16_t ikh = 0; ikh < KH; ikh++) { + for (uint16_t ikw = 0; ikw < KW; ikw++) { + int32_t iiw = (int32_t)iow * s0 + (int32_t)ikw * d0 - p0; + int32_t iih = is_2D ? ((int32_t)ioh * s1 + (int32_t)ikh * d1 - p1) : 0; + + uint16_t fp16_val = 0; // zero-padding + if (iiw >= 0 && iiw < IW && iih >= 0 && iih < IH) { + // src index: [in][iic][iih][iiw] — layout [IW, IH, IC, N] + uint64_t src_idx; + if (is_2D) { + src_idx = (uint64_t)in * IC * IH * IW + + (uint64_t)iic * IH * IW + + (uint64_t)iih * IW + + iiw; + } else { + src_idx = (uint64_t)in * IC * IW + + (uint64_t)iic * IW + + iiw; + } + // F32 → FP16 conversion + fp16_val = ggml_gtx_fp32_bits_to_fp16(src[src_idx]); + } + + uint32_t dst_row = (uint32_t)iic * KH * KW + ikh * KW + ikw; + dst[dst_col * dst_ne0 + dst_row] = fp16_val; + } + } + } + } + } + } +#endif +} + +//================================================================== +// POOL_2D: 2D max or average pooling +// +// Input: F32 [W, H, C, N] +// Output: F32 [OW, OH, C, N] +// +// pool_type: 0 = MAX, 1 = AVG +//================================================================== + +// Minimal F32 bit representation of -FLT_MAX +// IEEE 754: 0xFF7FFFFF = -3.4028235e+38 +#define F32_NEG_MAX_BITS 0xFF7FFFFFu diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-leaky-relu.c b/depend/ggml-gtx/src/ggml-gtx-kernel-leaky-relu.c new file mode 100644 index 0000000..cabb4fe --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-leaky-relu.c @@ -0,0 +1,101 @@ +//================================================================== +// GGML GTX kernel: LEAKY_RELU +// +// Descriptor wrapper around reference/leaky_relu.c. +// dst[i] = src0[i] >= 0 ? src0[i] : negative_slope * src0[i] +//================================================================== + +#include + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#define BANK_A GGML_GTX_L1_BANK_A_OFFSET +#define BANK_B GGML_GTX_L1_BANK_B_OFFSET +#define BANK_C GGML_GTX_L1_BANK_C_OFFSET +#define BANK_R GGML_GTX_L1_BANK_R_OFFSET +#define DTYPE 2u +#define SPU_NUM_PER_NEST GGML_GTX_SPU_NUM_PER_NEST +#define NOT_USE 0xBEEFu +#define FP16_ZERO 0x0000u + +static inline uint32_t elems_for_tid(uint32_t total_elems, uint8_t tid) { + const uint32_t q = total_elems / SPU_NUM_PER_NEST; + const uint32_t r = total_elems % SPU_NUM_PER_NEST; + return tid < r ? q + 1u : q; +} + +static inline uint32_t start_for_tid(uint32_t total_elems, uint8_t tid) { + const uint32_t q = total_elems / SPU_NUM_PER_NEST; + const uint32_t r = total_elems % SPU_NUM_PER_NEST; + if (tid < r) { + return (uint32_t) tid * (q + 1u); + } + return r * (q + 1u) + ((uint32_t) tid - r) * q; +} + +int ggml_gtx_kernel_leaky_relu_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_LEAKY_RELU_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_LEAKY_RELU_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + const uint32_t total_elems = desc->ne[0]; + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_L2_STASHED_N1 || + desc->nest_id >= GGML_GTX_NEST_NUM || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->elem_size != DTYPE || + total_elems == 0 || + total_elems > UINT16_MAX) { + return -2; + } + + const uint16_t negative_slope = ggml_gtx_fp32_bits_to_fp16((uint32_t) desc->op_params[0]); + + __split(); + __start_plan((uint8_t) desc->nest_id); + for (uint8_t tid = 0; tid < SPU_NUM_PER_NEST; tid++) { + const uint32_t elems_this_tid = elems_for_tid(total_elems, tid); + __start_thread(tid); + if (elems_this_tid > 0u) { + const uint32_t elem_start = start_for_tid(total_elems, tid); + const uint32_t chunk_off = elem_start * DTYPE; + const uint32_t chunk_bytes = elems_this_tid * DTYPE; + const uint16_t chunk_bytes_u16 = (uint16_t) chunk_bytes; + + __load((uint32_t) desc->l2_src_addr[0] + chunk_off, BANK_A, + chunk_bytes, chunk_bytes_u16, 1u, chunk_bytes_u16); + + __set_spm_addr(BANK_C, BANK_C, BANK_B, BANK_A); + __clamp_min(elems_this_tid, FP16_ZERO, 0u); + + __set_spm_addr(BANK_R, BANK_C, BANK_B, BANK_A); + __clamp_max(elems_this_tid, FP16_ZERO, 0u); + + __set_spm_addr(BANK_R, BANK_C, BANK_B, BANK_R); + __mul_vs(elems_this_tid, negative_slope, 0u); + + __set_spm_addr(BANK_R, BANK_C, BANK_C, BANK_R); + __add_vv(elems_this_tid); + + __store(BANK_R, (uint32_t) desc->l2_dst_addr + chunk_off, + chunk_bytes, chunk_bytes_u16, 1u, chunk_bytes_u16); + } + __end_thread(tid); + } + __end_plan((uint8_t) desc->nest_id); + __join(); + + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_leaky_relu_f16(arg); +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-mul-mat.c b/depend/ggml-gtx/src/ggml-gtx-kernel-mul-mat.c new file mode 100644 index 0000000..1bdf44a --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-mul-mat.c @@ -0,0 +1,265 @@ +//================================================================== +// GGML GTX kernel: MUL_MAT, L2-stashed single-nest tile +//================================================================== + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-kernel-args.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#define GGML_GTX_MUL_MAT_L1_A_BYTES 0x20000u +#define GGML_GTX_MUL_MAT_L1_B_BYTES 0x10000u +#define GGML_GTX_MUL_MAT_L1_R_BYTES 0x10000u +#define GGML_GTX_MUL_MAT_TILES_PER_PLAN 8u + +static void ggml_gtx_compute_matmul_l2_n1( + uint32_t l2_a, + uint32_t l2_b, + uint32_t l2_c, + uint16_t a_row, + uint16_t h_col, + uint8_t nest_id, + uint8_t spu_num_per_nest) +{ +#ifdef GGML_GTX_USE_GTX + const uint32_t dtype_size = GGML_GTX_DTYPE_SIZE; + const uint16_t not_use = 0xBEEF; + const uint16_t byte_h_col = (uint16_t) (h_col * dtype_size); + uint16_t rows_per_l1 = (uint16_t) (GGML_GTX_MUL_MAT_L1_A_BYTES / byte_h_col); + if (rows_per_l1 == 0u) { + rows_per_l1 = 1u; + } + + __split(); + __start_plan(nest_id); + __start_shared(); + __end_shared(); + + /* + * The active FPGA is n1s1. Keep several full 64 KiB A tiles in + * one plan and retain B in L1 across them. This preserves the + * verified N=1 result layout while avoiding one split/join and + * one B load per M tile. + */ + __start_thread(0u); + __set_spm_addr(GGML_GTX_L1_BANK_R_OFFSET, + GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, + GGML_GTX_L1_BANK_A_OFFSET); + + __load( + l2_b, + GGML_GTX_L1_BANK_B_OFFSET, + byte_h_col, + byte_h_col, + 1, + not_use + ); + + for (uint16_t m0 = 0u; m0 < a_row;) { + uint16_t rows = (uint16_t) (a_row - m0); + if (rows > rows_per_l1) { + rows = rows_per_l1; + } + const uint32_t a_off = (uint32_t)m0 * byte_h_col; + const uint32_t c_off = (uint32_t)m0 * dtype_size; + + __load( + l2_a + a_off, + GGML_GTX_L1_BANK_A_OFFSET, + byte_h_col, + byte_h_col, + rows, + not_use + ); + __mm(rows, h_col, 1); + __store( + GGML_GTX_L1_BANK_R_OFFSET, + l2_c + c_off, + dtype_size, + dtype_size, + rows, + not_use + ); + m0 = (uint16_t) (m0 + rows); + } + __end_thread(0u); + __end_plan(nest_id); + __join(); +#else + (void) l2_a; + (void) l2_b; + (void) l2_c; + (void) a_row; + (void) h_col; + (void) nest_id; + (void) spu_num_per_nest; +#endif +} + +static int ggml_gtx_compute_matmul_l2_block( + uint32_t l2_a, + uint32_t l2_b, + uint32_t l2_c, + uint16_t a_row, + uint16_t h_col, + uint16_t b_row, + uint8_t nest_id, + uint8_t spu_num_per_nest) +{ + const uint32_t dtype_size = GGML_GTX_DTYPE_SIZE; + const uint32_t k_bytes = (uint32_t)h_col * dtype_size; + if (a_row == 0u || h_col == 0u || b_row == 0u || + k_bytes == 0u || k_bytes > UINT16_MAX || + k_bytes > GGML_GTX_MUL_MAT_L1_B_BYTES) { + return -1; + } + + // Preserve the established decode/GEMV path exactly. Large-N vision + // workloads use the block path below. + if (b_row == 1u) { + ggml_gtx_compute_matmul_l2_n1( + l2_a, l2_b, l2_c, a_row, h_col, nest_id, spu_num_per_nest); + return 0; + } + + // L1 A, B, and R are separate address ranges. GGML_GTX_L1_BANK_SIZE is + // the size of the complete L1 address space, not the capacity of each + // bank: A is 128 KiB while B and R are 64 KiB. Using the total 384 KiB + // value here made a K=128, N=769 B load cross from B [0x20000,0x30000) + // through C and into R at 0x50000; N=768 ended exactly at the R boundary. + // Tile against the actual bank capacities instead. + // Choose a KxN B + // block first, then the largest M block that fits both A (MxK) and the + // transposed result (NxM). B is stored as N contiguous K-vectors, which + // is the native GTX ntn input contract; __mm_t emits N contiguous + // M-vectors, matching ggml's ne[0]-contiguous destination layout. + uint32_t n_block = GGML_GTX_MUL_MAT_L1_B_BYTES / k_bytes; + if (n_block > b_row) { + n_block = b_row; + } + if (n_block == 0u || n_block > UINT16_MAX) { + return -1; + } + + uint32_t m_block = GGML_GTX_MUL_MAT_L1_A_BYTES / k_bytes; + const uint32_t m_by_result = GGML_GTX_MUL_MAT_L1_R_BYTES / + (n_block * dtype_size); + if (m_block > m_by_result) { + m_block = m_by_result; + } + if (m_block > a_row) { + m_block = a_row; + } + // __store length is uint16 bytes. + if (m_block > UINT16_MAX / dtype_size) { + m_block = UINT16_MAX / dtype_size; + } + if (m_block == 0u) { + return -1; + } + + (void)spu_num_per_nest; + for (uint32_t m0 = 0u; m0 < a_row; m0 += m_block) { + const uint16_t cur_m = (uint16_t) + (((uint32_t)a_row - m0) < m_block ? (uint32_t)a_row - m0 : m_block); + + __split(); + __start_plan(nest_id); + __start_thread(0u); + __set_spm_addr(GGML_GTX_L1_BANK_R_OFFSET, + GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, + GGML_GTX_L1_BANK_A_OFFSET); + + // Reuse this A block for every N block in the descriptor. + __load(l2_a + m0 * k_bytes, + GGML_GTX_L1_BANK_A_OFFSET, + k_bytes, + (uint16_t)k_bytes, + cur_m, + k_bytes); + + for (uint32_t n0 = 0u; n0 < b_row; n0 += n_block) { + const uint16_t cur_n = (uint16_t) + (((uint32_t)b_row - n0) < n_block ? + (uint32_t)b_row - n0 : n_block); + + __load(l2_b + n0 * k_bytes, + GGML_GTX_L1_BANK_B_OFFSET, + k_bytes, + (uint16_t)k_bytes, + cur_n, + k_bytes); + + __mm_t(cur_m, h_col, cur_n); + + __store(GGML_GTX_L1_BANK_R_OFFSET, + l2_c + (n0 * (uint32_t)a_row + m0) * dtype_size, + (uint32_t)a_row * dtype_size, + (uint16_t)((uint32_t)cur_m * dtype_size), + cur_n, + (uint32_t)cur_m * dtype_size); + } + __end_thread(0u); + __end_plan(nest_id); + __join(); + } + + return 0; +} + +int ggml_gtx_kernel_mul_mat_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_MUL_MAT_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_MUL_MAT_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_L2_STASHED_N1 || + desc->nest_id >= GGML_GTX_NEST_NUM) { + return -2; + } + + const ggml_gtx_mul_mat_args_t * params = + (const ggml_gtx_mul_mat_args_t *)ggml_gtx_kernel_args( + karg, sizeof(ggml_gtx_mul_mat_args_t)); + if (params == 0 || + params->version != GGML_GTX_MUL_MAT_ARGS_VERSION || + params->size < sizeof(*params)) { + return -3; + } + + const uint32_t h_col = params->k; + const uint32_t a_row = params->m; + const uint32_t b_row = params->n; + if (h_col == 0 || a_row == 0 || b_row == 0 || + h_col != desc->src_ne[0][0] || + a_row != desc->src_ne[0][1] || + b_row != desc->src_ne[1][1] || + h_col > UINT16_MAX || a_row > UINT16_MAX || b_row > UINT16_MAX) { + return -4; + } + + if (ggml_gtx_compute_matmul_l2_block( + (uint32_t) desc->l2_src_addr[0], + (uint32_t) desc->l2_src_addr[1], + (uint32_t) desc->l2_dst_addr, + (uint16_t) a_row, + (uint16_t) h_col, + (uint16_t) b_row, + (uint8_t) desc->nest_id, + GGML_GTX_SPU_NUM_PER_NEST + ) != 0) { + return -5; + } + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_mul_mat_f16(arg); +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-mul.c b/depend/ggml-gtx/src/ggml-gtx-kernel-mul.c new file mode 100644 index 0000000..58b3fb8 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-mul.c @@ -0,0 +1,212 @@ +//================================================================== +// GGML GTX kernel: MUL, L2-stashed single-nest tile +//================================================================== + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-kernel-args.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#define GGML_GTX_BINARY_L1_MAX_ELEMS ((64u * 1024u - 2u) / GGML_GTX_DTYPE_SIZE) + +static uint32_t ggml_gtx_binary_rows(const ggml_gtx_kernel_launch_desc_t * desc) { + return desc->ne[1] * desc->ne[2] * desc->ne[3]; +} + +static uint32_t ggml_gtx_binary_row_for_tid(uint32_t rows, uint32_t tid) { + return (uint32_t) (((uint64_t) rows * tid) / GGML_GTX_SPU_NUM_PER_NEST); +} + +static uint32_t ggml_gtx_binary_rhs_row(const ggml_gtx_kernel_launch_desc_t * desc, + uint32_t row) { + const uint32_t i1 = row % desc->ne[1]; + const uint32_t q = row / desc->ne[1]; + const uint32_t i2 = q % desc->ne[2]; + const uint32_t i3 = q / desc->ne[2]; + const uint32_t j1 = desc->src_ne[1][1] == 1u ? 0u : i1; + const uint32_t j2 = desc->src_ne[1][2] == 1u ? 0u : i2; + const uint32_t j3 = desc->src_ne[1][3] == 1u ? 0u : i3; + return (j3 * desc->src_ne[1][2] + j2) * desc->src_ne[1][1] + j1; +} + +static void ggml_gtx_compute_mul_vv_l2(const ggml_gtx_kernel_launch_desc_t * desc) { +#ifdef GGML_GTX_USE_GTX + const uint32_t ne0 = desc->ne[0]; + const uint32_t rows = ggml_gtx_binary_rows(desc); + const uint32_t row_bytes = ne0 * GGML_GTX_DTYPE_SIZE; + const uint32_t rhs_row_bytes = desc->src_ne[1][0] * GGML_GTX_DTYPE_SIZE; + + __split(); + __start_plan((uint8_t) desc->nest_id); + for (uint32_t tid = 0u; tid < GGML_GTX_SPU_NUM_PER_NEST; ++tid) { + const uint32_t thread_row0 = ggml_gtx_binary_row_for_tid(rows, tid); + const uint32_t thread_row1 = ggml_gtx_binary_row_for_tid(rows, tid + 1u); + + __start_thread((uint8_t) tid); + __set_spm_addr(GGML_GTX_L1_BANK_R_OFFSET, + GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, + GGML_GTX_L1_BANK_A_OFFSET); + + uint32_t row = thread_row0; + while (row < thread_row1) { + const uint32_t i1 = row % desc->ne[1]; + const uint32_t rows_to_i1_boundary = desc->ne[1] - i1; + const uint32_t cols = ne0 < GGML_GTX_BINARY_L1_MAX_ELEMS ? + ne0 : GGML_GTX_BINARY_L1_MAX_ELEMS; + const uint32_t max_rows_l1 = GGML_GTX_BINARY_L1_MAX_ELEMS / cols; + uint32_t block_rows = thread_row1 - row; + if (block_rows > rows_to_i1_boundary) { + block_rows = rows_to_i1_boundary; + } + if (block_rows > max_rows_l1) { + block_rows = max_rows_l1; + } + + for (uint32_t col0 = 0u; col0 < ne0; col0 += cols) { + const uint32_t block_cols = ne0 - col0 < cols ? ne0 - col0 : cols; + const uint32_t block_elems = block_cols * block_rows; + const uint16_t block_bytes = (uint16_t) (block_cols * GGML_GTX_DTYPE_SIZE); + const uint32_t a_off = row * row_bytes + col0 * GGML_GTX_DTYPE_SIZE; + const uint32_t b_row = ggml_gtx_binary_rhs_row(desc, row); + const uint32_t b_off = b_row * rhs_row_bytes + + col0 * GGML_GTX_DTYPE_SIZE; + const uint32_t b_stride = desc->src_ne[1][1] == 1u ? + 0u : rhs_row_bytes; + + __load((uint32_t) desc->l2_src_addr[0] + a_off, + GGML_GTX_L1_BANK_A_OFFSET, + row_bytes, + block_bytes, + (uint16_t) block_rows, + block_bytes); + __load((uint32_t) desc->l2_src_addr[1] + b_off, + GGML_GTX_L1_BANK_B_OFFSET, + b_stride, + block_bytes, + (uint16_t) block_rows, + block_bytes); + __mul_vv(block_elems); + __store(GGML_GTX_L1_BANK_R_OFFSET, + (uint32_t) desc->l2_dst_addr + a_off, + row_bytes, + block_bytes, + (uint16_t) block_rows, + block_bytes); + } + row += block_rows; + } + __end_thread((uint8_t) tid); + } + __end_plan((uint8_t) desc->nest_id); + __join(); +#else + (void) desc; +#endif +} + +static void ggml_gtx_compute_mul_vs_l2(const ggml_gtx_kernel_launch_desc_t * desc, + const uint16_t * scalars) { + const uint32_t ne0 = desc->ne[0]; + const uint32_t rows = ggml_gtx_binary_rows(desc); + const uint32_t row_bytes = ne0 * GGML_GTX_DTYPE_SIZE; + const uint32_t cols = ne0 < GGML_GTX_BINARY_L1_MAX_ELEMS ? + ne0 : GGML_GTX_BINARY_L1_MAX_ELEMS; + const uint32_t max_rows_l1 = GGML_GTX_BINARY_L1_MAX_ELEMS / cols; + + uint32_t row = 0u; + while (row < rows) { + const uint32_t i1 = row % desc->ne[1]; + uint32_t block_rows = rows - row; + const uint32_t rows_to_i1_boundary = desc->ne[1] - i1; + if (block_rows > rows_to_i1_boundary) block_rows = rows_to_i1_boundary; + if (block_rows > max_rows_l1) block_rows = max_rows_l1; + + const uint16_t scalar = scalars[ggml_gtx_binary_rhs_row(desc, row)]; + for (uint32_t col0 = 0u; col0 < ne0; col0 += cols) { + const uint32_t block_cols = ne0 - col0 < cols ? ne0 - col0 : cols; + const uint32_t block_elems = block_cols * block_rows; + const uint16_t block_bytes = (uint16_t)(block_cols * GGML_GTX_DTYPE_SIZE); + const uint32_t a_off = row * row_bytes + col0 * GGML_GTX_DTYPE_SIZE; + const uint32_t l2_src = (uint32_t)desc->l2_src_addr[0] + a_off; + const uint32_t l2_dst = (uint32_t)desc->l2_dst_addr + a_off; + + __split(); + __start_plan((uint8_t)desc->nest_id); + __start_thread(0u); + __set_spm_addr(GGML_GTX_L1_BANK_R_OFFSET, + GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, + GGML_GTX_L1_BANK_A_OFFSET); + __load(l2_src, GGML_GTX_L1_BANK_A_OFFSET, + row_bytes, block_bytes, (uint16_t)block_rows, block_bytes); + __mul_vs(block_elems, scalar, 0u); + __store(GGML_GTX_L1_BANK_R_OFFSET, l2_dst, + row_bytes, block_bytes, (uint16_t)block_rows, block_bytes); + __end_thread(0u); + __end_plan((uint8_t)desc->nest_id); + __join(); + } + row += block_rows; + } +} + +int ggml_gtx_kernel_mul_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_MUL_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_MUL_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + const uint32_t elems = desc->ne[0]; + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_L2_STASHED_N1 || + desc->nest_id >= GGML_GTX_NEST_NUM || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->src_type[1] != GGML_GTX_TYPE_F16 || + desc->elem_size != GGML_GTX_DTYPE_SIZE || + elems == 0 || + desc->src_ne[0][0] != elems || + (desc->src_ne[1][0] != elems && desc->src_ne[1][0] != 1u) || + elems > UINT16_MAX) { + return -2; + } + + uint32_t rows = 1u; + for (uint32_t d = 1u; d < 4u; ++d) { + if (desc->src_ne[0][d] != desc->ne[d] || + (desc->src_ne[1][d] != 1u && desc->src_ne[1][d] != desc->ne[d]) || + desc->ne[d] == 0u || rows > UINT32_MAX / desc->ne[d]) { + return -2; + } + rows *= desc->ne[d]; + } + + if (desc->src_ne[1][0] == elems) { + ggml_gtx_compute_mul_vv_l2(desc); + return 0; + } + + const ggml_gtx_mul_args_t * params = + (const ggml_gtx_mul_args_t *)ggml_gtx_kernel_args( + karg, GGML_GTX_MUL_ARGS_HEADER_SIZE); + const uint64_t expected_scalar_count = + (uint64_t)desc->src_ne[1][1] * desc->src_ne[1][2] * desc->src_ne[1][3]; + const uint64_t expected_size = GGML_GTX_MUL_ARGS_HEADER_SIZE + + expected_scalar_count * sizeof(uint16_t); + if (params == 0 || params->version != GGML_GTX_MUL_ARGS_VERSION || + params->size != karg->desc.args_size || params->size != expected_size || + params->scalar_count != expected_scalar_count) { + return -3; + } + ggml_gtx_compute_mul_vs_l2(desc, params->scalars); + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_mul_f16(arg); +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-neg.c b/depend/ggml-gtx/src/ggml-gtx-kernel-neg.c new file mode 100644 index 0000000..86caf8f --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-neg.c @@ -0,0 +1,6 @@ +#define GGML_GTX_UNARY_KERNEL_ID GGML_GTX_KERNEL_NEG_F16 +#define GGML_GTX_UNARY_KERNEL_FN ggml_gtx_kernel_neg_f16 +#define GGML_GTX_UNARY_INPUT_BANK GGML_GTX_UNARY_BANK_A +#define GGML_GTX_UNARY_OUTPUT_BANK GGML_GTX_UNARY_BANK_R +#define GGML_GTX_UNARY_APPLY(elems) __neg_v(elems) +#include "ggml-gtx-kernel-unary-ref.h" diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-norm.c b/depend/ggml-gtx/src/ggml-gtx-kernel-norm.c new file mode 100644 index 0000000..c972b73 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-norm.c @@ -0,0 +1,185 @@ +//================================================================== +// GGML GTX kernel: NORM +//================================================================== + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#define GGML_GTX_NORM_L2_A 0x000000u +#define GGML_GTX_NORM_L2_RESULT 0x020000u + +#define GGML_GTX_NORM_BANK_A 0x00000u +#define GGML_GTX_NORM_BANK_B 0x20000u +#define GGML_GTX_NORM_BANK_C 0x30000u +#define GGML_GTX_NORM_BANK_R 0x50000u + +#define GGML_GTX_NORM_MAX_DMA_BYTES 65535u +#define GGML_GTX_NORM_NOT_USE 0xBEEFu + +static inline uint32_t ggml_gtx_norm_min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t ggml_gtx_norm_rows_for_tid(uint32_t total_rows, uint8_t tid) { + const uint32_t q = total_rows / GGML_GTX_SPU_NUM_PER_NEST; + const uint32_t r = total_rows % GGML_GTX_SPU_NUM_PER_NEST; + return tid < r ? q + 1u : q; +} + +static inline uint32_t ggml_gtx_norm_row_start_for_tid(uint32_t total_rows, uint8_t tid) { + const uint32_t q = total_rows / GGML_GTX_SPU_NUM_PER_NEST; + const uint32_t r = total_rows % GGML_GTX_SPU_NUM_PER_NEST; + + if (tid < r) { + return (uint32_t) tid * (q + 1u); + } + + return r * (q + 1u) + ((uint32_t) tid - r) * q; +} + +static inline uint16_t ggml_gtx_norm_active_tid_mask(uint32_t total_rows) { + const uint32_t active_tid_count = + ggml_gtx_norm_min_u32(total_rows, GGML_GTX_SPU_NUM_PER_NEST); + + if (active_tid_count >= GGML_GTX_SPU_NUM_PER_NEST) { + return 0xFFFFu; + } + + return (uint16_t) ((1u << active_tid_count) - 1u); +} + +static inline float ggml_gtx_norm_f32_from_bits(uint32_t bits) { + union { + uint32_t u; + float f; + } v; + v.u = bits; + return v.f; +} + +static inline uint32_t ggml_gtx_norm_f32_to_bits(float f) { + union { + float f; + uint32_t u; + } v; + v.f = f; + return v.u; +} + +static inline float ggml_gtx_norm_fp16_to_f32(uint16_t h) { + return ggml_gtx_norm_f32_from_bits(ggml_gtx_fp16_to_fp32_bits(h)); +} + +static inline uint16_t ggml_gtx_norm_f32_to_fp16(float f) { + return ggml_gtx_fp32_bits_to_fp16(ggml_gtx_norm_f32_to_bits(f)); +} + +static inline float ggml_gtx_norm_rsqrtf(float x) { + if (x <= 0.0f) { + return 0.0f; + } + + const float xhalf = 0.5f * x; + union { + float f; + uint32_t i; + } v; + v.f = x; + v.i = 0x5f3759dfu - (v.i >> 1); + v.f = v.f * (1.5f - xhalf * v.f * v.f); + v.f = v.f * (1.5f - xhalf * v.f * v.f); + return v.f; +} + +void ggml_gtx_compute_norm( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t rows, + uint16_t cols, + uint16_t epsilon); + +int ggml_gtx_kernel_norm_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_NORM_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_NORM_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + const uint64_t rows64 = (uint64_t) desc->ne[1] * + (uint64_t) desc->ne[2] * + (uint64_t) desc->ne[3]; + if (desc->dst_addr == 0 || + desc->src_addr[0] == 0 || + desc->ne[0] == 0 || + rows64 == 0 || + desc->ne[0] > UINT16_MAX || + rows64 > UINT16_MAX || + desc->src_ne[0][0] != desc->ne[0] || + desc->src_ne[0][1] != desc->ne[1] || + desc->src_ne[0][2] != desc->ne[2] || + desc->src_ne[0][3] != desc->ne[3] || + desc->nb[0] != GGML_GTX_DTYPE_SIZE || + desc->src_nb[0][0] != GGML_GTX_DTYPE_SIZE || + desc->nb[1] != desc->ne[0] * GGML_GTX_DTYPE_SIZE || + desc->src_nb[0][1] != desc->src_ne[0][0] * GGML_GTX_DTYPE_SIZE) { + return -2; + } + + ggml_gtx_compute_norm(desc->src_addr[0], + desc->dst_addr, + (uint16_t) rows64, + (uint16_t) desc->ne[0], + ggml_gtx_fp32_bits_to_fp16((uint32_t) desc->op_params[0])); + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_norm_f16(arg); +} + +void ggml_gtx_compute_norm( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t rows, + uint16_t cols, + uint16_t epsilon) +{ +#ifdef GGML_GTX_USE_GTX + if (rows == 0 || cols == 0) { + return; + } + + volatile const uint16_t * src = (volatile const uint16_t *) (uintptr_t) ddr_input; + volatile uint16_t * dst = (volatile uint16_t *) (uintptr_t) ddr_result; + const float eps = ggml_gtx_norm_fp16_to_f32(epsilon); + + for (uint16_t row = 0; row < rows; ++row) { + const uint32_t row_base = (uint32_t) row * cols; + float mean = 0.0f; + for (uint16_t col = 0; col < cols; ++col) { + mean += ggml_gtx_norm_fp16_to_f32(src[row_base + col]); + } + mean /= (float) cols; + + float variance = 0.0f; + for (uint16_t col = 0; col < cols; ++col) { + const float d = ggml_gtx_norm_fp16_to_f32(src[row_base + col]) - mean; + variance += d * d; + } + variance /= (float) cols; + + const float scale = ggml_gtx_norm_rsqrtf(variance + eps); + for (uint16_t col = 0; col < cols; ++col) { + const float v = (ggml_gtx_norm_fp16_to_f32(src[row_base + col]) - mean) * scale; + dst[row_base + col] = ggml_gtx_norm_f32_to_fp16(v); + } + } +#else + (void) ddr_input; (void) ddr_result; (void) rows; (void) cols; (void) epsilon; +#endif +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-pad.c b/depend/ggml-gtx/src/ggml-gtx-kernel-pad.c new file mode 100644 index 0000000..bae36d2 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-pad.c @@ -0,0 +1,94 @@ +//================================================================== +// GGML GTX kernel: PAD +// +// L2-only right-side zero padding for a host-lowered raw view. +//================================================================== + +#include + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +int ggml_gtx_kernel_pad_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_PAD_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_PAD_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_L2_STASHED_N1 || + desc->l2_dst_addr == 0 || + desc->l2_src_addr[0] == 0 || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16) { + return -2; + } + + const uint32_t src_cols = desc->src_ne[0][0]; + const uint32_t src_rows = desc->src_ne[0][1]; + const uint32_t src_channels = desc->src_ne[0][2] ? desc->src_ne[0][2] : 1u; + const uint32_t dst_cols = desc->ne[0]; + const uint32_t dst_rows = desc->ne[1]; + const uint32_t dst_channels = desc->ne[2] ? desc->ne[2] : 1u; + const uint32_t batches = desc->ne[3] ? desc->ne[3] : 1u; + + const int32_t lp0 = desc->op_params[0]; + const int32_t rp0 = desc->op_params[1]; + const int32_t lp1 = desc->op_params[2]; + const int32_t rp1 = desc->op_params[3]; + const int32_t lp2 = desc->op_params[4]; + const int32_t rp2 = desc->op_params[5]; + const int32_t lp3 = desc->op_params[6]; + const int32_t rp3 = desc->op_params[7]; + const int32_t circular = desc->op_params[8]; + + if (src_cols == 0 || src_rows == 0 || src_channels == 0 || + dst_cols == 0 || dst_rows == 0 || dst_channels == 0 || + lp0 != 0 || lp1 != 0 || lp2 != 0 || lp3 != 0 || + rp0 < 0 || rp1 < 0 || rp2 < 0 || rp3 != 0 || + circular != 0 || + dst_cols != src_cols + (uint32_t) rp0 || + dst_rows != src_rows + (uint32_t) rp1 || + dst_channels != src_channels + (uint32_t) rp2 || + batches != (desc->src_ne[0][3] ? desc->src_ne[0][3] : 1u) || + desc->src_nb[0][0] != GGML_GTX_DTYPE_SIZE || + desc->nb[0] != GGML_GTX_DTYPE_SIZE) { + return -3; + } + + const uint32_t src_row_bytes = src_cols * sizeof(uint16_t); + const uint32_t dst_row_bytes = dst_cols * sizeof(uint16_t); + const uint64_t dst_rows_total = (uint64_t)dst_rows * dst_channels * batches; + if (src_row_bytes > UINT16_MAX || dst_row_bytes > UINT16_MAX || + dst_rows_total == 0 || dst_rows_total > UINT16_MAX) { + return -4; + } + + __start_plan(0); + __start_shared(); + __fill(desc->l2_dst_addr, dst_row_bytes, (uint16_t)dst_row_bytes, + (uint16_t)dst_rows_total, 0u, 0); + for (uint32_t b = 0; b < batches; ++b) { + for (uint32_t c = 0; c < src_channels; ++c) { + const uint32_t src_plane = (b * src_channels + c) * src_rows * src_row_bytes; + const uint32_t dst_plane = (b * dst_channels + c) * dst_rows * dst_row_bytes; + __copy((uint32_t)desc->l2_src_addr[0] + src_plane, + (uint32_t)desc->l2_dst_addr + dst_plane, + src_row_bytes, (uint16_t)src_row_bytes, + (uint16_t)src_rows, dst_row_bytes); + } + } + __end_shared(); + __end_plan(0); + + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_pad_f16(arg); +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-relu.c b/depend/ggml-gtx/src/ggml-gtx-kernel-relu.c new file mode 100644 index 0000000..0ee04bd --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-relu.c @@ -0,0 +1,6 @@ +#define GGML_GTX_UNARY_KERNEL_ID GGML_GTX_KERNEL_RELU_F16 +#define GGML_GTX_UNARY_KERNEL_FN ggml_gtx_kernel_relu_f16 +#define GGML_GTX_UNARY_INPUT_BANK GGML_GTX_UNARY_BANK_R +#define GGML_GTX_UNARY_OUTPUT_BANK GGML_GTX_UNARY_BANK_A +#define GGML_GTX_UNARY_APPLY(elems) __relu(elems, GGML_GTX_UNARY_BANK_A, GGML_GTX_UNARY_BANK_R) +#include "ggml-gtx-kernel-unary-ref.h" diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-repeat.c b/depend/ggml-gtx/src/ggml-gtx-kernel-repeat.c new file mode 100644 index 0000000..7f9bab6 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-repeat.c @@ -0,0 +1,152 @@ +//================================================================== +// GGML GTX kernel: REPEAT +//================================================================== + +#include +#include +#include + +#include "ggml-gtx-baremetal.h" +#include "ggml-gtx-compute.h" +#include "ggml-gtx-port.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +int ggml_gtx_kernel_repeat_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_REPEAT_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_REPEAT_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_L2_STASHED_N1 || + desc->l2_dst_addr == 0 || + desc->l2_src_addr[0] == 0 || + desc->ne[0] == 0 || + desc->ne[1] == 0 || + desc->ne[2] == 0 || + desc->ne[3] == 0 || + desc->src_ne[0][0] == 0 || + desc->src_ne[0][1] == 0 || + desc->src_ne[0][2] == 0 || + desc->src_ne[0][3] == 0 || + desc->ne[0] > UINT16_MAX / GGML_GTX_DTYPE_SIZE || + desc->src_ne[0][0] > UINT16_MAX / GGML_GTX_DTYPE_SIZE) { + return -2; + } + + for (int i = 0; i < 4; ++i) { + if (desc->ne[i] > UINT16_MAX || + desc->src_ne[0][i] > UINT16_MAX || + desc->ne[i] < desc->src_ne[0][i] || + (desc->ne[i] % desc->src_ne[0][i]) != 0) { + return -3; + } + } + + ggml_gtx_compute_repeat(desc->l2_src_addr[0], + desc->l2_dst_addr, + (uint16_t) desc->src_ne[0][0], + (uint16_t) desc->src_ne[0][1], + (uint16_t) desc->src_ne[0][2], + (uint16_t) desc->src_ne[0][3], + (uint16_t) desc->ne[0], + (uint16_t) desc->ne[1], + (uint16_t) desc->ne[2], + (uint16_t) desc->ne[3], + GGML_GTX_DTYPE_SIZE, + 1, + 1); + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_repeat_f16(arg); +} + +void ggml_gtx_compute_repeat( + uint64_t ddr_src, + uint64_t ddr_dst, + uint16_t ne00, + uint16_t ne01, + uint16_t ne02, + uint16_t ne03, + uint16_t ne0, + uint16_t ne1, + uint16_t ne2, + uint16_t ne3, + uint16_t elem_size, + uint8_t nest_num, + uint8_t spu_num_per_nest) +{ + (void) nest_num; + (void) spu_num_per_nest; + +#ifdef GGML_GTX_USE_GTX + const uint32_t src_row_bytes = (uint32_t)ne00 * elem_size; + const uint32_t dst_row_bytes = (uint32_t)ne0 * elem_size; + const uint32_t src_plane_rows = (uint32_t)ne01; + const uint32_t src_plane_count = (uint32_t)ne02; + const uint32_t repeat0 = ne0 / ne00; + const uint32_t repeat1 = ne1 / ne01; + + __start_plan(0); + __start_shared(); + for (uint32_t i3 = 0; i3 < ne3; ++i3) { + const uint32_t s3 = i3 % ne03; + for (uint32_t i2 = 0; i2 < ne2; ++i2) { + const uint32_t s2 = i2 % ne02; + const uint32_t src_plane = (s3 * src_plane_count + s2) * src_plane_rows; + const uint32_t dst_plane = (i3 * (uint32_t)ne2 + i2) * (uint32_t)ne1; + for (uint32_t r1 = 0; r1 < repeat1; ++r1) { + const uint32_t dst_row0 = dst_plane + r1 * src_plane_rows; + for (uint32_t r0 = 0; r0 < repeat0; ++r0) { + __copy((uint32_t)ddr_src + src_plane * src_row_bytes, + (uint32_t)ddr_dst + dst_row0 * dst_row_bytes + r0 * src_row_bytes, + src_row_bytes, + (uint16_t)src_row_bytes, + (uint16_t)src_plane_rows, + dst_row_bytes); + } + } + } + } + __end_shared(); + __end_plan(0); +#else + + volatile const uint8_t * src = (volatile const uint8_t *)(uintptr_t)ddr_src; + volatile uint8_t * dst = (volatile uint8_t *)(uintptr_t)ddr_dst; + + const uint64_t src_nb0 = elem_size; + const uint64_t src_nb1 = (uint64_t) ne00 * src_nb0; + const uint64_t src_nb2 = (uint64_t) ne01 * src_nb1; + const uint64_t src_nb3 = (uint64_t) ne02 * src_nb2; + + uint64_t dst_off = 0; + for (uint16_t i3 = 0; i3 < ne3; ++i3) { + const uint16_t s3 = i3 % ne03; + for (uint16_t i2 = 0; i2 < ne2; ++i2) { + const uint16_t s2 = i2 % ne02; + for (uint16_t i1 = 0; i1 < ne1; ++i1) { + const uint16_t s1 = i1 % ne01; + for (uint16_t i0 = 0; i0 < ne0; ++i0) { + const uint16_t s0 = i0 % ne00; + const uint64_t src_off = (uint64_t) s3 * src_nb3 + + (uint64_t) s2 * src_nb2 + + (uint64_t) s1 * src_nb1 + + (uint64_t) s0 * src_nb0; + for (uint16_t eb = 0; eb < elem_size; ++eb) { + dst[dst_off + eb] = src[src_off + eb]; + } + dst_off += elem_size; + } + } + } + } +#endif +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-rms-norm.c b/depend/ggml-gtx/src/ggml-gtx-kernel-rms-norm.c new file mode 100644 index 0000000..108b493 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-rms-norm.c @@ -0,0 +1,354 @@ +//================================================================== +// GGML GTX kernel: RMS_NORM +//================================================================== + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +int ggml_gtx_kernel_rms_norm_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_RMS_NORM_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_RMS_NORM_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + const uint64_t rows64 = (uint64_t) desc->ne[1] * + (uint64_t) desc->ne[2] * + (uint64_t) desc->ne[3]; + if (desc->dst_addr == 0 || + desc->src_addr[0] == 0 || + desc->ne[0] == 0 || + rows64 == 0 || + desc->ne[0] > UINT16_MAX) { + return -2; + } + + if (desc->memory_mode == GGML_GTX_MEMORY_MODE_L2_STASHED_N1) { + const uint64_t row_bytes = (uint64_t) desc->ne[0] * sizeof(uint16_t); + if (desc->nest_id >= GGML_GTX_NEST_NUM || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->elem_size != GGML_GTX_DTYPE_SIZE || + desc->l2_src_addr[0] == 0 || + desc->l2_dst_addr == 0 || + row_bytes == 0 || + row_bytes > GGML_GTX_L1_BANK_SIZE || + rows64 > UINT16_MAX) { + return -3; + } + + ggml_gtx_compute_rmsnorm_l2((uint32_t) desc->l2_src_addr[0], + (uint32_t) desc->l2_dst_addr, + (uint16_t) rows64, + (uint16_t) desc->ne[0], + ggml_gtx_fp32_bits_to_fp16((uint32_t) desc->op_params[0]), + 1u, + GGML_GTX_SPU_NUM_PER_NEST); + return 0; + } + + if (rows64 > UINT16_MAX) { + return -4; + } + + ggml_gtx_compute_rmsnorm(desc->src_addr[0], + desc->dst_addr, + (uint16_t) rows64, + (uint16_t) desc->ne[0], + ggml_gtx_fp32_bits_to_fp16((uint32_t) desc->op_params[0]), + GGML_GTX_NEST_NUM, + GGML_GTX_SPU_NUM_PER_NEST); + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_rms_norm_f16(arg); +} + +void ggml_gtx_compute_rmsnorm( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint16_t epsilon, + uint8_t nest_num, + uint8_t spu_num_per_nest) +{ +#ifdef GGML_GTX_USE_GTX + ggml_gtx_debug_dump_fp16_head("RMS_A", ddr_input); + const uint16_t not_use = 0xBEEF; + const uint32_t cal_unit = i_col; + const uint32_t cal_unit_byte = (uint32_t)i_col * GGML_GTX_DTYPE_SIZE; + const uint32_t cal_max = cal_unit_byte != 0 ? (GGML_GTX_L1_BANK_SIZE / cal_unit_byte) : 0; + const uint32_t cal_num = i_row; + const uint8_t eff_thread = cal_num == 0 ? 1 : (cal_num / spu_num_per_nest ? spu_num_per_nest : (uint8_t)cal_num); + const uint32_t cal_quot = eff_thread != 0 ? (cal_num / eff_thread) : 0; + const uint32_t cal_mod = eff_thread != 0 ? (cal_num % eff_thread) : 0; + + uint32_t bank_offset_shared = 0; + uint32_t bank_offset_thread = 0; + uint32_t cal_offset = 0; + uint16_t exe_cnt = 0; + + GGML_GTX_UNUSED(nest_num); + + if (cal_unit == 0 || cal_unit_byte == 0 || cal_num == 0 || cal_max == 0 || eff_thread == 0) { + return; + } + + __split(); + __start_plani(0); + __start_shared(); + for (uint8_t thread_id = 0; thread_id < eff_thread; thread_id++) { + const uint32_t cal_num_per_thread = thread_id < cal_mod ? (cal_quot + 1) : cal_quot; + const uint32_t offset_thread = + ((GGML_GTX_MIN(thread_id, cal_mod) * (cal_quot + 1)) + + (GGML_GTX_MAX(0, (int32_t)thread_id - (int32_t)cal_mod) * cal_quot)) * cal_unit_byte; + + __load(ddr_input + offset_thread, + GGML_GTX_L2_INPUT + bank_offset_shared, + cal_unit_byte, cal_unit_byte, + cal_num_per_thread, cal_unit_byte); + __credit_ld(0x1 << thread_id, not_use); + bank_offset_shared += GGML_GTX_L2_BANK_OFFSET; + } + __end_shared(); + + for (uint8_t thread_id = 0; thread_id < eff_thread; thread_id++) { + const uint32_t cal_num_per_thread = thread_id < cal_mod ? (cal_quot + 1) : cal_quot; + const uint32_t cal_last = (cal_num_per_thread % cal_max) ? (cal_num_per_thread % cal_max) : cal_max; + + __start_thread(thread_id); + __set_spm_addr_A(GGML_GTX_L1_BANK_A_OFFSET); + __set_spm_addr_R(GGML_GTX_L1_BANK_R_OFFSET); + __credit_chk(not_use); + __load(GGML_GTX_L2_INPUT + bank_offset_thread, + GGML_GTX_L1_BANK_A_OFFSET, + cal_unit_byte, cal_unit_byte, + cal_last, not_use); + __credit_ld(not_use, not_use); + __rmsnorm(cal_unit, + GGML_GTX_L1_BANK_A_OFFSET, + 0, + GGML_GTX_L1_BANK_R_OFFSET, + epsilon); + bank_offset_thread += GGML_GTX_L2_BANK_OFFSET; + __end_thread(thread_id); + } + __end_plani(0); + __join(); + + exe_cnt++; + bank_offset_shared = 0; + bank_offset_thread = 0; + + for (uint32_t iter = 0; iter + 1 < cal_quot; iter++) { + cal_offset += cal_unit_byte; + + __split(); + __start_plani(0); + for (uint8_t thread_id = 0; thread_id < eff_thread; thread_id++) { + __start_thread(thread_id); + const uint32_t l1_A_addr = GGML_GTX_L1_BANK_A_OFFSET + cal_offset; + const uint32_t l1_R_addr = GGML_GTX_L1_BANK_R_OFFSET + cal_offset; + __set_spm_addr_A(l1_A_addr); + __set_spm_addr_R(l1_R_addr); + __rmsnorm(cal_unit, l1_A_addr, 0, l1_R_addr, epsilon); + __end_thread(thread_id); + } + __end_plani(0); + __join(); + + exe_cnt++; + } + + cal_offset += cal_unit_byte; + + __split(); + __start_plani(0); + __start_shared(); + for (uint8_t thread_id = 0; thread_id < eff_thread; thread_id++) { + const uint32_t cal_num_per_thread = thread_id < cal_mod ? (cal_quot + 1) : cal_quot; + const uint32_t offset_thread = + ((GGML_GTX_MIN(thread_id, cal_mod) * (cal_quot + 1)) + + (GGML_GTX_MAX(0, (int32_t)thread_id - (int32_t)cal_mod) * cal_quot)) * cal_unit_byte; + const uint16_t target = (uint16_t)(0x1U << thread_id); + + __credit_chk(target); + __store(GGML_GTX_L2_RESULT + bank_offset_shared, + ddr_result + offset_thread, + cal_unit_byte, cal_unit_byte, + cal_num_per_thread, cal_unit_byte); + __credit_st(target); + bank_offset_shared += GGML_GTX_L2_BANK_OFFSET; + } + __end_shared(); + + for (uint8_t thread_id = 0; thread_id < eff_thread; thread_id++) { + const uint32_t cal_num_per_thread = thread_id < cal_mod ? (cal_quot + 1) : cal_quot; + const uint32_t cal_last = (cal_num_per_thread % cal_max) ? (cal_num_per_thread % cal_max) : cal_max; + + __start_thread(thread_id); + if (exe_cnt < cal_num_per_thread) { + const uint32_t l1_A_addr = GGML_GTX_L1_BANK_A_OFFSET + cal_offset; + const uint32_t l1_R_addr = GGML_GTX_L1_BANK_R_OFFSET + cal_offset; + __set_spm_addr_A(l1_A_addr); + __set_spm_addr_R(l1_R_addr); + __rmsnorm(cal_unit, l1_A_addr, 0, l1_R_addr, epsilon); + } + + __store(GGML_GTX_L1_BANK_R_OFFSET, + GGML_GTX_L2_RESULT + bank_offset_thread, + cal_unit_byte, cal_unit_byte, + cal_last, not_use); + __credit_st(not_use); + bank_offset_thread += GGML_GTX_L2_BANK_OFFSET; + __end_thread(thread_id); + } + __end_plani(0); + __join(); + + ggml_gtx_debug_dump_fp16_head("RMS_R", ddr_result); +#else + (void)ddr_input; (void)ddr_result; (void)i_row; (void)i_col; + (void)epsilon; (void)nest_num; (void)spu_num_per_nest; +#endif +} + +void ggml_gtx_compute_rmsnorm_l2( + uint32_t l2_input, + uint32_t l2_result, + uint16_t i_row, + uint16_t i_col, + uint16_t epsilon, + uint8_t nest_num, + uint8_t spu_num_per_nest) +{ +#ifdef GGML_GTX_USE_GTX + if (i_row == 0 || i_col == 0 || nest_num == 0 || spu_num_per_nest == 0) { + return; + } + + ggml_gtx_tile_plan_t plan; + ggml_gtx_tile_plan_init(&plan, i_row, i_col, nest_num, spu_num_per_nest); + if (plan.max_cal_per_thread == 0) { + return; + } + + __split(); + + for (uint8_t nest_id = 0; nest_id < plan.eff_nest_int; nest_id++) { + ggml_gtx_nest_tile_t * nest = &plan.nests[nest_id]; + + if (nest->cal_num > 0) { + __start_plan(nest_id); + + for (uint8_t thread_id = 0; thread_id < nest->eff_thread_int; thread_id++) { + ggml_gtx_thread_tile_t * thread = &nest->threads[thread_id]; + + if (thread->cal_num > 0) { + __start_thread(thread_id); + + __set_spm_addr_A(GGML_GTX_L1_BANK_A_OFFSET); + __set_spm_addr_R(GGML_GTX_L1_BANK_R_OFFSET); + + uint32_t l1_A_addr = GGML_GTX_L1_BANK_A_OFFSET; + uint32_t l1_R_addr = GGML_GTX_L1_BANK_R_OFFSET; + uint32_t offset_cal = 0; + + for (uint16_t j = 0; j < thread->iteration; j++) { + uint32_t next_offset = thread->offset + offset_cal; + + __load(l2_input + next_offset, + GGML_GTX_L1_BANK_A_OFFSET, + plan.unit_per_thread, + plan.unit_per_thread, + plan.max_cal_per_thread, + 0xBEEF); + + for (uint16_t k = 0; k < plan.max_cal_per_thread; k++) { + __rmsnorm(i_col, + l1_A_addr, + 0, + l1_R_addr, + epsilon); + + l1_A_addr += plan.unit_per_thread; + l1_R_addr += plan.unit_per_thread; + __set_spm_addr_A(l1_A_addr); + __set_spm_addr_R(l1_R_addr); + } + + l1_A_addr = GGML_GTX_L1_BANK_A_OFFSET; + l1_R_addr = GGML_GTX_L1_BANK_R_OFFSET; + __set_spm_addr_A(GGML_GTX_L1_BANK_A_OFFSET); + __set_spm_addr_R(GGML_GTX_L1_BANK_R_OFFSET); + + __store(GGML_GTX_L1_BANK_R_OFFSET, + l2_result + next_offset, + plan.unit_per_thread, + plan.unit_per_thread, + plan.max_cal_per_thread, + 0xBEEF); + + offset_cal += plan.unit_per_thread * plan.max_cal_per_thread; + } + + if (thread->remain > 0) { + uint32_t last_offset = thread->offset + offset_cal; + + __load(l2_input + last_offset, + GGML_GTX_L1_BANK_A_OFFSET, + plan.unit_per_thread, + plan.unit_per_thread, + thread->remain, + 0xBEEF); + + for (uint16_t k = 0; k < thread->remain; k++) { + __rmsnorm(i_col, + l1_A_addr, + 0, + l1_R_addr, + epsilon); + + l1_A_addr += plan.unit_per_thread; + l1_R_addr += plan.unit_per_thread; + __set_spm_addr_A(l1_A_addr); + __set_spm_addr_R(l1_R_addr); + } + + __set_spm_addr_A(GGML_GTX_L1_BANK_A_OFFSET); + __set_spm_addr_R(GGML_GTX_L1_BANK_R_OFFSET); + + __store(GGML_GTX_L1_BANK_R_OFFSET, + l2_result + last_offset, + plan.unit_per_thread, + plan.unit_per_thread, + thread->remain, + 0xBEEF); + } + + __end_thread(thread_id); + } + } + + __end_plan(nest_id); + } + } + + __join(); +#else + (void)l2_input; (void)l2_result; + (void)i_row; (void)i_col; (void)epsilon; + (void)nest_num; (void)spu_num_per_nest; +#endif +} + +//================================================================== +// Matrix Multiplication +//================================================================== diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-rope.c b/depend/ggml-gtx/src/ggml-gtx-kernel-rope.c new file mode 100644 index 0000000..34de9eb --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-rope.c @@ -0,0 +1,849 @@ +//================================================================== +// GGML Baremetal - RoPE (Rotary Position Embedding) Kernel +// +// GTX path now follows the CPU rope semantics directly on the original +// FP16 tensor layout instead of adapting the small packed-row reference +// micro-kernel contract. +//================================================================== + +#include + +#include "ggml-gtx-compute.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#ifdef GGML_GTX_USE_GTX +#include "ggml-gtx/intrinsics/intrin.h" +#endif + +static int ggml_gtx_compute_rope_l2(const ggml_gtx_kernel_launch_desc_t * desc); + +int ggml_gtx_kernel_rope_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_ROPE_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_ROPE_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + if (desc->memory_mode == GGML_GTX_MEMORY_MODE_L2_STASHED_N1) { + return ggml_gtx_compute_rope_l2(desc); + } + + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_DDR_DIRECT || + desc->dst_addr == 0 || desc->src_addr[0] == 0 || desc->src_addr[1] == 0) { + return -2; + } + + ggml_gtx_compute_rope(desc->src_addr[0], + desc->src_addr[1], + desc->dst_addr, + (uint16_t) desc->src_ne[0][0], + (uint16_t) desc->src_ne[0][1], + (uint16_t) desc->src_ne[0][2], + (uint16_t) desc->src_ne[0][3], + (uint16_t) desc->op_params[1], + (uint8_t) desc->op_params[2], + (uint32_t) desc->op_params[5], + (uint32_t) desc->op_params[6], + (uint32_t) desc->op_params[8], + GGML_GTX_NEST_NUM, + GGML_GTX_SPU_NUM_PER_NEST); + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_rope_f16(arg); +} + +static float ggml_gtx_bits_to_float(uint32_t bits) { + union { uint32_t u; float f; } value_bits; + value_bits.u = bits; + return value_bits.f; +} + +static uint32_t ggml_gtx_float_to_bits(float value) { + union { uint32_t u; float f; } value_bits; + value_bits.f = value; + return value_bits.u; +} + +static float ggml_gtx_expf(float arg) { + if (arg > 88.0f) { + arg = 88.0f; + } + if (arg < -88.0f) { + arg = -88.0f; + } + + const float ln2 = 0.69314718056f; + int exponent = (int)(arg / ln2); + float reduced = arg - ((float) exponent * ln2); + float reduced_sq = reduced * reduced; + float poly = 1.0f + reduced + reduced_sq * 0.5f + reduced * reduced_sq * (1.0f / 6.0f) + reduced_sq * reduced_sq * (1.0f / 24.0f); + + if (exponent >= 0) { + while (exponent-- > 0) { + poly *= 2.0f; + } + } else { + while (exponent++ < 0) { + poly *= 0.5f; + } + } + + return poly; +} + +static float ggml_gtx_logf(float arg) { + if (arg <= 0.0f) { + return -3.402823466e+38f; + } + + int exponent = 0; + while (arg >= 2.0f) { + arg *= 0.5f; + exponent++; + } + while (arg < 1.0f) { + arg *= 2.0f; + exponent--; + } + + const float y = (arg - 1.0f) / (arg + 1.0f); + const float y_sq = y * y; + float acc = y; + float term = y; + + term *= y_sq; + acc += term / 3.0f; + term *= y_sq; + acc += term / 5.0f; + term *= y_sq; + acc += term / 7.0f; + term *= y_sq; + acc += term / 9.0f; + + const float result = 2.0f * acc + ((float) exponent * 0.69314718056f); + return result; +} + +static int ggml_gtx_floor_to_i32(float arg) { + int truncated = (int)arg; + if ((float)truncated > arg) { + truncated--; + } + return truncated; +} + +static float ggml_gtx_sinf(float arg) { + const float half_pi = 1.57079632679f; + const float inv_half_pi = 0.63661977236f; + int quadrant = ggml_gtx_floor_to_i32(arg * inv_half_pi + (arg >= 0.0f ? 0.5f : -0.5f)); + float reduced = arg - (float)quadrant * half_pi; + float sign = 1.0f; + + switch (quadrant & 3) { + case 0: + break; + case 1: { + float tmp = reduced; + reduced = half_pi - tmp; + } break; + case 2: + sign = -1.0f; + break; + default: { + float tmp = reduced; + reduced = half_pi - tmp; + sign = -1.0f; + } break; + } + + const float x2 = reduced * reduced; + const float poly = (((((-1.0f / 39916800.0f) * x2 + (1.0f / 362880.0f)) * x2 + - (1.0f / 5040.0f)) * x2 + (1.0f / 120.0f)) * x2 + - (1.0f / 6.0f)) * x2 + 1.0f; + + return sign * reduced * poly; +} + +static float ggml_gtx_cosf(float arg) { + return ggml_gtx_sinf(arg + 1.57079632679f); +} + +#ifdef GGML_GTX_USE_GTX +#define GGML_GTX_ROPE_VEC_ELEMS 4u +#define GGML_GTX_ROPE_VEC_BYTES (GGML_GTX_ROPE_VEC_ELEMS * 2u) +#define GGML_GTX_ROPE_SVR_BYTES 32u +#define GGML_GTX_ROPE_FP16_ONE 0x3C00u +#define GGML_GTX_ROPE_FP16_NEG_HALF 0xB800u +#define GGML_GTX_ROPE_FP16_INV24 0x2955u +#define GGML_GTX_ROPE_FP16_NEG_INV6 0xB155u +#define GGML_GTX_ROPE_FP16_INV120 0x2044u +#define GGML_GTX_ROPE_FP16_INV_2PI 0x3118u +#define GGML_GTX_ROPE_FP16_2PI 0x4648u +#define GGML_GTX_ROPE_SIGN_MASK0 0x3C00BC003C00BC00ULL +#define GGML_GTX_SINCOS_VEC_ELEMS 16u +#define GGML_GTX_SINCOS_VEC_BYTES (GGML_GTX_SINCOS_VEC_ELEMS * 2u) + +static void ggml_gtx_sincos_vec_inline( + uint64_t src, + uint64_t dst, + uint32_t scratch_base, + uint8_t nest_id, + uint8_t thread_id, + uint8_t sincos_sel, + uint8_t l2_io) +{ + const uint32_t work_a = scratch_base; + const uint32_t work_b = scratch_base + GGML_GTX_ROPE_SCRATCH_WORK_B_OFFSET; + __start_plan(nest_id); + __start_shared(); + if (l2_io) { + __copy((uint32_t)src, work_a, + GGML_GTX_SINCOS_VEC_BYTES, GGML_GTX_SINCOS_VEC_BYTES, + 1, GGML_GTX_SINCOS_VEC_BYTES); + } else { + __load(src, work_a, + GGML_GTX_SINCOS_VEC_BYTES, GGML_GTX_SINCOS_VEC_BYTES, + 1, GGML_GTX_SINCOS_VEC_BYTES); + } + __end_shared(); + __end_plan(nest_id); + + __start_plan(nest_id); + __start_thread(thread_id); + __load(work_a, GGML_GTX_L1_BANK_A_OFFSET, + GGML_GTX_SINCOS_VEC_BYTES, GGML_GTX_SINCOS_VEC_BYTES, + 1, GGML_GTX_SINCOS_VEC_BYTES); + + __load_svr(GGML_GTX_L1_BANK_A_OFFSET, 0); + __mul_is(0, GGML_GTX_ROPE_FP16_INV_2PI, 0, 0); + __rne_i(0, 0); + + __wrspr(0x800 + 4, 0, 0x012BC248C2493C00ULL, 0); + __store_svr(GGML_GTX_L1_BANK_A_OFFSET + GGML_GTX_SINCOS_VEC_BYTES, 0); + __store_svr(GGML_GTX_L1_BANK_B_OFFSET, 1); + + __store(GGML_GTX_L1_BANK_A_OFFSET, work_a, + GGML_GTX_SINCOS_VEC_BYTES, GGML_GTX_SINCOS_VEC_BYTES, + 1, GGML_GTX_SINCOS_VEC_BYTES); + for (uint8_t i = 0; i < 3; ++i) { + __store(GGML_GTX_L1_BANK_A_OFFSET + GGML_GTX_SINCOS_VEC_BYTES, + work_a + (uint32_t)(i + 1) * GGML_GTX_SINCOS_VEC_BYTES, + GGML_GTX_SINCOS_VEC_BYTES, GGML_GTX_SINCOS_VEC_BYTES, + 1, GGML_GTX_SINCOS_VEC_BYTES); + } + __end_thread(thread_id); + __end_plan(nest_id); + + __start_plan(nest_id); + __start_shared(); + __tpose(work_a, work_b, 16, 4, 4, 1, 2, 0, 0b00, 1); + __end_shared(); + __end_plan(nest_id); + + __start_plan(nest_id); + __start_thread(thread_id); + __load(work_b, GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_SINCOS_VEC_BYTES, GGML_GTX_SINCOS_VEC_BYTES, + 4, GGML_GTX_SINCOS_VEC_BYTES); + + __set_spm_addr(GGML_GTX_L1_BANK_R_OFFSET, GGML_GTX_L1_BANK_A_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, GGML_GTX_L1_BANK_C_OFFSET); + __mm(16, 4, 1); + + __load_svr(GGML_GTX_L1_BANK_R_OFFSET, 0); + __mul_is(0, 0x3518, 0, 0); + __rne_i(0, 0); + + __wrspr(0x800 + 4, 0, 0x0095BE48BE493C00ULL, 0); + __store_svr(GGML_GTX_L1_BANK_R_OFFSET + GGML_GTX_SINCOS_VEC_BYTES, 0); + __store_svr(GGML_GTX_L1_BANK_B_OFFSET, 1); + + __store(GGML_GTX_L1_BANK_R_OFFSET, work_a, + GGML_GTX_SINCOS_VEC_BYTES, GGML_GTX_SINCOS_VEC_BYTES, + 1, GGML_GTX_SINCOS_VEC_BYTES); + for (uint8_t i = 0; i < 3; ++i) { + __store(GGML_GTX_L1_BANK_R_OFFSET + GGML_GTX_SINCOS_VEC_BYTES, + work_a + (uint32_t)(i + 1) * GGML_GTX_SINCOS_VEC_BYTES, + GGML_GTX_SINCOS_VEC_BYTES, GGML_GTX_SINCOS_VEC_BYTES, + 1, GGML_GTX_SINCOS_VEC_BYTES); + } + __end_thread(thread_id); + __end_plan(nest_id); + + __start_plan(nest_id); + __start_shared(); + __tpose(work_a, work_b, 16, 4, 4, 1, 2, 0, 0b00, 1); + __end_shared(); + __end_plan(nest_id); + + __start_plan(nest_id); + __start_thread(thread_id); + __load(work_b, GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_SINCOS_VEC_BYTES, GGML_GTX_SINCOS_VEC_BYTES, + 4, GGML_GTX_SINCOS_VEC_BYTES); + + __mm(16, 4, 1); + + __mul_is(0, 0x3800, 1, 0); + __ceil_i(1, 1); + __mul_is(1, 0x4000, 1, 0); + __sub_ii(0, 1, 1); + __add_is(1, 0x3C00, 2, 0); + __add_ii(1, 2, 0); + __store_svr(GGML_GTX_L1_BANK_C_OFFSET, 0); + + __load_svr(GGML_GTX_L1_BANK_R_OFFSET, 1); + __mvsvr(1, 2, 0); + __mul_ii(1, 2, 2); + + if (sincos_sel) { + __wrspr(0x800 + 4, 0, 0x3C003C003C003C00ULL, 0); + __cpsvr(1, 3); + __store_svr(GGML_GTX_L1_BANK_R_OFFSET, 1); + } + + for (uint8_t i = 0; i < (uint8_t)(3 + sincos_sel); ++i) { + __mul_ii(1, 2, 1); + __store_svr(GGML_GTX_L1_BANK_R_OFFSET + (uint32_t)(i + 1) * GGML_GTX_SINCOS_VEC_BYTES, 1); + } + + __store(GGML_GTX_L1_BANK_R_OFFSET, work_a, + GGML_GTX_SINCOS_VEC_BYTES, GGML_GTX_SINCOS_VEC_BYTES, + (uint8_t)(4 + sincos_sel), GGML_GTX_SINCOS_VEC_BYTES); + __end_thread(thread_id); + __end_plan(nest_id); + + __start_plan(nest_id); + __start_shared(); + __tpose(work_a, work_b, 16, (uint8_t)(4 + sincos_sel), 4, 1, 2, 0, 0b00, 1); + __end_shared(); + __end_plan(nest_id); + + __start_plan(nest_id); + __start_thread(thread_id); + __load(work_b, GGML_GTX_L1_BANK_A_OFFSET, + GGML_GTX_SINCOS_VEC_BYTES, GGML_GTX_SINCOS_VEC_BYTES, + (uint8_t)(4 + sincos_sel), GGML_GTX_SINCOS_VEC_BYTES); + + if (sincos_sel) { + __copy(GGML_GTX_L1_BANK_R_OFFSET, GGML_GTX_L1_BANK_B_OFFSET, 2, 2, 1, 2); + __wrspr(0x800 + 4, 0, 0x01A095B02955B800ULL, 0); + __store_svr(GGML_GTX_L1_BANK_B_OFFSET + GGML_GTX_SINCOS_VEC_BYTES, 1); + __copy(GGML_GTX_L1_BANK_B_OFFSET + GGML_GTX_SINCOS_VEC_BYTES, + GGML_GTX_L1_BANK_B_OFFSET + 2, + 8, 8, 1, 2); + } else { + __wrspr(0x800 + 4, 0, 0x8A802044B1553C00ULL, 0); + __store_svr(GGML_GTX_L1_BANK_B_OFFSET, 1); + } + + __set_spm_addr(GGML_GTX_L1_BANK_R_OFFSET, GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, GGML_GTX_L1_BANK_A_OFFSET); + __mm(16, (uint8_t)(4 + sincos_sel), 1); + + __set_spm_addr(GGML_GTX_L1_BANK_A_OFFSET, GGML_GTX_L1_BANK_B_OFFSET, + GGML_GTX_L1_BANK_C_OFFSET, GGML_GTX_L1_BANK_R_OFFSET); + __mul_vv(16); + + __store(GGML_GTX_L1_BANK_A_OFFSET, work_a, + GGML_GTX_SINCOS_VEC_BYTES, GGML_GTX_SINCOS_VEC_BYTES, + 1, GGML_GTX_SINCOS_VEC_BYTES); + __end_thread(thread_id); + __end_plan(nest_id); + + __start_plan(nest_id); + __start_shared(); + if (l2_io) { + __copy(work_a, (uint32_t)dst, + GGML_GTX_SINCOS_VEC_BYTES, GGML_GTX_SINCOS_VEC_BYTES, + 1, GGML_GTX_SINCOS_VEC_BYTES); + } else { + __store(work_a, dst, + GGML_GTX_SINCOS_VEC_BYTES, GGML_GTX_SINCOS_VEC_BYTES, + 1, GGML_GTX_SINCOS_VEC_BYTES); + } + __end_shared(); + __end_plan(nest_id); +} + +static void ggml_gtx_rope_range_reduce_to_A(uint32_t n) { + const uint32_t n_bytes = n * 2; + + __copy(GGML_GTX_L1_BANK_A_OFFSET, GGML_GTX_L1_BANK_C_OFFSET, + n_bytes, n_bytes, 1, n_bytes); + + __mul_vs(n, GGML_GTX_ROPE_FP16_INV_2PI, 0); + __copy(GGML_GTX_L1_BANK_R_OFFSET, GGML_GTX_L1_BANK_A_OFFSET, + n_bytes, n_bytes, 1, n_bytes); + __rne_v(n); + __copy(GGML_GTX_L1_BANK_R_OFFSET, GGML_GTX_L1_BANK_A_OFFSET, + n_bytes, n_bytes, 1, n_bytes); + __mul_vs(n, GGML_GTX_ROPE_FP16_2PI, 0); + __copy(GGML_GTX_L1_BANK_R_OFFSET, GGML_GTX_L1_BANK_B_OFFSET, + n_bytes, n_bytes, 1, n_bytes); + __copy(GGML_GTX_L1_BANK_C_OFFSET, GGML_GTX_L1_BANK_A_OFFSET, + n_bytes, n_bytes, 1, n_bytes); + __sub_vv(n); + __copy(GGML_GTX_L1_BANK_R_OFFSET, GGML_GTX_L1_BANK_A_OFFSET, + n_bytes, n_bytes, 1, n_bytes); +} +#endif + +static void ggml_gtx_compute_rope_cpu( + uint64_t ddr_src0, + uint64_t ddr_src1, + uint64_t ddr_dst, + uint16_t ne0, + uint16_t ne1, + uint16_t ne2, + uint16_t ne3, + uint16_t n_dims, + uint8_t mode, + uint32_t freq_base_b, + uint32_t freq_scale_b, + uint32_t attn_factor_b, + uint8_t nest_num, + uint8_t spu_num_per_nest) +{ + (void)nest_num; + (void)spu_num_per_nest; + + volatile uint16_t * src0_fp16 = (volatile uint16_t *)(uintptr_t)ddr_src0; + volatile int32_t * pos = (volatile int32_t *)(uintptr_t)ddr_src1; + volatile uint16_t * dst_fp16 = (volatile uint16_t *)(uintptr_t)ddr_dst; + + const float freq_base = ggml_gtx_bits_to_float(freq_base_b); + const float freq_scale = ggml_gtx_bits_to_float(freq_scale_b); + const float attn_factor = ggml_gtx_bits_to_float(attn_factor_b); + const uint16_t half_dims = n_dims / 2; + const float theta_scale = ggml_gtx_expf((-2.0f / (float)n_dims) * ggml_gtx_logf(freq_base)); + + for (uint16_t i3 = 0; i3 < ne3; ++i3) { + for (uint16_t i2 = 0; i2 < ne2; ++i2) { + const float p = (float)pos[i2] * freq_scale; + + for (uint16_t i1 = 0; i1 < ne1; ++i1) { + const uint64_t base_idx = (uint64_t)i3 * ne2 * ne1 * ne0 + + (uint64_t)i2 * ne1 * ne0 + + (uint64_t)i1 * ne0; + + for (uint16_t i0 = 0; i0 < ne0; ++i0) { + dst_fp16[base_idx + i0] = src0_fp16[base_idx + i0]; + } + + if (mode == 2) { + float theta = p * attn_factor; + for (uint16_t k = 0; k < half_dims; ++k) { + const float c = ggml_gtx_cosf(theta); + const float s = ggml_gtx_sinf(theta); + const uint16_t i_a = k; + const uint16_t i_b = (uint16_t)(k + half_dims); + + const float x0 = ggml_gtx_bits_to_float(ggml_gtx_fp16_to_fp32_bits(src0_fp16[base_idx + i_a])); + const float x1 = ggml_gtx_bits_to_float(ggml_gtx_fp16_to_fp32_bits(src0_fp16[base_idx + i_b])); + + dst_fp16[base_idx + i_a] = ggml_gtx_fp32_bits_to_fp16(ggml_gtx_float_to_bits(x0 * c - x1 * s)); + dst_fp16[base_idx + i_b] = ggml_gtx_fp32_bits_to_fp16(ggml_gtx_float_to_bits(x0 * s + x1 * c)); + theta *= theta_scale; + } + } else { + float theta = p * attn_factor; + for (uint16_t k = 0; k < half_dims; ++k) { + const float c = ggml_gtx_cosf(theta); + const float s = ggml_gtx_sinf(theta); + const uint16_t i_a = (uint16_t)(2 * k); + const uint16_t i_b = (uint16_t)(2 * k + 1); + + const float x0 = ggml_gtx_bits_to_float(ggml_gtx_fp16_to_fp32_bits(src0_fp16[base_idx + i_a])); + const float x1 = ggml_gtx_bits_to_float(ggml_gtx_fp16_to_fp32_bits(src0_fp16[base_idx + i_b])); + + dst_fp16[base_idx + i_a] = ggml_gtx_fp32_bits_to_fp16(ggml_gtx_float_to_bits(x0 * c - x1 * s)); + dst_fp16[base_idx + i_b] = ggml_gtx_fp32_bits_to_fp16(ggml_gtx_float_to_bits(x0 * s + x1 * c)); + theta *= theta_scale; + } + } + } + } + } +} + +static int ggml_gtx_compute_rope_l2(const ggml_gtx_kernel_launch_desc_t * desc) { +#ifdef GGML_GTX_USE_GTX + const uint32_t ne0 = desc->ne[0]; + const uint32_t ne1 = desc->ne[1]; + const uint32_t ne2 = desc->ne[2]; + const uint32_t ne3 = desc->ne[3]; + const uint32_t n_dims = (uint32_t)desc->op_params[1]; + const uint32_t mode = (uint32_t)desc->op_params[2]; + const uint32_t required = ggml_gtx_rope_scratch_size(ne2, n_dims); + const uint32_t pair_count = n_dims / 2u; + const uint32_t pairs_padded = (pair_count + 15u) & ~15u; + const uint32_t table_bytes = ne2 * pairs_padded * sizeof(uint16_t); + const uint32_t scratch = desc->l2_scratch_addr; + const uint32_t cos_l2 = scratch; + const uint32_t sin_l2 = cos_l2 + table_bytes; + const uint32_t row_bytes = ne0 * sizeof(uint16_t); + const uint32_t rows_total = ne1 * ne2 * ne3; + + if (mode != 0u || desc->nest_id != 0u || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->src_type[1] != GGML_GTX_TYPE_I32 || + desc->elem_size != sizeof(uint16_t) || + desc->l2_src_addr[0] == 0u || desc->l2_src_addr[1] == 0u || + desc->src_addr[1] == 0u || + desc->l2_dst_addr == 0u || required == 0u || + required > desc->l2_scratch_capacity || + n_dims > ne0 || ne0 > UINT16_MAX || ne1 > UINT16_MAX || + ne2 > UINT16_MAX || ne3 > UINT16_MAX || row_bytes > UINT16_MAX || + table_bytes > 64u * 1024u || rows_total == 0u || rows_total > UINT16_MAX) { + return -2; + } + + const float freq_base = ggml_gtx_bits_to_float((uint32_t)desc->op_params[5]); + const float freq_scale = ggml_gtx_bits_to_float((uint32_t)desc->op_params[6]); + const float attn_factor = ggml_gtx_bits_to_float((uint32_t)desc->op_params[8]); + const float theta_scale = ggml_gtx_expf((-2.0f / (float)n_dims) * ggml_gtx_logf(freq_base)); + volatile const int32_t * pos = (volatile const int32_t *)(uintptr_t)desc->src_addr[1]; + + /* The tensor stays L2 resident. Position ids are bounded scalar metadata; + * the control core expands them into FP16 cos/sin tables in dynamic L2 + * scratch, avoiding a tensor-sized DDR workspace or writeback. */ + __split(); + for (uint32_t i2 = 0; i2 < ne2; ++i2) { + float theta = (float)pos[i2] * freq_scale * attn_factor; + for (uint32_t pair_base = 0; pair_base < pairs_padded; pair_base += 16u) { + uint64_t cos_words[4] = { 0u, 0u, 0u, 0u }; + uint64_t sin_words[4] = { 0u, 0u, 0u, 0u }; + for (uint32_t lane = 0; lane < 16u; ++lane) { + const uint32_t pair = pair_base + lane; + uint16_t cos_h = GGML_GTX_ROPE_FP16_ONE; + uint16_t sin_h = 0u; + if (pair < pair_count) { + cos_h = ggml_gtx_fp32_bits_to_fp16(ggml_gtx_float_to_bits(ggml_gtx_cosf(theta))); + sin_h = ggml_gtx_fp32_bits_to_fp16(ggml_gtx_float_to_bits(ggml_gtx_sinf(theta))); + theta *= theta_scale; + } + cos_words[lane / 4u] |= (uint64_t)cos_h << ((lane % 4u) * 16u); + sin_words[lane / 4u] |= (uint64_t)sin_h << ((lane % 4u) * 16u); + } + + const uint32_t table_off = (i2 * pairs_padded + pair_base) * sizeof(uint16_t); + __start_plan(0); + __start_thread(0); + for (uint32_t word = 0; word < 4u; ++word) { + __wrspr(0x800u + word, 0, cos_words[word], 0); + __wrspr(0x800u + 4u + word, 0, sin_words[word], 0); + } + __store_svr(GGML_GTX_L1_BANK_A_OFFSET, 0); + __store(GGML_GTX_L1_BANK_A_OFFSET, cos_l2 + table_off, + 32u, 32u, 1, 32u); + __store_svr(GGML_GTX_L1_BANK_A_OFFSET, 1); + __store(GGML_GTX_L1_BANK_A_OFFSET, sin_l2 + table_off, + 32u, 32u, 1, 32u); + __end_thread(0); + __end_plan(0); + } + } + __join(); + + __start_plan(0); + __start_shared(); + __copy((uint32_t)desc->l2_src_addr[0], (uint32_t)desc->l2_dst_addr, + row_bytes, (uint16_t)row_bytes, (uint16_t)rows_total, (uint16_t)row_bytes); + __end_shared(); + __end_plan(0); + + __split(); + __start_plan(0); + for (uint32_t pair_base = 0; pair_base < pair_count; pair_base += GGML_GTX_SPU_NUM_PER_NEST) { + const uint32_t eff_spu = pair_base + GGML_GTX_SPU_NUM_PER_NEST <= pair_count ? + GGML_GTX_SPU_NUM_PER_NEST : pair_count - pair_base; + for (uint32_t tid = 0; tid < eff_spu; ++tid) { + __start_thread((uint8_t)tid); + __set_spm_addr(GGML_GTX_L1_BANK_R_OFFSET, GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, GGML_GTX_L1_BANK_A_OFFSET); + const uint32_t pair = pair_base + tid; + const uint32_t pair_off = pair * 2u * sizeof(uint16_t); + for (uint32_t i3 = 0; i3 < ne3; ++i3) { + for (uint32_t i2 = 0; i2 < ne2; ++i2) { + const uint32_t row0 = (i3 * ne2 + i2) * ne1; + const uint32_t cache_off = i2 * pairs_padded * sizeof(uint16_t) + + pair * sizeof(uint16_t); + __load((uint32_t)desc->l2_src_addr[0] + row0 * row_bytes + pair_off, + GGML_GTX_L1_BANK_A_OFFSET, + row_bytes, 2u * sizeof(uint16_t), (uint16_t)ne1, + 2u * sizeof(uint16_t)); + __load(cos_l2 + cache_off, GGML_GTX_L1_BANK_B_OFFSET, + table_bytes, sizeof(uint16_t), 2, sizeof(uint16_t)); + __copy(GGML_GTX_L1_BANK_B_OFFSET + sizeof(uint16_t), + GGML_GTX_L1_BANK_B_OFFSET + 2u * sizeof(uint16_t), + sizeof(uint16_t), sizeof(uint16_t), 1, sizeof(uint16_t)); + __copy(GGML_GTX_L1_BANK_B_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET + 3u * sizeof(uint16_t), + sizeof(uint16_t), sizeof(uint16_t), 1, sizeof(uint16_t)); + __set_spm_addr_A(GGML_GTX_L1_BANK_B_OFFSET + 2u * sizeof(uint16_t)); + __set_spm_addr_R(GGML_GTX_L1_BANK_B_OFFSET + sizeof(uint16_t)); + __neg_v(1); + __set_spm_addr(GGML_GTX_L1_BANK_R_OFFSET, GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, GGML_GTX_L1_BANK_A_OFFSET); + __mm(ne1, 2, 2); + __store(GGML_GTX_L1_BANK_R_OFFSET, + (uint32_t)desc->l2_dst_addr + row0 * row_bytes + pair_off, + row_bytes, 2u * sizeof(uint16_t), (uint16_t)ne1, + 2u * sizeof(uint16_t)); + } + } + __end_thread((uint8_t)tid); + } + } + __end_plan(0); + __join(); + return 0; +#else + (void)desc; + return -2; +#endif +} + +void ggml_gtx_compute_rope( + uint64_t ddr_src0, + uint64_t ddr_src1, + uint64_t ddr_dst, + uint16_t ne0, + uint16_t ne1, + uint16_t ne2, + uint16_t ne3, + uint16_t n_dims, + uint8_t mode, + uint32_t freq_base_b, + uint32_t freq_scale_b, + uint32_t attn_factor_b, + uint8_t nest_num, + uint8_t spu_num_per_nest) +{ +#ifdef GGML_GTX_USE_GTX + if (mode != 0 || n_dims < 2 || n_dims > ne0 || (n_dims & 1u) != 0 || (n_dims % GGML_GTX_ROPE_VEC_ELEMS) != 0) { + ggml_gtx_compute_rope_cpu( + ddr_src0, ddr_src1, ddr_dst, + ne0, ne1, ne2, ne3, n_dims, mode, + freq_base_b, freq_scale_b, attn_factor_b, + nest_num, spu_num_per_nest); + return; + } + volatile uint16_t * src0_fp16 = (volatile uint16_t *)(uintptr_t)ddr_src0; + volatile int32_t * pos = (volatile int32_t *)(uintptr_t)ddr_src1; + volatile uint16_t * dst_fp16 = (volatile uint16_t *)(uintptr_t)ddr_dst; + + const float freq_base = ggml_gtx_bits_to_float(freq_base_b); + const float freq_scale = ggml_gtx_bits_to_float(freq_scale_b); + const float attn_factor = ggml_gtx_bits_to_float(attn_factor_b); + const float freq_log = ggml_gtx_logf(freq_base); + const float n_dims_f = (float)n_dims; + const float exp_factor = -2.0f / n_dims_f; + const float exp_arg = exp_factor * freq_log; + const float theta_scale = ggml_gtx_expf(exp_arg); + const uint16_t pair_count = n_dims / 2; + const uint16_t row_bytes = ne0 * 2; + const uint32_t rows_total = (uint32_t)ne1 * ne2 * ne3; + const uint32_t row_stride_l2 = row_bytes; + const uint32_t cache_row_bytes = (uint32_t)pair_count * 2; + + volatile uint16_t * cache_ddr = (volatile uint16_t *)(uintptr_t)GGML_GTX_DDR_TEMP_0; + volatile uint16_t * theta_ddr = (volatile uint16_t *)(uintptr_t)GGML_GTX_DDR_TEMP_1; + + for (uint16_t i3 = 0; i3 < ne3; ++i3) { + for (uint16_t i2 = 0; i2 < ne2; ++i2) { + for (uint16_t i1 = 0; i1 < ne1; ++i1) { + const uint64_t base_idx = (uint64_t)i3 * ne2 * ne1 * ne0 + + (uint64_t)i2 * ne1 * ne0 + + (uint64_t)i1 * ne0; + volatile uint16_t * dst_row = dst_fp16 + base_idx; + const volatile uint16_t * src_row = src0_fp16 + base_idx; + + for (uint16_t i0 = n_dims; i0 < ne0; ++i0) { + dst_row[i0] = src_row[i0]; + } + } + } + } + + for (uint16_t i2 = 0; i2 < ne2; ++i2) { + float theta = (float)pos[i2] * freq_scale * attn_factor; + for (uint16_t pair = 0; pair < pair_count; ++pair) { + const uint32_t theta_off = (uint32_t)i2 * cache_row_bytes + (uint32_t)pair * 2; + theta_ddr[theta_off / 2] = ggml_gtx_fp32_bits_to_fp16(ggml_gtx_float_to_bits(theta)); + theta *= theta_scale; + } + } + + { + const uint32_t theta_elems = (uint32_t)ne2 * pair_count; + const uint32_t full_vec_elems = theta_elems & ~(GGML_GTX_SINCOS_VEC_ELEMS - 1u); + + for (uint32_t elem = 0; elem < full_vec_elems; elem += GGML_GTX_SINCOS_VEC_ELEMS) { + const uint64_t theta_addr = GGML_GTX_DDR_TEMP_1 + (uint64_t)elem * 2; + ggml_gtx_sincos_vec_inline(theta_addr, + GGML_GTX_DDR_TEMP_0 + (uint64_t)ne2 * cache_row_bytes + (uint64_t)elem * 2, + 0u, + 0, + 0, + 0, + 0); + ggml_gtx_sincos_vec_inline(theta_addr, + GGML_GTX_DDR_TEMP_0 + (uint64_t)elem * 2, + 0u, + 0, + 0, + 1, + 0); + } + + if (full_vec_elems < theta_elems) { + for (uint32_t elem = full_vec_elems; elem < theta_elems; ++elem) { + const float theta = ggml_gtx_bits_to_float(ggml_gtx_fp16_to_fp32_bits(theta_ddr[elem])); + cache_ddr[elem] = ggml_gtx_fp32_bits_to_fp16(ggml_gtx_float_to_bits(ggml_gtx_cosf(theta))); + cache_ddr[(uint32_t)ne2 * pair_count + elem] = ggml_gtx_fp32_bits_to_fp16(ggml_gtx_float_to_bits(ggml_gtx_sinf(theta))); + } + } + } + + { + const uint8_t nest_id = 0; + + __start_plan(nest_id); + __start_shared(); + __load( + ddr_src0, + GGML_GTX_L2_A, + row_bytes, + row_bytes, + rows_total, + row_bytes + ); + + __copy( + GGML_GTX_L2_A, + GGML_GTX_L2_RESULT, + row_bytes, + row_bytes, + rows_total, + row_bytes + ); + + __load( + GGML_GTX_DDR_TEMP_0, + GGML_GTX_L2_B, + cache_row_bytes, + cache_row_bytes, + ne2 * 2, + cache_row_bytes + ); + __end_shared(); + __end_plan(nest_id); + } + + __split(); + + { + const uint8_t nest_id = 0; + + __start_plan(nest_id); + for (uint16_t pair_base = 0; pair_base < pair_count; pair_base += spu_num_per_nest) { + const uint16_t eff_spu = (uint16_t)((pair_base + spu_num_per_nest <= pair_count) ? spu_num_per_nest : (pair_count - pair_base)); + for (uint16_t thread_id = 0; thread_id < eff_spu; ++thread_id) { + __start_thread(thread_id); + __set_spm_addr(GGML_GTX_L1_BANK_R_OFFSET, GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, GGML_GTX_L1_BANK_A_OFFSET); + + const uint16_t pair_idx = (uint16_t)(pair_base + thread_id); + const uint32_t pair_off_bytes = (uint32_t)pair_idx * 2 * 2; + + for (uint16_t i3 = 0; i3 < ne3; ++i3) { + for (uint16_t i2 = 0; i2 < ne2; ++i2) { + const uint32_t seq_pair_off = (uint32_t)i2 * cache_row_bytes + (uint32_t)pair_idx * 2; + const uint32_t row0_idx = ((uint32_t)i3 * ne2 + i2) * ne1; + + __load( + GGML_GTX_L2_A + row0_idx * row_stride_l2 + pair_off_bytes, + GGML_GTX_L1_BANK_A_OFFSET, + row_bytes, + 2 * 2, + ne1, + 2 * 2 + ); + + __load( + GGML_GTX_L2_B + seq_pair_off, + GGML_GTX_L1_BANK_B_OFFSET, + ne2 * cache_row_bytes, + 2, + 2, + 2 + ); + + __copy(GGML_GTX_L1_BANK_B_OFFSET + 2, + GGML_GTX_L1_BANK_B_OFFSET + 4, + 2, + 2, + 1, + 2); + __copy(GGML_GTX_L1_BANK_B_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET + 6, + 2, + 2, + 1, + 2); + __set_spm_addr_A(GGML_GTX_L1_BANK_B_OFFSET + 4); + __set_spm_addr_R(GGML_GTX_L1_BANK_B_OFFSET + 2); + __neg_v(1); + __set_spm_addr(GGML_GTX_L1_BANK_R_OFFSET, GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, GGML_GTX_L1_BANK_A_OFFSET); + + __mm(ne1, 2, 2); + + __store( + GGML_GTX_L1_BANK_R_OFFSET, + GGML_GTX_L2_RESULT + row0_idx * row_stride_l2 + pair_off_bytes, + row_bytes, + 2 * 2, + ne1, + 2 * 2 + ); + } + } + __end_thread(thread_id); + } + } + __end_plan(nest_id); + } + + __join(); + + __start_plan(0); + __start_shared(); + __store( + GGML_GTX_L2_RESULT, + ddr_dst, + row_bytes, + row_bytes, + rows_total, + row_bytes + ); + __end_shared(); + __end_plan(0); +#else + ggml_gtx_compute_rope_cpu( + ddr_src0, ddr_src1, ddr_dst, + ne0, ne1, ne2, ne3, n_dims, mode, + freq_base_b, freq_scale_b, attn_factor_b, + nest_num, spu_num_per_nest); +#endif +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-scale.c b/depend/ggml-gtx/src/ggml-gtx-kernel-scale.c new file mode 100644 index 0000000..91a93d8 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-scale.c @@ -0,0 +1,126 @@ +//================================================================== +// GGML GTX kernel: SCALE, L2-stashed single-nest tile +// +// dst[i] = src0[i] * scale + bias +//================================================================== + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +static void ggml_gtx_compute_scale_l2_n1( + uint32_t l2_a, + uint32_t l2_c, + uint16_t elems, + uint16_t scale, + uint16_t bias, + uint8_t nest_id, + uint8_t spu_num_per_nest) +{ +#ifdef GGML_GTX_USE_GTX + const uint32_t dtype_size = GGML_GTX_DTYPE_SIZE; + const uint16_t not_use = 0xBEEF; + uint16_t spu_count = elems < spu_num_per_nest ? elems : (uint16_t) spu_num_per_nest; + if (spu_count == 0) { + spu_count = 1; + } + + uint16_t elems_per_spu[GGML_GTX_SPU_NUM_PER_NEST]; + const uint16_t elem_quot = (uint16_t) (elems / spu_count); + const uint16_t elem_mod = (uint16_t) (elems % spu_count); + for (uint16_t thread_id = 0; thread_id < GGML_GTX_SPU_NUM_PER_NEST; ++thread_id) { + elems_per_spu[thread_id] = + thread_id < spu_count ? (uint16_t) (elem_quot + (thread_id < elem_mod ? 1 : 0)) : 0; + } + + uint32_t elem_offset = 0; + + __split(); + + __start_plan(nest_id); + + __set_spm_addr(GGML_GTX_L1_BANK_R_OFFSET, + GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, + GGML_GTX_L1_BANK_A_OFFSET); + + for (uint16_t thread_id = 0; thread_id < spu_count; ++thread_id) { + const uint16_t thread_elems = elems_per_spu[thread_id]; + const uint32_t byte_offset = elem_offset * dtype_size; + + __start_thread(thread_id); + + __load(l2_a + byte_offset, + GGML_GTX_L1_BANK_A_OFFSET, + thread_elems, + thread_elems, + dtype_size, + not_use); + + __fmadd_vss(thread_elems, scale, bias, 0u); + + __store(GGML_GTX_L1_BANK_R_OFFSET, + l2_c + byte_offset, + thread_elems, + thread_elems, + dtype_size, + not_use); + + __end_thread(thread_id); + + elem_offset += thread_elems; + } + + __end_plan(nest_id); + + __join(); +#else + (void) l2_a; + (void) l2_c; + (void) elems; + (void) scale; + (void) bias; + (void) nest_id; + (void) spu_num_per_nest; +#endif +} + +int ggml_gtx_kernel_scale_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_SCALE_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_SCALE_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + const uint32_t elems = desc->ne[0]; + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_L2_STASHED_N1 || + desc->nest_id >= GGML_GTX_NEST_NUM || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->elem_size != GGML_GTX_DTYPE_SIZE || + elems == 0 || + elems > UINT16_MAX) { + return -2; + } + + const uint16_t scale = ggml_gtx_fp32_bits_to_fp16((uint32_t) desc->op_params[0]); + const uint16_t bias = ggml_gtx_fp32_bits_to_fp16((uint32_t) desc->op_params[1]); + ggml_gtx_compute_scale_l2_n1( + (uint32_t) desc->l2_src_addr[0], + (uint32_t) desc->l2_dst_addr, + (uint16_t) elems, + scale, + bias, + (uint8_t) desc->nest_id, + GGML_GTX_SPU_NUM_PER_NEST + ); + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_scale_f16(arg); +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-set-rows.c b/depend/ggml-gtx/src/ggml-gtx-kernel-set-rows.c new file mode 100644 index 0000000..44073be --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-set-rows.c @@ -0,0 +1,204 @@ +//================================================================== +// GGML GTX kernel: SET_ROWS +//================================================================== + +#include +#include +#include + +#include "ggml-gtx-baremetal.h" +#include "ggml-gtx-compute.h" +#include "ggml-gtx-port.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#ifdef GGML_GTX_USE_GTX +#include "ggml-gtx-plan.h" +#include "ggml-gtx-config.h" +#include "ggml-gtx/intrinsics/intrin.h" +#endif + +static int ggml_gtx_compute_set_rows_l2_n1(const ggml_gtx_kernel_launch_desc_t * desc) { +#ifdef GGML_GTX_USE_GTX + if (desc == NULL || + desc->l2_src_addr[0] == 0u || + desc->l2_dst_addr == 0u || + desc->src_addr[1] == 0u || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->dtype != GGML_GTX_TYPE_F16 || + (desc->src_type[1] != GGML_GTX_TYPE_I32 && desc->src_type[1] != GGML_GTX_TYPE_I64) || + desc->ne[0] == 0u || desc->ne[1] == 0u || + desc->ne[2] == 0u || desc->ne[3] == 0u || + desc->src_ne[0][0] != desc->ne[0] || + desc->src_ne[0][1] == 0u || + desc->src_ne[0][2] != desc->ne[2] || + desc->src_ne[0][3] != desc->ne[3] || + desc->src_ne[0][1] > desc->ne[1]) { + return -1; + } + + const uint32_t row_bytes = desc->ne[0] * sizeof(uint16_t); + if (row_bytes == 0u || row_bytes > UINT16_MAX) { + return -2; + } + + const uint32_t nr = desc->src_ne[0][1]; + const uint32_t ne2 = desc->ne[2]; + const uint32_t ne3 = desc->ne[3]; + const uint32_t n_vocab = desc->ne[1]; + const uint32_t ne_idx1 = desc->src_ne[1][1] == 0u ? 1u : desc->src_ne[1][1]; + const uint32_t ne_idx2 = desc->src_ne[1][2] == 0u ? 1u : desc->src_ne[1][2]; + const uint32_t idx_size = desc->src_type[1] == GGML_GTX_TYPE_I64 ? 8u : 4u; + const uint64_t l2_src = desc->l2_src_addr[0]; + const uint64_t l2_dst = desc->l2_dst_addr; + + __start_plan(desc->nest_id); + __start_shared(); + for (uint32_t i3 = 0; i3 < ne3; ++i3) { + const uint32_t sel_batch = i3 % ne_idx2; + for (uint32_t i2 = 0; i2 < ne2; ++i2) { + const uint32_t sel_channel = i2 % ne_idx1; + for (uint32_t i = 0; i < nr; ++i) { + const uint64_t selected_index = + ((uint64_t)sel_batch * ne_idx1 + sel_channel) * nr + i; + const uint64_t row = idx_size == 8u + ? ((volatile const uint64_t *)(uintptr_t)desc->src_addr[1])[selected_index] + : ((volatile const uint32_t *)(uintptr_t)desc->src_addr[1])[selected_index]; + if (row >= n_vocab) { + __end_shared(); + __end_plan(desc->nest_id); + return -3; + } + + const uint64_t src_off = + (((uint64_t)i3 * ne2 + i2) * nr + i) * row_bytes; + const uint64_t dst_off = + (((uint64_t)i3 * ne2 + i2) * n_vocab + row) * row_bytes; + __copy((uint32_t)(l2_src + src_off), + (uint32_t)(l2_dst + dst_off), + row_bytes, row_bytes, 1u, (uint16_t)row_bytes); + } + } + } + __end_shared(); + __end_plan(desc->nest_id); + return 0; +#else + (void)desc; + return -1; +#endif +} + +int ggml_gtx_kernel_set_rows_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_SET_ROWS_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_SET_ROWS_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + if (desc->memory_mode == GGML_GTX_MEMORY_MODE_L2_STASHED_N1) { + return ggml_gtx_compute_set_rows_l2_n1(desc); + } + + if (desc->dst_addr == 0 || + desc->src_addr[0] == 0 || + desc->src_addr[1] == 0 || + desc->src_addr[2] == 0) { + return -2; + } + + ggml_gtx_compute_set_rows(desc->src_addr[0], + desc->src_addr[1], + desc->src_addr[2], + desc->dst_addr, + (uint16_t) desc->src_ne[0][0], + (uint16_t) desc->src_ne[0][1], + (uint16_t) desc->ne[2], + (uint16_t) desc->ne[3], + (uint16_t) desc->ne[1], + (uint16_t) desc->src_ne[1][1], + (uint16_t) desc->src_ne[1][2], + desc->src_type[1] == GGML_GTX_TYPE_I64 ? 8 : 4); + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_set_rows_f16(arg); +} + +void ggml_gtx_compute_set_rows( + uint64_t ddr_data, + uint64_t ddr_idx, + uint64_t ddr_src_base, + uint64_t ddr_dst, + uint16_t nc, + uint16_t nr, + uint16_t ne2, + uint16_t ne3, + uint16_t n_vocab, + uint16_t ne_idx1, + uint16_t ne_idx2, + uint16_t idx_size) +{ + const uint16_t dtype_size = 2; + const uint32_t row_bytes = (uint32_t)nc * dtype_size; + const uint32_t total_rows = (uint32_t)n_vocab * (uint32_t)ne2 * (uint32_t)ne3; + if (ddr_src_base != ddr_dst) { + __copy_mem( + ddr_src_base, + ddr_dst, + row_bytes, + row_bytes, + total_rows, + row_bytes, + row_bytes >> 16); + } + + for (uint16_t i3 = 0; i3 < ne3; i3++) { + const uint16_t sel_batch = ne_idx2 != 0 ? (uint16_t)(i3 % ne_idx2) : 0; + + for (uint16_t i2 = 0; i2 < ne2; i2++) { + const uint16_t sel_channel = ne_idx1 != 0 ? (uint16_t)(i2 % ne_idx1) : 0; + + for (uint16_t i = 0; i < nr; i++) { + const uint16_t sel_height = i; + const uint64_t selected_index = (uint64_t)((sel_batch * ne_idx1 + sel_channel) * nr + sel_height); + const uint64_t row = (idx_size == 8) + ? ((volatile uint64_t *)(uintptr_t)ddr_idx)[selected_index] + : ((volatile uint32_t *)(uintptr_t)ddr_idx)[selected_index]; + + __copy_mem( + ddr_data + (((uint64_t)i3 * ne2 + i2) * nr + i) * row_bytes, + ddr_dst + (((uint64_t)i3 * ne2 + i2) * n_vocab + row) * row_bytes, + row_bytes, + row_bytes, + 1, + row_bytes, + row_bytes >> 16); + } + } + } + +} + +//================================================================== +// OUT_PROD: Outer product (used in LoRA weight updates) +// +// src0: F32 [m, n, q0, r0] (a) +// src1: F32 [p, n, q1, r1] (b) +// dst: F32 [m, p, q1, r1] +// +// Math: dst = src0 @ src1^T +// For each batch slice (i2, i3): +// for i1 in [0, p): +// for i01 in [0, n): +// for i0 in [0, m): +// dst[i0, i1, i2, i3] += src0[i0, i01, i02, i03] * src1[i1, i01, i2, i3] +// +// Broadcasting: src0 is broadcastable into src1 on dims 2,3 +// i02 = i2 / dps2, i03 = i3 / dps3 +//================================================================== diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-sgn.c b/depend/ggml-gtx/src/ggml-gtx-kernel-sgn.c new file mode 100644 index 0000000..7b19207 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-sgn.c @@ -0,0 +1,6 @@ +#define GGML_GTX_UNARY_KERNEL_ID GGML_GTX_KERNEL_SGN_F16 +#define GGML_GTX_UNARY_KERNEL_FN ggml_gtx_kernel_sgn_f16 +#define GGML_GTX_UNARY_INPUT_BANK GGML_GTX_UNARY_BANK_A +#define GGML_GTX_UNARY_OUTPUT_BANK GGML_GTX_UNARY_BANK_R +#define GGML_GTX_UNARY_APPLY(elems) __sign_v(elems) +#include "ggml-gtx-kernel-unary-ref.h" diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-silu.c b/depend/ggml-gtx/src/ggml-gtx-kernel-silu.c new file mode 100644 index 0000000..12841e0 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-silu.c @@ -0,0 +1,6 @@ +#define GGML_GTX_UNARY_KERNEL_ID GGML_GTX_KERNEL_SILU_F16 +#define GGML_GTX_UNARY_KERNEL_FN ggml_gtx_kernel_silu_f16 +#define GGML_GTX_UNARY_INPUT_BANK GGML_GTX_UNARY_BANK_R +#define GGML_GTX_UNARY_OUTPUT_BANK GGML_GTX_UNARY_BANK_A +#define GGML_GTX_UNARY_APPLY(elems) __silu(elems, GGML_GTX_UNARY_BANK_A, GGML_GTX_UNARY_BANK_B, GGML_GTX_UNARY_BANK_R) +#include "ggml-gtx-kernel-unary-ref.h" diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-sin-core.h b/depend/ggml-gtx/src/ggml-gtx-kernel-sin-core.h new file mode 100644 index 0000000..6cd7a3f --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-sin-core.h @@ -0,0 +1,261 @@ +//================================================================== +// Copyright : (C) Supergate - All Rights Reserved +// Project : GSF / VTS +// Test : timestep_embedding +// Description : GTX timestep_embedding kernel +// Author : kyumlee ( NPU Div - NPU Core Design Team ) +// Last Update : 2026/02/26 +//================================================================== + + +// Intrinsic header file +#include "gtx/intrin.h" +#include "gtx/print.h" +#include "gtx_verify.h" +#include "gtx_test.h" +//#include "gtx_ckernel_param.h" +#include + +#define NEST_TARGET_ID 0 // Target nest ID +#define SPU_NUM_PER_NEST 16 // SPU num per nest +#define SINCOS_DTYPE 2 // FP16 + +#define SINCOS_LEGACY_L2_A 0x20000 +#define SINCOS_LEGACY_L2_B 0x40000 + +#define SINCOS_BANK_A 0X00000 +#define SINCOS_BANK_B 0X20000 +#define SINCOS_BANK_C 0X30000 +#define SINCOS_BANK_R 0X50000 + +#define SINCOS_SVR_ADDR 0x800 + +static void sincos_vec_impl( + uint8_t NEST_NUM, + uint8_t SPU_NUM, + uint64_t DDR_SRC_ADDR, + uint64_t DDR_DST_ADDR, + uint8_t SINCOS_SEL, + uint32_t SINCOS_L2_A, + uint32_t SINCOS_L2_B, + uint8_t L2_ONLY +){ + /* + if(SINCOS_SEL < 0 || SINCOS_SEL > 1){ + printf("\n[SINCOS] SEL ERROR\n"); + return 0; + } + */ + uint8_t nest_id = NEST_NUM; + uint8_t thread_id = SPU_NUM; + uint16_t vec_size = 16 * SINCOS_DTYPE; + + __start_plan(nest_id); + + __start_shared(); + if (L2_ONLY) { + __copy((uint32_t)DDR_SRC_ADDR, SINCOS_L2_A, + vec_size, vec_size, 1, vec_size); + } else { + __load(DDR_SRC_ADDR, SINCOS_L2_A, vec_size, vec_size, 1, vec_size); + } + __end_shared(); + + __end_plan(nest_id); + + __start_plan(nest_id); + + __start_thread(thread_id); + __load(SINCOS_L2_A, SINCOS_BANK_A, vec_size, vec_size, 1, vec_size); + + //x to svr, m = round(x/2pi) + __load_svr(SINCOS_BANK_A,0); + __mul_is(0,0x3118,0,0); + __rne_i(0,0); + + //make two vector[x,m,m,m]. [1,-2pi_1, -2pi_2, -2pi_3] + __wrspr(SINCOS_SVR_ADDR+4, 0, + 0x012BC248C2493C00, 0); + __store_svr(SINCOS_BANK_A+vec_size, 0); + __store_svr(SINCOS_BANK_B, 1); + + __store(SINCOS_BANK_A, SINCOS_L2_A, + vec_size, + vec_size, + 1, + vec_size); + for(int i=0; i<3; i++){ + __store(SINCOS_BANK_A + vec_size, SINCOS_L2_A + (i+1) * vec_size, + vec_size, + vec_size, + 1, + vec_size); + } + __end_thread(thread_id); + __end_plan(nest_id); + + __start_plan(nest_id); + __start_shared(); + __tpose(SINCOS_L2_A, SINCOS_L2_B, + 16, 4, 4, 1, 2, 0, 0b00, 1); + __end_shared(); + __end_plan(nest_id); + + __start_plan(nest_id); + __start_thread(thread_id); + __load(SINCOS_L2_B, SINCOS_BANK_C, vec_size, vec_size, 4, vec_size); + + //matmul two vector (= x0) + __set_spm_addr(SINCOS_BANK_R, SINCOS_BANK_A, SINCOS_BANK_B, SINCOS_BANK_C); + __mm(16, 4, 1); + + //x0 to svr, k = round(x0/pi) + __load_svr(SINCOS_BANK_R, 0); + __mul_is(0,0x3518, 0,0); + __rne_i(0,0); + + //make two vector[x0,k,k,k]. [1,-pi_1, -pi_2, -pi_3] + __wrspr(SINCOS_SVR_ADDR+4, 0, + 0x0095BE48BE493C00, 0); + __store_svr(SINCOS_BANK_R+vec_size, 0); + __store_svr(SINCOS_BANK_B, 1); + + __store(SINCOS_BANK_R, SINCOS_L2_A, + vec_size, + vec_size, + 1, + vec_size); + for(int i=0; i<3; i++){ + __store(SINCOS_BANK_R + vec_size, SINCOS_L2_A + (i+1) * vec_size, + vec_size, + vec_size, + 1, + vec_size); + } + + __end_thread(thread_id); + __end_plan(nest_id); + + __start_plan(nest_id); + __start_shared(); + __tpose(SINCOS_L2_A, SINCOS_L2_B, + 16, 4, 4, 1, 2, 0, 0b00, 1); + __end_shared(); + __end_plan(nest_id); + + __start_plan(nest_id); + __start_thread(thread_id); + __load(SINCOS_L2_B, SINCOS_BANK_C, vec_size, vec_size, 4, vec_size); + + //matmul two vector (= r) + __mm(16, 4, 1); + + //k transform to -1(odd) or 1(even) + __mul_is(0, 0x3800, 1, 0); + __ceil_i(1,1); + __mul_is(1, 0x4000, 1, 0); + __sub_ii(0,1,1); + __add_is(1, 0x3C00, 2, 0); + __add_ii(1,2,0); + __store_svr(SINCOS_BANK_C, 0); + + //maclaurin + __load_svr(SINCOS_BANK_R, 1); + __mvsvr(1,2,0); + __mul_ii(1,2,2); + + if(SINCOS_SEL){ + __wrspr(SINCOS_SVR_ADDR+4, 0, 0x3C003C003C003C00, 0); + __cpsvr(1,3); + __store_svr(SINCOS_BANK_R,1); + } + + for(int i=0; i<3+SINCOS_SEL; i++){ + __mul_ii(1,2,1); + __store_svr(SINCOS_BANK_R+(i+1)*vec_size,1); + } + + __store(SINCOS_BANK_R, SINCOS_L2_A, + vec_size, + vec_size, + 4+SINCOS_SEL, + vec_size); + + __end_thread(thread_id); + __end_plan(nest_id); + + __start_plan(nest_id); + __start_shared(); + __tpose(SINCOS_L2_A, SINCOS_L2_B, + 16, 4+SINCOS_SEL, 4, 1, 2, 0, 0b00, 1); + __end_shared(); + __end_plan(nest_id); + + __start_plan(nest_id); + __start_thread(thread_id); + __load(SINCOS_L2_B, SINCOS_BANK_A, vec_size, vec_size, 4+SINCOS_SEL, vec_size); + + if(SINCOS_SEL){ + __copy(SINCOS_BANK_R, SINCOS_BANK_B, + SINCOS_DTYPE,SINCOS_DTYPE,1,SINCOS_DTYPE); + __wrspr(SINCOS_SVR_ADDR+4, 0, 0x01A095B02955B800,0); + __store_svr(SINCOS_BANK_B+vec_size,1); + __copy(SINCOS_BANK_B+vec_size, SINCOS_BANK_B+SINCOS_DTYPE, + 4*SINCOS_DTYPE, 4*SINCOS_DTYPE, 1, SINCOS_DTYPE); + } + else { + __wrspr(SINCOS_SVR_ADDR+4, 0, 0x8A802044B1553C00,0); + __store_svr(SINCOS_BANK_B,1); + } + + __set_spm_addr(SINCOS_BANK_R,SINCOS_BANK_C,SINCOS_BANK_B,SINCOS_BANK_A); + __mm(16,4+SINCOS_SEL,1); + + // k transform * maclaurin + __set_spm_addr(SINCOS_BANK_A,SINCOS_BANK_B,SINCOS_BANK_C,SINCOS_BANK_R); + __mul_vv(16); + + __store(SINCOS_BANK_A, SINCOS_L2_A, vec_size, vec_size, 1, vec_size); + __end_thread(thread_id); + + __end_plan(nest_id); + + __start_plan(nest_id); + + __start_shared(); + if (L2_ONLY) { + __copy(SINCOS_L2_A, (uint32_t)DDR_DST_ADDR, + vec_size, vec_size, 1, vec_size); + } else { + __store(SINCOS_L2_A, DDR_DST_ADDR, vec_size, vec_size, 1, vec_size); + } + __end_shared(); + + __end_plan(nest_id); + + return; +} + +void sincos_vec_l2( + uint8_t NEST_NUM, + uint8_t SPU_NUM, + uint32_t L2_SRC_ADDR, + uint32_t L2_DST_ADDR, + uint8_t SINCOS_SEL, + uint32_t SINCOS_L2_A, + uint32_t SINCOS_L2_B +){ + sincos_vec_impl(NEST_NUM, SPU_NUM, L2_SRC_ADDR, L2_DST_ADDR, SINCOS_SEL, + SINCOS_L2_A, SINCOS_L2_B, 1); +} + +void sincos_vec( + uint8_t NEST_NUM, + uint8_t SPU_NUM, + uint64_t DDR_SRC_ADDR, + uint64_t DDR_DST_ADDR, + uint8_t SINCOS_SEL +){ + sincos_vec_impl(NEST_NUM, SPU_NUM, DDR_SRC_ADDR, DDR_DST_ADDR, SINCOS_SEL, + SINCOS_LEGACY_L2_A, SINCOS_LEGACY_L2_B, 0); +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-sin.c b/depend/ggml-gtx/src/ggml-gtx-kernel-sin.c new file mode 100644 index 0000000..1e868ad --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-sin.c @@ -0,0 +1,71 @@ +//================================================================== +// GGML GTX kernel: SIN +// +// Descriptor wrapper around the host-owned GTX sine core. +//================================================================== + +#include + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#include "ggml-gtx-kernel-sin-core.h" + +int ggml_gtx_kernel_sin_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_SIN_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_SIN_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_L2_STASHED_N1 || + desc->l2_dst_addr == 0 || + desc->l2_src_addr[0] == 0 || + desc->l2_scratch_capacity < 0x40200u || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->nb[0] != SINCOS_DTYPE || + desc->src_nb[0][0] != SINCOS_DTYPE) { + return -2; + } + + const uint64_t elems64 = (uint64_t) desc->ne[0] * + (uint64_t) desc->ne[1] * + (uint64_t) desc->ne[2] * + (uint64_t) desc->ne[3]; + if (elems64 == 0 || (elems64 % 16u) != 0u) { + return -3; + } + + for (int i = 0; i < 4; ++i) { + if (desc->src_ne[0][i] != desc->ne[i]) { + return -4; + } + } + if (desc->nb[1] != desc->ne[0] * SINCOS_DTYPE || + desc->src_nb[0][1] != desc->src_ne[0][0] * SINCOS_DTYPE || + (desc->ne[2] > 1 && desc->nb[2] != desc->nb[1] * desc->ne[1]) || + (desc->ne[3] > 1 && desc->nb[3] != desc->nb[2] * desc->ne[2]) || + (desc->src_ne[0][2] > 1 && desc->src_nb[0][2] != desc->src_nb[0][1] * desc->src_ne[0][1]) || + (desc->src_ne[0][3] > 1 && desc->src_nb[0][3] != desc->src_nb[0][2] * desc->src_ne[0][2])) { + return -5; + } + + for (uint64_t elem = 0; elem < elems64; elem += 16u) { + const uint64_t off = elem * SINCOS_DTYPE; + sincos_vec_l2(0, 0, desc->l2_src_addr[0] + off, desc->l2_dst_addr + off, 0, + desc->l2_scratch_addr + 0x20000u, + desc->l2_scratch_addr + 0x40000u); + } + + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_sin_f16(arg); +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-soft-max.c b/depend/ggml-gtx/src/ggml-gtx-kernel-soft-max.c new file mode 100644 index 0000000..920331f --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-soft-max.c @@ -0,0 +1,690 @@ +//================================================================== +// GGML GTX kernel: SOFT_MAX +//================================================================== + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +static float ggml_gtx_softmax_bits_to_float(uint32_t bits); +static uint32_t ggml_gtx_softmax_float_to_bits(float value); +static float ggml_gtx_softmax_expf(float arg); + +#define GGML_GTX_SOFTMAX_L1_A_BYTES (128u * 1024u) +#define GGML_GTX_SOFTMAX_L1_B_BYTES (64u * 1024u) + +int ggml_gtx_kernel_soft_max_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_SOFT_MAX_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_SOFT_MAX_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + const uint64_t rows64 = (uint64_t) desc->ne[1] * + (uint64_t) desc->ne[2] * + (uint64_t) desc->ne[3]; + if (desc->dst_addr == 0 || + desc->src_addr[0] == 0 || + desc->ne[0] == 0 || + rows64 == 0 || + desc->ne[0] > UINT16_MAX) { + return -2; + } + + const float scale = ggml_gtx_softmax_bits_to_float((uint32_t) desc->op_params[0]); + const float max_bias = ggml_gtx_softmax_bits_to_float((uint32_t) desc->op_params[1]); + + if (desc->memory_mode == GGML_GTX_MEMORY_MODE_L2_STASHED_N1) { + const uint64_t row_bytes = (uint64_t) desc->ne[0] * sizeof(uint16_t); + if (desc->nest_id >= GGML_GTX_NEST_NUM || + desc->src_addr[1] != 0 || + scale != 1.0f || + max_bias != 0.0f || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->elem_size != GGML_GTX_DTYPE_SIZE || + desc->l2_src_addr[0] == 0 || + row_bytes == 0 || + row_bytes > GGML_GTX_L1_BANK_SIZE || + rows64 > UINT16_MAX) { + return -3; + } + + ggml_gtx_compute_softmax_l2((uint32_t) desc->l2_src_addr[0], + (uint32_t) desc->l2_dst_addr, + (uint16_t) rows64, + (uint16_t) desc->ne[0], + 1u, + GGML_GTX_SPU_NUM_PER_NEST); + return 0; + } + + if (desc->src_addr[1] == 0 && scale == 1.0f && max_bias == 0.0f) { + const uint64_t row_bytes = (uint64_t) desc->ne[0] * sizeof(uint16_t); + uint64_t row_base = 0; + while (row_base < rows64) { + const uint64_t remaining = rows64 - row_base; + const uint16_t chunk_rows = (uint16_t) + (remaining > UINT16_MAX ? UINT16_MAX : remaining); + const uint64_t byte_off = row_base * row_bytes; + ggml_gtx_compute_softmax(desc->src_addr[0] + byte_off, + desc->dst_addr + byte_off, + chunk_rows, + (uint16_t) desc->ne[0], + GGML_GTX_NEST_NUM, + GGML_GTX_SPU_NUM_PER_NEST); + row_base += chunk_rows; + } + } else { + if (desc->ne[1] > UINT16_MAX || + desc->ne[2] > UINT16_MAX || + desc->ne[3] > UINT16_MAX || + desc->src_ne[1][2] > UINT16_MAX || + desc->src_ne[1][3] > UINT16_MAX) { + return -3; + } + ggml_gtx_compute_softmax_ext(desc->src_addr[0], + desc->src_addr[1], + desc->dst_addr, + (uint16_t) desc->ne[0], + (uint16_t) desc->ne[1], + (uint16_t) desc->ne[2], + (uint16_t) desc->ne[3], + desc->nb[1], + desc->nb[2], + desc->nb[3], + (uint16_t) desc->src_ne[1][2], + (uint16_t) desc->src_ne[1][3], + desc->src_nb[1][1], + desc->src_nb[1][2], + desc->src_nb[1][3], + scale, + max_bias); + } + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_soft_max_f16(arg); +} + +static float ggml_gtx_softmax_bits_to_float(uint32_t bits) { + union { uint32_t u; float f; } value_bits; + value_bits.u = bits; + return value_bits.f; +} + +static uint32_t ggml_gtx_softmax_float_to_bits(float value) { + union { uint32_t u; float f; } value_bits; + value_bits.f = value; + return value_bits.u; +} + +static float ggml_gtx_softmax_expf(float arg) { + if (arg < -20.0f) { + return 0.0f; + } + if (arg > 20.0f) { + arg = 20.0f; + } + + const float ln2 = 0.69314718056f; + int exponent = (int)(arg / ln2); + float reduced = arg - ((float) exponent * ln2); + + if (reduced < -0.5f * ln2) { + reduced += ln2; + --exponent; + } else if (reduced > 0.5f * ln2) { + reduced -= ln2; + ++exponent; + } + + const float reduced_sq = reduced * reduced; + const float poly = 1.0f + reduced + reduced_sq * 0.5f + + reduced * reduced_sq * (1.0f / 6.0f) + + reduced_sq * reduced_sq * (1.0f / 24.0f); + + if (exponent > 30) { + exponent = 30; + } + if (exponent < -30) { + return 0.0f; + } + + return exponent >= 0 ? poly * (float)(1u << exponent) + : poly / (float)(1u << -exponent); +} + +void ggml_gtx_compute_softmax( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest) +{ +#ifdef GGML_GTX_USE_GTX + // Following reference_usage.c __n4_softmax__ pattern + ggml_gtx_tile_plan_t plan; + ggml_gtx_tile_plan_init(&plan, i_row, i_col, nest_num, spu_num_per_nest); + + uint32_t offset_cal = 0; + + __split(); + + for (uint8_t nest_id = 0; nest_id < plan.eff_nest_int; nest_id++) { + ggml_gtx_nest_tile_t * nest = &plan.nests[nest_id]; + + if (nest->cal_num > 0) { + __start_plan(nest_id); + __start_shared(); + + // Load input tensor + for (uint8_t thread_id = 0; thread_id < nest->eff_thread_int; thread_id++) { + ggml_gtx_thread_tile_t * thread = &nest->threads[thread_id]; + __load_cr( + ddr_input + thread->offset, + GGML_GTX_L2_INPUT + GGML_GTX_L2_BANK_OFFSET * thread_id, + plan.unit_per_thread, + plan.unit_per_thread, + thread->cal_num, + plan.unit_per_thread, + 1, + 0x1 << thread_id, + 0xBEEF + ); + } + + // Store results + for (uint8_t thread_id = 0; thread_id < nest->eff_thread_int; thread_id++) { + ggml_gtx_thread_tile_t * thread = &nest->threads[thread_id]; + __credit_chk(0x1 << thread_id); + __store_cr( + GGML_GTX_L2_RESULT + GGML_GTX_L2_BANK_OFFSET * thread_id, + ddr_result + thread->offset, + plan.unit_per_thread, + plan.unit_per_thread, + thread->cal_num, + plan.unit_per_thread, + 1, + 0x1 << thread_id + ); + } + + __end_shared(); + + // Thread: L2SPM <-> L1SPM + Compute + for (uint8_t thread_id = 0; thread_id < nest->eff_thread_int; thread_id++) { + ggml_gtx_thread_tile_t * thread = &nest->threads[thread_id]; + + if (thread->cal_num > 0) { + __start_thread(thread_id); + + __set_spm_addr(GGML_GTX_L1_BANK_R_OFFSET, GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, GGML_GTX_L1_BANK_A_OFFSET); + + uint32_t l1_A_addr = GGML_GTX_L1_BANK_A_OFFSET; + uint32_t l1_R_addr = GGML_GTX_L1_BANK_R_OFFSET; + + __credit_chk(0xBEEF); + + offset_cal = 0; + + // Full iterations + for (uint16_t j = 0; j < thread->iteration; j++) { + uint32_t next_offset = GGML_GTX_L2_BANK_OFFSET * thread_id + offset_cal; + + __load( + GGML_GTX_L2_INPUT + next_offset, + GGML_GTX_L1_BANK_A_OFFSET, + plan.unit_per_thread, + plan.unit_per_thread, + plan.max_cal_per_thread, + 0xBEEF + ); + + // Process each row in the tile + for (uint16_t k = 0; k < plan.max_cal_per_thread; k++) { + // [max.vs] find max value (L1 A bank -> SVR_0) + // 0xFBFF = -inf in FP16 + __max_vs(i_col, 0xFBFF, 0x0, 0b0); + + // [esum] exponential sum (L1 A bank & SVR_0 -> SVR_1) + __esum(i_col, 0xBEEF, 0xBEEF, 0x1, 0x180); + + // [softmax] compute softmax (L1 A bank & SVR_1 -> L1 R bank) + __softmax(i_col, 0xBEEF, 0xBEEF, 0x184); + + // Advance bank pointers + l1_A_addr += plan.unit_per_thread; + l1_R_addr += plan.unit_per_thread; + __set_spm_addr_A(l1_A_addr); + __set_spm_addr_R(l1_R_addr); + } + + // Reset bank pointers + l1_A_addr = GGML_GTX_L1_BANK_A_OFFSET; + l1_R_addr = GGML_GTX_L1_BANK_R_OFFSET; + __set_spm_addr_A(GGML_GTX_L1_BANK_A_OFFSET); + __set_spm_addr_R(GGML_GTX_L1_BANK_R_OFFSET); + + __store( + GGML_GTX_L1_BANK_R_OFFSET, + GGML_GTX_L2_RESULT + next_offset, + plan.unit_per_thread, + plan.unit_per_thread, + plan.max_cal_per_thread, + 0xBEEF + ); + + offset_cal += plan.unit_per_thread * plan.max_cal_per_thread; + } + + // Remaining elements + if (thread->remain > 0) { + uint32_t last_offset = GGML_GTX_L2_BANK_OFFSET * thread_id + offset_cal; + + __load_cr( + GGML_GTX_L2_INPUT + last_offset, + GGML_GTX_L1_BANK_A_OFFSET, + plan.unit_per_thread, + plan.unit_per_thread, + thread->remain, + 0xBEEF, + 1, + 0xBEEF, + 0xBEEF + ); + + for (uint16_t k = 0; k < thread->remain; k++) { + __max_vs(i_col, 0xFBFF, 0x0, 0b0); + __esum(i_col, 0xBEEF, 0xBEEF, 0x1, 0x180); + __softmax(i_col, 0xBEEF, 0xBEEF, 0x184); + + l1_A_addr += plan.unit_per_thread; + l1_R_addr += plan.unit_per_thread; + __set_spm_addr_A(l1_A_addr); + __set_spm_addr_R(l1_R_addr); + } + + __set_spm_addr_A(GGML_GTX_L1_BANK_A_OFFSET); + __set_spm_addr_R(GGML_GTX_L1_BANK_R_OFFSET); + + __store_cr( + GGML_GTX_L1_BANK_R_OFFSET, + GGML_GTX_L2_RESULT + last_offset, + plan.unit_per_thread, + plan.unit_per_thread, + thread->remain, + 0xBEEF, + 1, + 0xBEEF + ); + } + + offset_cal = 0; + + __end_thread(thread_id); + } + } + + __end_plan(nest_id); + } + } + + __join(); +#else + (void)ddr_input; (void)ddr_result; + (void)i_row; (void)i_col; + (void)nest_num; (void)spu_num_per_nest; +#endif +} + +void ggml_gtx_compute_softmax_l2( + uint32_t l2_input, + uint32_t l2_result, + uint16_t i_row, + uint16_t i_col, + uint8_t nest_num, + uint8_t spu_num_per_nest) +{ +#ifdef GGML_GTX_USE_GTX + if (i_row == 0 || i_col == 0 || nest_num == 0 || spu_num_per_nest == 0) { + return; + } + + ggml_gtx_tile_plan_t plan; + ggml_gtx_tile_plan_init(&plan, i_row, i_col, nest_num, spu_num_per_nest); + if (plan.max_cal_per_thread == 0) { + return; + } + + __split(); + + for (uint8_t nest_id = 0; nest_id < plan.eff_nest_int; nest_id++) { + ggml_gtx_nest_tile_t * nest = &plan.nests[nest_id]; + + if (nest->cal_num > 0) { + __start_plan(nest_id); + + for (uint8_t thread_id = 0; thread_id < nest->eff_thread_int; thread_id++) { + ggml_gtx_thread_tile_t * thread = &nest->threads[thread_id]; + + if (thread->cal_num > 0) { + __start_thread(thread_id); + + __set_spm_addr(GGML_GTX_L1_BANK_R_OFFSET, GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, GGML_GTX_L1_BANK_A_OFFSET); + + uint32_t l1_A_addr = GGML_GTX_L1_BANK_A_OFFSET; + uint32_t l1_R_addr = GGML_GTX_L1_BANK_R_OFFSET; + uint32_t offset_cal = 0; + + for (uint16_t j = 0; j < thread->iteration; j++) { + uint32_t next_offset = thread->offset + offset_cal; + + __load( + l2_input + next_offset, + GGML_GTX_L1_BANK_A_OFFSET, + plan.unit_per_thread, + plan.unit_per_thread, + plan.max_cal_per_thread, + 0xBEEF + ); + + for (uint16_t k = 0; k < plan.max_cal_per_thread; k++) { + __max_vs(i_col, 0xFBFF, 0x0, 0b0); + __esum(i_col, 0xBEEF, 0xBEEF, 0x1, 0x180); + __softmax(i_col, 0xBEEF, 0xBEEF, 0x184); + + l1_A_addr += plan.unit_per_thread; + l1_R_addr += plan.unit_per_thread; + __set_spm_addr_A(l1_A_addr); + __set_spm_addr_R(l1_R_addr); + } + + l1_A_addr = GGML_GTX_L1_BANK_A_OFFSET; + l1_R_addr = GGML_GTX_L1_BANK_R_OFFSET; + __set_spm_addr_A(GGML_GTX_L1_BANK_A_OFFSET); + __set_spm_addr_R(GGML_GTX_L1_BANK_R_OFFSET); + + __store( + GGML_GTX_L1_BANK_R_OFFSET, + l2_result + next_offset, + plan.unit_per_thread, + plan.unit_per_thread, + plan.max_cal_per_thread, + 0xBEEF + ); + + offset_cal += plan.unit_per_thread * plan.max_cal_per_thread; + } + + if (thread->remain > 0) { + uint32_t last_offset = thread->offset + offset_cal; + + __load( + l2_input + last_offset, + GGML_GTX_L1_BANK_A_OFFSET, + plan.unit_per_thread, + plan.unit_per_thread, + thread->remain, + 0xBEEF + ); + + for (uint16_t k = 0; k < thread->remain; k++) { + __max_vs(i_col, 0xFBFF, 0x0, 0b0); + __esum(i_col, 0xBEEF, 0xBEEF, 0x1, 0x180); + __softmax(i_col, 0xBEEF, 0xBEEF, 0x184); + + l1_A_addr += plan.unit_per_thread; + l1_R_addr += plan.unit_per_thread; + __set_spm_addr_A(l1_A_addr); + __set_spm_addr_R(l1_R_addr); + } + + __set_spm_addr_A(GGML_GTX_L1_BANK_A_OFFSET); + __set_spm_addr_R(GGML_GTX_L1_BANK_R_OFFSET); + + __store( + GGML_GTX_L1_BANK_R_OFFSET, + l2_result + last_offset, + plan.unit_per_thread, + plan.unit_per_thread, + thread->remain, + 0xBEEF + ); + } + + __end_thread(thread_id); + } + } + + __end_plan(nest_id); + } + } + + __join(); +#else + (void)l2_input; (void)l2_result; + (void)i_row; (void)i_col; + (void)nest_num; (void)spu_num_per_nest; +#endif +} + +#ifdef GGML_GTX_USE_GTX +static void ggml_gtx_compute_softmax_ext_npu_plane( + uint64_t ddr_input, + uint64_t ddr_mask, + uint64_t ddr_result, + uint16_t rows, + uint16_t cols, + float scale) +{ + const uint32_t row_bytes = (uint32_t)cols * sizeof(uint16_t); + const uint32_t l1_limit = ddr_mask != 0u ? + GGML_GTX_SOFTMAX_L1_B_BYTES : GGML_GTX_SOFTMAX_L1_A_BYTES; + uint32_t max_rows = l1_limit / row_bytes; + const uint32_t dma_rows = UINT16_MAX / row_bytes; + if (max_rows > dma_rows) { + max_rows = dma_rows; + } + if (max_rows == 0u) { + return; + } + + const uint16_t scale_f16 = ggml_gtx_fp32_bits_to_fp16( + ggml_gtx_softmax_float_to_bits(scale)); + + for (uint32_t row0 = 0u; row0 < rows; row0 += max_rows) { + const uint16_t block_rows = (uint16_t) + (((uint32_t)rows - row0) < max_rows ? (uint32_t)rows - row0 : max_rows); + const uint64_t byte_off = (uint64_t)row0 * row_bytes; + + /* Keep the three asynchronous domains in separate completed plans. + * FPGA credit chaining is intentionally not used here: each join is + * the completion boundary for the following phase. */ + __split(); + __start_plan(0u); + __start_shared(); + __load(ddr_input + byte_off, + GGML_GTX_L2_A, + row_bytes, row_bytes, block_rows, row_bytes); + if (ddr_mask != 0u) { + __load(ddr_mask + byte_off, + GGML_GTX_L2_B, + row_bytes, row_bytes, block_rows, row_bytes); + } + __end_shared(); + __end_plan(0u); + __join(); + + __split(); + __start_plan(0u); + __start_thread(0u); + /* The reference softmax intentionally aliases A and R. + * This makes scale/add results immediately visible as the + * next operation's A input without an R-to-A transfer. */ + __set_spm_addr(GGML_GTX_L1_BANK_A_OFFSET, + GGML_GTX_L1_BANK_C_OFFSET, + GGML_GTX_L1_BANK_B_OFFSET, + GGML_GTX_L1_BANK_A_OFFSET); + __load(GGML_GTX_L2_A, + GGML_GTX_L1_BANK_A_OFFSET, + row_bytes, (uint16_t)row_bytes, + block_rows, row_bytes); + if (ddr_mask != 0u) { + __load(GGML_GTX_L2_B, + GGML_GTX_L1_BANK_B_OFFSET, + row_bytes, (uint16_t)row_bytes, + block_rows, row_bytes); + } + + if (scale != 1.0f) { + __mul_vs((uint32_t)cols * block_rows, scale_f16, 0u); + } + if (ddr_mask != 0u) { + __add_vv((uint32_t)cols * block_rows); + } + + for (uint16_t row = 0u; row < block_rows; ++row) { + const uint32_t l1_off = (uint32_t)row * row_bytes; + __set_spm_addr_A(GGML_GTX_L1_BANK_A_OFFSET + l1_off); + __set_spm_addr_R(GGML_GTX_L1_BANK_A_OFFSET + l1_off); + __max_vs(cols, 0xFBFF, 0x0, 0x0); + __esum(cols, 0xBEEF, 0xBEEF, 0x1, 0x180); + __softmax(cols, 0xBEEF, 0xBEEF, 0x184); + } + __set_spm_addr_A(GGML_GTX_L1_BANK_A_OFFSET); + __set_spm_addr_R(GGML_GTX_L1_BANK_A_OFFSET); + __store(GGML_GTX_L1_BANK_A_OFFSET, + GGML_GTX_L2_RESULT, + row_bytes, (uint16_t)row_bytes, + block_rows, row_bytes); + __end_thread(0u); + __end_plan(0u); + __join(); + + __split(); + __start_plan(0u); + __start_shared(); + __store(GGML_GTX_L2_RESULT, + ddr_result + byte_off, + row_bytes, row_bytes, block_rows, row_bytes); + __end_shared(); + __end_plan(0u); + __join(); + } +} +#endif + +void ggml_gtx_compute_softmax_ext( + uint64_t ddr_input, + uint64_t ddr_mask, + uint64_t ddr_result, + uint16_t ne0, + uint16_t ne1, + uint16_t ne2, + uint16_t ne3, + uint32_t dst_nb1, + uint32_t dst_nb2, + uint32_t dst_nb3, + uint16_t mask_ne2, + uint16_t mask_ne3, + uint32_t mask_nb1, + uint32_t mask_nb2, + uint32_t mask_nb3, + float scale, + float max_bias) +{ + volatile uint16_t * src = (volatile uint16_t *)(uintptr_t)ddr_input; + volatile uint16_t * dst = (volatile uint16_t *)(uintptr_t)ddr_result; + volatile uint16_t * mask = (volatile uint16_t *)(uintptr_t)ddr_mask; + + const uint32_t src_nb1 = dst_nb1; + const uint32_t src_nb2 = dst_nb2; + const uint32_t src_nb3 = dst_nb3; + const uint16_t safe_mask_ne2 = mask_ne2 == 0 ? 1 : mask_ne2; + const uint16_t safe_mask_ne3 = mask_ne3 == 0 ? 1 : mask_ne3; + +#ifdef GGML_GTX_USE_GTX + if (max_bias == 0.0f && ne0 > 0u && + (uint32_t)ne0 * sizeof(uint16_t) <= GGML_GTX_SOFTMAX_L1_B_BYTES) { + for (uint16_t i3 = 0; i3 < ne3; ++i3) { + for (uint16_t i2 = 0; i2 < ne2; ++i2) { + const uint64_t src_plane = ddr_input + + (uint64_t)i2 * src_nb2 + (uint64_t)i3 * src_nb3; + const uint64_t dst_plane = ddr_result + + (uint64_t)i2 * dst_nb2 + (uint64_t)i3 * dst_nb3; + uint64_t mask_plane = 0u; + if (ddr_mask != 0u) { + const uint16_t mi2 = (uint16_t)(i2 % safe_mask_ne2); + const uint16_t mi3 = (uint16_t)(i3 % safe_mask_ne3); + mask_plane = ddr_mask + + (uint64_t)mi2 * mask_nb2 + (uint64_t)mi3 * mask_nb3; + } + ggml_gtx_compute_softmax_ext_npu_plane( + src_plane, mask_plane, dst_plane, ne1, ne0, scale); + } + } + return; + } +#endif + + for (uint16_t i3 = 0; i3 < ne3; ++i3) { + for (uint16_t i2 = 0; i2 < ne2; ++i2) { + for (uint16_t i1 = 0; i1 < ne1; ++i1) { + const uint64_t row_off = ((uint64_t)i1 * src_nb1 + + (uint64_t)i2 * src_nb2 + + (uint64_t)i3 * src_nb3) / 2U; + const uint64_t dst_off = ((uint64_t)i1 * dst_nb1 + + (uint64_t)i2 * dst_nb2 + + (uint64_t)i3 * dst_nb3) / 2U; + + uint64_t mask_off = 0; + if (ddr_mask != 0) { + const uint16_t mi2 = (uint16_t)(i2 % safe_mask_ne2); + const uint16_t mi3 = (uint16_t)(i3 % safe_mask_ne3); + mask_off = ((uint64_t)i1 * mask_nb1 + + (uint64_t)mi2 * mask_nb2 + + (uint64_t)mi3 * mask_nb3) / 2U; + } + + float row_max = -3.402823466e+38f; + for (uint16_t i0 = 0; i0 < ne0; ++i0) { + float v = ggml_gtx_softmax_bits_to_float(ggml_gtx_fp16_to_fp32_bits(src[row_off + i0])) * scale; + if (ddr_mask != 0) { + v += ggml_gtx_softmax_bits_to_float(ggml_gtx_fp16_to_fp32_bits(mask[mask_off + i0])); + } + if (v > row_max) { + row_max = v; + } + } + + float sum = 0.0f; + for (uint16_t i0 = 0; i0 < ne0; ++i0) { + float v = ggml_gtx_softmax_bits_to_float(ggml_gtx_fp16_to_fp32_bits(src[row_off + i0])) * scale; + if (ddr_mask != 0) { + v += ggml_gtx_softmax_bits_to_float(ggml_gtx_fp16_to_fp32_bits(mask[mask_off + i0])); + } + const float e = ggml_gtx_softmax_expf(v - row_max); + dst[dst_off + i0] = ggml_gtx_fp32_bits_to_fp16(ggml_gtx_softmax_float_to_bits(e)); + sum += e; + } + + const float inv_sum = sum > 0.0f ? 1.0f / sum : 0.0f; + for (uint16_t i0 = 0; i0 < ne0; ++i0) { + const float e = ggml_gtx_softmax_bits_to_float(ggml_gtx_fp16_to_fp32_bits(dst[dst_off + i0])); + dst[dst_off + i0] = ggml_gtx_fp32_bits_to_fp16(ggml_gtx_softmax_float_to_bits(e * inv_sum)); + } + } + } + } + + (void)max_bias; +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-swiglu.c b/depend/ggml-gtx/src/ggml-gtx-kernel-swiglu.c new file mode 100644 index 0000000..b8c914b --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-swiglu.c @@ -0,0 +1,511 @@ +#include +#include +#include + +#include "ggml-gtx-baremetal.h" +#include "ggml-gtx-compute.h" +#include "ggml-gtx-plan.h" +#include "ggml-gtx-port.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#ifdef GGML_GTX_USE_GTX +#include "ggml-gtx/intrinsics/intrin.h" +#endif + +int ggml_gtx_kernel_swiglu_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_SWIGLU_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_SWIGLU_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + const uint64_t rows64 = (uint64_t) desc->src_ne[0][1] * + (uint64_t) desc->src_ne[0][2] * + (uint64_t) desc->src_ne[0][3]; + if (desc->dst_addr == 0 || + desc->src_addr[0] == 0 || + desc->src_ne[0][0] == 0 || + rows64 == 0 || + desc->src_ne[0][0] > UINT16_MAX) { + return -2; + } + + if (desc->memory_mode == GGML_GTX_MEMORY_MODE_L2_STASHED_N1) { + const uint32_t n_cols = desc->src_ne[0][0]; + const uint32_t half = n_cols / 2u; + const uint64_t half_bytes = (uint64_t)half * GGML_GTX_DTYPE_SIZE; + if (desc->nest_id >= GGML_GTX_NEST_NUM || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->src_addr[1] != 0 || + desc->l2_src_addr[0] == 0 || + desc->l2_dst_addr == 0 || + desc->elem_size != GGML_GTX_DTYPE_SIZE || + (n_cols & 1u) != 0u || + half == 0u || + desc->ne[0] != half || + desc->ne[1] != desc->src_ne[0][1] || + desc->ne[2] != desc->src_ne[0][2] || + desc->ne[3] != desc->src_ne[0][3] || + half > UINT16_MAX || + half_bytes > GGML_GTX_L1_BANK_SIZE || + rows64 > UINT16_MAX) { + return -3; + } + + ggml_gtx_compute_swiglu_l2((uint32_t)desc->l2_src_addr[0], + (uint32_t)desc->l2_dst_addr, + (uint16_t)rows64, + (uint16_t)n_cols, + (uint8_t)desc->op_params[1], + (uint8_t)desc->nest_id, + GGML_GTX_SPU_NUM_PER_NEST); + return 0; + } + + if (rows64 > UINT16_MAX) { + return -4; + } + + ggml_gtx_compute_swiglu(desc->src_addr[0], + desc->src_addr[1], + desc->dst_addr, + (uint16_t) rows64, + (uint16_t) desc->src_ne[0][0], + (uint8_t) desc->op_params[1], + GGML_GTX_NEST_NUM, + GGML_GTX_SPU_NUM_PER_NEST); + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_swiglu_f16(arg); +} + +//================================================================== +// GLU variants: SWIGLU, GEGLU, REGLU +// +// Single-tensor form: split row in half. +// If NOT swapped: input = first half, gate = second half +// If swapped: gate = first half, input = second half +// +// Formula: activation(input) * gate +// SWIGLU: SiLU(input) * gate +// GEGLU: GELU(input) * gate +// REGLU: ReLU(input) * gate +// +// Input: [n_cols, n_rows, ...] FP16 (n_cols = 2 * output_width) +// Output: [n_cols/2, n_rows, ...] FP16 +// +// Strategy: Load input half → Bank A, apply activation → Bank R, +// Load gate half → Bank B, __mul_vv → R = activation * gate. +// Uses bank remapping for multi-step within single tile. +// +// For now: uses per-row GTX ISA processing. Each row processed as: +// 1. Load input_half (n_cols/2 elements) → L2 INPUT +// 2. Load gate_half (n_cols/2 elements) → L2 second area +// 3. Thread: load input → A, apply activation → R, remap, load gate → B +// → mul_vv → R, store R +// +// Actually, simplest correct approach: process per-row via a custom +// tile loop using single NEST/SPU, loading both halves. +//================================================================== + +#define GLU_TYPE_SWIGLU 0 +#define GLU_TYPE_GEGLU 1 +#define GLU_TYPE_REGLU 2 + +#ifndef GGML_GTX_UNUSED +#define GGML_GTX_UNUSED(x) (void)(x) +#endif + +static float ggml_gtx_glu_bare_expf(float x) { + float term = 1.0f; + float sum = 1.0f; + + for (int i = 1; i <= 10; ++i) { + term *= x / (float) i; + sum += term; + } + + return sum; +} + +static void ggml_gtx_compute_swiglu_packed( + uint64_t ddr_input, + uint64_t ddr_result, + uint16_t n_rows, + uint16_t n_cols, + uint8_t swapped, + uint8_t nest_num, + uint8_t spu_num_per_nest) +{ +#ifdef GGML_GTX_USE_GTX + GGML_GTX_UNUSED(nest_num); + const uint16_t not_use = 0xBEEF; + const uint32_t dtype_size = GGML_GTX_DTYPE_SIZE; + const uint32_t l1_bank_size = 64U * 1024U * 2U; + const uint32_t offset_l2_bank = 0x100000U; + const uint32_t base_l2_value = 0x000000U; + const uint32_t base_l2_gate = 0x060000U; + const uint32_t base_l2_result = 0x0C0000U; + const uint32_t base_l1_a = 0x00000U; + const uint32_t base_l1_b = 0x20000U; + const uint32_t base_l1_r = 0x40000U; + + const uint32_t half = n_cols / 2U; + const uint32_t cal_unit_origin_byte = n_cols * dtype_size; + const uint32_t cal_unit = half; + const uint32_t cal_unit_byte = cal_unit * dtype_size; + const uint32_t cal_max = l1_bank_size / cal_unit_byte; + const uint32_t cal_max_size = cal_max * cal_unit; + const uint32_t offset_cal_max = cal_max * cal_unit_byte; + const uint32_t cal_num = n_rows; + const uint32_t eff_thread = cal_num > spu_num_per_nest ? spu_num_per_nest : cal_num; + const uint32_t cal_quot = cal_num / eff_thread; + const uint32_t cal_mod = cal_num % eff_thread; + const uint32_t input_half_off = swapped ? cal_unit_byte : 0; + const uint32_t gate_half_off = swapped ? 0 : cal_unit_byte; + + /* + * Keep the DDR transfers and SPU work in separate plans. The FPGA + * runtime completes each split/join before the next stage starts, so + * this does not depend on the shared/thread credit handshake used by + * the original combined plan. + */ + __split(); + __start_plani(0); + __start_shared(); + for (uint8_t thread_id = 0; thread_id < eff_thread; ++thread_id) { + const uint32_t cal_num_per_thread = thread_id < cal_mod ? cal_quot + 1U : cal_quot; + const uint32_t offset_load_thread = + ((thread_id < cal_mod ? thread_id : cal_mod) * (cal_quot + 1U) + + (thread_id > cal_mod ? (thread_id - cal_mod) : 0U) * cal_quot) * cal_unit_origin_byte; + const uint32_t bank_offset = (uint32_t)thread_id * offset_l2_bank; + + __load( + ddr_input + offset_load_thread + input_half_off, + base_l2_value + bank_offset, + cal_unit_origin_byte, + cal_unit_byte, + cal_num_per_thread, + cal_unit_byte + ); + + __load( + ddr_input + offset_load_thread + gate_half_off, + base_l2_gate + bank_offset, + cal_unit_origin_byte, + cal_unit_byte, + cal_num_per_thread, + cal_unit_byte + ); + } + __end_shared(); + __end_plani(0); + __join(); + + __split(); + __start_plani(0); + for (uint8_t thread_id = 0; thread_id < eff_thread; ++thread_id) { + __start_thread(thread_id); + __set_spm_addr_A(base_l1_a); + __set_spm_addr_R(base_l1_r); + + const uint32_t cal_num_per_thread = thread_id < cal_mod ? cal_quot + 1U : cal_quot; + const uint32_t sequence_num_per_thread = (cal_num_per_thread + cal_max - 1U) / cal_max; + const uint32_t iteration = sequence_num_per_thread - 1U; + const uint32_t cal_last = (cal_num_per_thread % cal_max) ? (cal_num_per_thread % cal_max) : cal_max; + const uint32_t bank_offset = (uint32_t)thread_id * offset_l2_bank; + const uint32_t l2_value_thread = base_l2_value + bank_offset; + const uint32_t l2_gate_thread = base_l2_gate + bank_offset; + const uint32_t l2_result_thread = base_l2_result + bank_offset; + uint32_t offset_cal = 0; + + for (uint32_t iter = 0; iter < iteration; ++iter) { + __load(l2_value_thread + offset_cal, base_l1_r, + cal_unit_byte, cal_unit_byte, cal_max, not_use); + __load(l2_gate_thread + offset_cal, base_l1_b, + cal_unit_byte, cal_unit_byte, cal_max, not_use); + + __sigm(cal_max_size); + __set_spm_addr_B(base_l1_r); + __mul_vv(cal_max_size); + __set_spm_addr_A(base_l1_b); + __mul_vv(cal_max_size); + __set_spm_addr_A(base_l1_a); + __set_spm_addr_B(base_l1_b); + + __store(base_l1_r, l2_result_thread + offset_cal, + cal_unit_byte, cal_unit_byte, cal_max, not_use); + + offset_cal += offset_cal_max; + } + + __load(l2_value_thread + offset_cal, base_l1_r, + cal_unit_byte, cal_unit_byte, cal_last, not_use); + __load(l2_gate_thread + offset_cal, base_l1_b, + cal_unit_byte, cal_unit_byte, cal_last, not_use); + + __sigm(cal_unit * cal_last); + __set_spm_addr_B(base_l1_r); + __mul_vv(cal_unit * cal_last); + __set_spm_addr_A(base_l1_b); + __mul_vv(cal_unit * cal_last); + __set_spm_addr_A(base_l1_a); + __set_spm_addr_B(base_l1_b); + + __store(base_l1_r, l2_result_thread + offset_cal, + cal_unit_byte, cal_unit_byte, cal_last, not_use); + __end_thread(thread_id); + } + __end_plani(0); + __join(); + + __split(); + __start_plani(0); + __start_shared(); + for (uint8_t thread_id = 0; thread_id < eff_thread; ++thread_id) { + const uint32_t cal_num_per_thread = thread_id < cal_mod ? cal_quot + 1U : cal_quot; + const uint32_t offset_store_thread = + ((thread_id < cal_mod ? thread_id : cal_mod) * (cal_quot + 1U) + + (thread_id > cal_mod ? (thread_id - cal_mod) : 0U) * cal_quot) * cal_unit_byte; + const uint32_t bank_offset = (uint32_t)thread_id * offset_l2_bank; + + __store( + base_l2_result + bank_offset, + ddr_result + offset_store_thread, + cal_unit_byte, + cal_unit_byte, + cal_num_per_thread, + cal_unit_byte + ); + } + __end_shared(); + __end_plani(0); + __join(); +#else + GGML_GTX_UNUSED(nest_num); + GGML_GTX_UNUSED(spu_num_per_nest); + + volatile uint32_t * src = (volatile uint32_t *)(uintptr_t)ddr_input; + volatile uint32_t * dst = (volatile uint32_t *)(uintptr_t)ddr_result; + uint16_t half = n_cols / 2; + + for (uint16_t row = 0; row < n_rows; ++row) { + volatile uint32_t * row_base = src + (uint32_t)row * n_cols; + volatile uint32_t * x_ptr = swapped ? row_base + half : row_base; + volatile uint32_t * g_ptr = swapped ? row_base : row_base + half; + volatile uint32_t * d_ptr = dst + (uint32_t)row * half; + + for (uint16_t k = 0; k < half; ++k) { + union { uint32_t u; float f; } vx, vg, vo; + vx.u = x_ptr[k]; + vg.u = g_ptr[k]; + float sig = 1.0f / (1.0f + ggml_gtx_glu_bare_expf(-vx.f)); + vo.f = (vx.f * sig) * vg.f; + d_ptr[k] = vo.u; + } + } +#endif +} + +void ggml_gtx_compute_swiglu_l2( + uint32_t l2_input, + uint32_t l2_result, + uint16_t n_rows, + uint16_t n_cols, + uint8_t swapped, + uint8_t nest_id, + uint8_t spu_num_per_nest) +{ +#ifdef GGML_GTX_USE_GTX + if (n_rows == 0 || n_cols == 0 || (n_cols & 1U) != 0U || spu_num_per_nest == 0) { + return; + } + + const uint16_t not_use = 0xBEEF; + const uint32_t dtype_size = GGML_GTX_DTYPE_SIZE; + const uint32_t l1_bank_size = 64U * 1024U * 2U; + const uint32_t base_l1_a = 0x00000U; + const uint32_t base_l1_b = 0x20000U; + const uint32_t base_l1_r = 0x40000U; + + const uint32_t half = n_cols / 2U; + const uint32_t input_row_bytes = n_cols * dtype_size; + const uint32_t output_row_bytes = half * dtype_size; + const uint32_t cal_unit = half; + const uint32_t cal_unit_byte = output_row_bytes; + const uint32_t cal_max = l1_bank_size / cal_unit_byte; + if (cal_max == 0U) { + return; + } + + const uint32_t cal_max_size = cal_max * cal_unit; + const uint32_t input_offset_cal_max = cal_max * input_row_bytes; + const uint32_t output_offset_cal_max = cal_max * output_row_bytes; + const uint32_t cal_num = n_rows; + const uint32_t eff_thread = cal_num > spu_num_per_nest ? spu_num_per_nest : cal_num; + const uint32_t cal_quot = cal_num / eff_thread; + const uint32_t cal_mod = cal_num % eff_thread; + const uint32_t input_half_off = swapped ? output_row_bytes : 0U; + const uint32_t gate_half_off = swapped ? 0U : output_row_bytes; + + __split(); + + __start_plan(nest_id); + + for (uint8_t thread_id = 0; thread_id < eff_thread; ++thread_id) { + + __start_thread(thread_id); + + __set_spm_addr_A(base_l1_a); + __set_spm_addr_B(base_l1_b); + __set_spm_addr_R(base_l1_r); + + const uint32_t cal_num_per_thread = thread_id < cal_mod ? cal_quot + 1U : cal_quot; + const uint32_t start_row = + (thread_id < cal_mod ? thread_id : cal_mod) * (cal_quot + 1U) + + (thread_id > cal_mod ? (thread_id - cal_mod) : 0U) * cal_quot; + const uint32_t sequence_num_per_thread = (cal_num_per_thread + cal_max - 1U) / cal_max; + const uint32_t iteration = sequence_num_per_thread - 1U; + const uint32_t cal_last = (cal_num_per_thread % cal_max) ? (cal_num_per_thread % cal_max) : cal_max; + uint32_t input_offset_cal = start_row * input_row_bytes; + uint32_t output_offset_cal = start_row * output_row_bytes; + + for (uint32_t iter = 0; iter < iteration; ++iter) { + __load(l2_input + input_offset_cal + input_half_off, base_l1_r, + input_row_bytes, cal_unit_byte, cal_max, not_use); + __load(l2_input + input_offset_cal + gate_half_off, base_l1_b, + input_row_bytes, cal_unit_byte, cal_max, not_use); + + __sigm(cal_max_size); + __set_spm_addr_B(base_l1_r); + __mul_vv(cal_max_size); + __set_spm_addr_A(base_l1_b); + __mul_vv(cal_max_size); + __set_spm_addr_A(base_l1_a); + __set_spm_addr_B(base_l1_b); + + __store(base_l1_r, l2_result + output_offset_cal, + cal_unit_byte, cal_unit_byte, cal_max, not_use); + + input_offset_cal += input_offset_cal_max; + output_offset_cal += output_offset_cal_max; + } + + __load(l2_input + input_offset_cal + input_half_off, base_l1_r, + input_row_bytes, cal_unit_byte, cal_last, not_use); + __load(l2_input + input_offset_cal + gate_half_off, base_l1_b, + input_row_bytes, cal_unit_byte, cal_last, not_use); + + __sigm(cal_unit * cal_last); + __set_spm_addr_B(base_l1_r); + __mul_vv(cal_unit * cal_last); + __set_spm_addr_A(base_l1_b); + __mul_vv(cal_unit * cal_last); + __set_spm_addr_A(base_l1_a); + __set_spm_addr_B(base_l1_b); + + __store(base_l1_r, l2_result + output_offset_cal, + cal_unit_byte, cal_unit_byte, cal_last, not_use); + + __end_thread(thread_id); + } + + __end_plan(nest_id); + + __join(); +#else + GGML_GTX_UNUSED(l2_input); + GGML_GTX_UNUSED(l2_result); + GGML_GTX_UNUSED(n_rows); + GGML_GTX_UNUSED(n_cols); + GGML_GTX_UNUSED(swapped); + GGML_GTX_UNUSED(nest_id); + GGML_GTX_UNUSED(spu_num_per_nest); +#endif +} + +void ggml_gtx_compute_swiglu( + uint64_t ddr_input, + uint64_t ddr_gate, + uint64_t ddr_result, + uint16_t n_rows, + uint16_t n_cols, + uint8_t swapped, + uint8_t nest_num, + uint8_t spu_num_per_nest) +{ + if (ddr_gate == 0) { + ggml_gtx_compute_swiglu_packed( + ddr_input, ddr_result, n_rows, n_cols, + swapped, nest_num, spu_num_per_nest); + return; + } + + const uint32_t half_bytes = (uint32_t)n_cols * 2; + const uint32_t packed_row_bytes = half_bytes * 2; + const uint32_t temp_bytes = 0x00200000U; + const uint64_t ddr_packed = GGML_GTX_DDR_TEMP_0; + const uint16_t max_rows_per_chunk = + (uint16_t)(temp_bytes / packed_row_bytes); + + if (max_rows_per_chunk == 0 || n_cols > 32767U) { + return; + } + + for (uint32_t row_base = 0; row_base < n_rows; row_base += max_rows_per_chunk) { + uint16_t rows_chunk = (uint16_t)((uint32_t)n_rows - row_base); + if (rows_chunk > max_rows_per_chunk) { + rows_chunk = max_rows_per_chunk; + } + +#ifdef GGML_GTX_USE_GTX + for (uint16_t row = 0; row < rows_chunk; ++row) { + const uint64_t absolute_row = row_base + row; + const uint64_t src_row = ddr_input + absolute_row * half_bytes; + const uint64_t gate_row = ddr_gate + absolute_row * half_bytes; + const uint64_t packed_row = ddr_packed + (uint64_t)row * packed_row_bytes; + + __copy_mem(src_row, + packed_row, + half_bytes, half_bytes, + 1, half_bytes, 0); + __copy_mem(gate_row, + packed_row + half_bytes, + half_bytes, half_bytes, + 1, half_bytes, 0); + + } +#else + volatile uint16_t * packed = (volatile uint16_t *)(uintptr_t)ddr_packed; + volatile uint16_t * src = (volatile uint16_t *)(uintptr_t)ddr_input; + volatile uint16_t * gate = (volatile uint16_t *)(uintptr_t)ddr_gate; + + for (uint16_t row = 0; row < rows_chunk; ++row) { + const uint32_t absolute_row = row_base + row; + volatile uint16_t * packed_row = packed + (uint32_t)row * (uint32_t)(n_cols * 2); + volatile uint16_t * src_row = src + absolute_row * n_cols; + volatile uint16_t * gate_row = gate + absolute_row * n_cols; + + for (uint16_t col = 0; col < n_cols; ++col) { + packed_row[col] = src_row[col]; + packed_row[n_cols + col] = gate_row[col]; + } + } +#endif + + ggml_gtx_compute_swiglu_packed( + ddr_packed, + ddr_result + (uint64_t)row_base * half_bytes, + rows_chunk, + (uint16_t)(n_cols * 2), + 0, nest_num, spu_num_per_nest); + } +} diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-tanh.c b/depend/ggml-gtx/src/ggml-gtx-kernel-tanh.c new file mode 100644 index 0000000..f44527d --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-tanh.c @@ -0,0 +1,6 @@ +#define GGML_GTX_UNARY_KERNEL_ID GGML_GTX_KERNEL_TANH_F16 +#define GGML_GTX_UNARY_KERNEL_FN ggml_gtx_kernel_tanh_f16 +#define GGML_GTX_UNARY_INPUT_BANK GGML_GTX_UNARY_BANK_R +#define GGML_GTX_UNARY_OUTPUT_BANK GGML_GTX_UNARY_BANK_A +#define GGML_GTX_UNARY_APPLY(elems) __tanh(elems) +#include "ggml-gtx-kernel-unary-ref.h" diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-unary-ref.h b/depend/ggml-gtx/src/ggml-gtx-kernel-unary-ref.h new file mode 100644 index 0000000..087c183 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-unary-ref.h @@ -0,0 +1,98 @@ +#ifndef GGML_GTX_KERNEL_UNARY_REF_H +#define GGML_GTX_KERNEL_UNARY_REF_H + +#include + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#define GGML_GTX_UNARY_BANK_A GGML_GTX_L1_BANK_A_OFFSET +#define GGML_GTX_UNARY_BANK_B GGML_GTX_L1_BANK_B_OFFSET +#define GGML_GTX_UNARY_BANK_C GGML_GTX_L1_BANK_C_OFFSET +#define GGML_GTX_UNARY_BANK_R GGML_GTX_L1_BANK_R_OFFSET +#define GGML_GTX_UNARY_DTYPE 2u +#define GGML_GTX_UNARY_SPUS GGML_GTX_SPU_NUM_PER_NEST +#define GGML_GTX_UNARY_TOKEN 0xBEEFu + +static inline uint32_t ggml_gtx_unary_min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t ggml_gtx_unary_elems_for_tid(uint32_t total_elems, uint8_t tid) { + const uint32_t q = total_elems / GGML_GTX_UNARY_SPUS; + const uint32_t r = total_elems % GGML_GTX_UNARY_SPUS; + return tid < r ? q + 1u : q; +} + +static inline uint32_t ggml_gtx_unary_start_for_tid(uint32_t total_elems, uint8_t tid) { + const uint32_t q = total_elems / GGML_GTX_UNARY_SPUS; + const uint32_t r = total_elems % GGML_GTX_UNARY_SPUS; + if (tid < r) { + return (uint32_t) tid * (q + 1u); + } + return r * (q + 1u) + ((uint32_t) tid - r) * q; +} + +static int GGML_GTX_UNARY_KERNEL_FN(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_UNARY_KERNEL_ID || + karg->desc.kernel_id != GGML_GTX_UNARY_KERNEL_ID) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + const uint32_t elems = desc->ne[0]; + if (desc->memory_mode != GGML_GTX_MEMORY_MODE_L2_STASHED_N1 || + desc->nest_id >= GGML_GTX_NEST_NUM || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16 || + desc->elem_size != GGML_GTX_UNARY_DTYPE || + elems == 0 || + elems > UINT16_MAX) { + return -2; + } + + __split(); + __start_plan((uint8_t) desc->nest_id); + for (uint8_t tid = 0; tid < GGML_GTX_UNARY_SPUS; tid++) { + const uint32_t elems_this_tid = ggml_gtx_unary_elems_for_tid(elems, tid); + __start_thread(tid); + if (elems_this_tid > 0u) { + const uint32_t elem_start = ggml_gtx_unary_start_for_tid(elems, tid); + const uint32_t chunk_off = elem_start * GGML_GTX_UNARY_DTYPE; + const uint32_t chunk_bytes = elems_this_tid * GGML_GTX_UNARY_DTYPE; + const uint16_t chunk_bytes_u16 = (uint16_t) chunk_bytes; + + __load((uint32_t) desc->l2_src_addr[0] + chunk_off, + GGML_GTX_UNARY_INPUT_BANK, + chunk_bytes, + chunk_bytes_u16, + 1u, + chunk_bytes_u16); + __set_spm_addr(GGML_GTX_UNARY_BANK_R, GGML_GTX_UNARY_BANK_C, + GGML_GTX_UNARY_BANK_B, GGML_GTX_UNARY_BANK_A); + GGML_GTX_UNARY_APPLY(elems_this_tid); + __store(GGML_GTX_UNARY_OUTPUT_BANK, + (uint32_t) desc->l2_dst_addr + chunk_off, + chunk_bytes, + chunk_bytes_u16, + 1u, + chunk_bytes_u16); + } + __end_thread(tid); + } + __end_plan((uint8_t) desc->nest_id); + __join(); + + return 0; +} + +int kernel_main(void * arg) { + return GGML_GTX_UNARY_KERNEL_FN(arg); +} + +#endif diff --git a/depend/ggml-gtx/src/ggml-gtx-kernel-upscale.c b/depend/ggml-gtx/src/ggml-gtx-kernel-upscale.c new file mode 100644 index 0000000..f129cff --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx-kernel-upscale.c @@ -0,0 +1,358 @@ +//================================================================== +// GGML GTX kernel: UPSCALE +// +// Descriptor wrapper around the reference nearest-neighbor 2x kernel +// structure. +//================================================================== + +#include + +#include "ggml-gtx/ggml-gtx-kernel-common.h" +#include "ggml-gtx/ggml-gtx-protocol.h" + +#define GGML_GTX_SCALE_MODE_NEAREST 0 +#define GGML_GTX_SCALE_MODE_BILINEAR 1 +#define GGML_GTX_SCALE_MODE_BICUBIC 2 +#define GGML_GTX_SCALE_FLAG_ALIGN_CORNERS (1 << 8) +#define GGML_GTX_SCALE_FLAG_ANTIALIAS (1 << 9) + +#define DTYPE 2 +#define SCALE_FACTOR 2 +#define SPU_NUM_PER_NEST GGML_GTX_SPU_NUM_PER_NEST +#define L2_A 0x000000 +#define L2_RESULT 0x002000 +#define BANK_A GGML_GTX_L1_BANK_A_OFFSET +#define BANK_B GGML_GTX_L1_BANK_B_OFFSET +#define BANK_C GGML_GTX_L1_BANK_C_OFFSET +#define BANK_R GGML_GTX_L1_BANK_R_OFFSET +#define ACTIVE_TID_MASK ((1u << SPU_NUM_PER_NEST) - 1u) + +static float ggml_gtx_upscale_bits_to_float(uint32_t bits) { + union { uint32_t u; float f; } value_bits; + value_bits.u = bits; + return value_bits.f; +} + +static uint32_t ggml_gtx_upscale_float_to_bits(float value) { + union { uint32_t u; float f; } value_bits; + value_bits.f = value; + return value_bits.u; +} + +static float ggml_gtx_upscale_load_f16(uint64_t addr) { + const uint16_t h = *(volatile const uint16_t *)(uintptr_t)addr; + return ggml_gtx_upscale_bits_to_float(ggml_gtx_fp16_to_fp32_bits(h)); +} + +static void ggml_gtx_upscale_store_f16(uint64_t addr, float value) { + *(volatile uint16_t *)(uintptr_t)addr = + ggml_gtx_fp32_bits_to_fp16(ggml_gtx_upscale_float_to_bits(value)); +} + +static int32_t ggml_gtx_upscale_floor_f32(float value) { + const int32_t iv = (int32_t)value; + return ((float)iv > value) ? iv - 1 : iv; +} + +static uint32_t ggml_gtx_upscale_clamp_i32(int32_t value, uint32_t hi) { + if (value < 0) { + return 0; + } + if ((uint32_t)value > hi) { + return hi; + } + return (uint32_t)value; +} + +static uint32_t ggml_gtx_upscale_map_nearest(uint32_t dst_idx, float scale, uint32_t hi) { + uint32_t mapped = (uint32_t)((float)dst_idx / scale); + if (mapped > hi) { + mapped = hi; + } + return mapped; +} + +static float ggml_gtx_upscale_weight1(float x) { + const float a = -0.75f; + return ((a + 2.0f) * x - (a + 3.0f)) * x * x + 1.0f; +} + +static float ggml_gtx_upscale_weight2(float x) { + const float a = -0.75f; + return ((a * x - 5.0f * a) * x + 8.0f * a) * x - 4.0f * a; +} + +static float ggml_gtx_upscale_bicubic(float p0, float p1, float p2, float p3, float x) { + const float w0 = ggml_gtx_upscale_weight2(x + 1.0f); + const float w1 = ggml_gtx_upscale_weight1(x); + const float w2 = ggml_gtx_upscale_weight1(1.0f - x); + const float w3 = ggml_gtx_upscale_weight2(2.0f - x); + return p0 * w0 + p1 * w1 + p2 * w2 + p3 * w3; +} + +static float ggml_gtx_upscale_load_src(const ggml_gtx_kernel_launch_desc_t * desc, + uint32_t x, + uint32_t y, + uint32_t z, + uint32_t w) { + const uint64_t addr = desc->src_addr[0] + + (uint64_t)x * desc->src_nb[0][0] + + (uint64_t)y * desc->src_nb[0][1] + + (uint64_t)z * desc->src_nb[0][2] + + (uint64_t)w * desc->src_nb[0][3]; + return ggml_gtx_upscale_load_f16(addr); +} + +static int ggml_gtx_kernel_upscale_generic_f16(const ggml_gtx_kernel_launch_desc_t * desc) { + const uint32_t mode_flags = (uint32_t)desc->op_params[0]; + const uint32_t mode = mode_flags & 0xffu; + const int align_corners = (mode_flags & GGML_GTX_SCALE_FLAG_ALIGN_CORNERS) != 0; + + if ((mode_flags & GGML_GTX_SCALE_FLAG_ANTIALIAS) != 0) { + return -5; + } + if (mode != GGML_GTX_SCALE_MODE_NEAREST && + mode != GGML_GTX_SCALE_MODE_BILINEAR && + mode != GGML_GTX_SCALE_MODE_BICUBIC) { + return -6; + } + + const uint32_t src_width = desc->src_ne[0][0]; + const uint32_t src_height = desc->src_ne[0][1]; + const uint32_t src_depth = desc->src_ne[0][2]; + const uint32_t src_batches = desc->src_ne[0][3]; + const uint32_t dst_width = desc->ne[0]; + const uint32_t dst_height = desc->ne[1]; + const uint32_t dst_depth = desc->ne[2]; + const uint32_t dst_batches = desc->ne[3]; + + if (src_width == 0 || src_height == 0 || src_depth == 0 || src_batches == 0 || + dst_width == 0 || dst_height == 0 || dst_depth == 0 || dst_batches == 0 || + src_depth != dst_depth || src_batches != dst_batches) { + return -7; + } + + float scale_x = (float)dst_width / (float)src_width; + float scale_y = (float)dst_height / (float)src_height; + if (align_corners) { + if (src_width > 1 && dst_width > 1) { + scale_x = (float)(dst_width - 1) / (float)(src_width - 1); + } + if (src_height > 1 && dst_height > 1) { + scale_y = (float)(dst_height - 1) / (float)(src_height - 1); + } + } + + const float pixel_offset = align_corners ? 0.0f : 0.5f; + const uint32_t max_src_x = src_width - 1; + const uint32_t max_src_y = src_height - 1; + + for (uint32_t b = 0; b < dst_batches; ++b) { + for (uint32_t z = 0; z < dst_depth; ++z) { + for (uint32_t y = 0; y < dst_height; ++y) { + for (uint32_t x = 0; x < dst_width; ++x) { + float value = 0.0f; + + if (mode == GGML_GTX_SCALE_MODE_NEAREST) { + const uint32_t sx = ggml_gtx_upscale_map_nearest(x, scale_x, max_src_x); + const uint32_t sy = ggml_gtx_upscale_map_nearest(y, scale_y, max_src_y); + value = ggml_gtx_upscale_load_src(desc, sx, sy, z, b); + } else if (mode == GGML_GTX_SCALE_MODE_BILINEAR) { + const float fx = ((float)x + pixel_offset) / scale_x - pixel_offset; + const float fy = ((float)y + pixel_offset) / scale_y - pixel_offset; + const int32_t x0_i = ggml_gtx_upscale_floor_f32(fx); + const int32_t y0_i = ggml_gtx_upscale_floor_f32(fy); + const uint32_t x0 = ggml_gtx_upscale_clamp_i32(x0_i, max_src_x); + const uint32_t y0 = ggml_gtx_upscale_clamp_i32(y0_i, max_src_y); + const uint32_t x1 = ggml_gtx_upscale_clamp_i32(x0_i + 1, max_src_x); + const uint32_t y1 = ggml_gtx_upscale_clamp_i32(y0_i + 1, max_src_y); + float dx = fx - (float)x0_i; + float dy = fy - (float)y0_i; + if (dx < 0.0f) dx = 0.0f; + if (dx > 1.0f) dx = 1.0f; + if (dy < 0.0f) dy = 0.0f; + if (dy > 1.0f) dy = 1.0f; + + const float a = ggml_gtx_upscale_load_src(desc, x0, y0, z, b); + const float c = ggml_gtx_upscale_load_src(desc, x0, y1, z, b); + const float b0 = ggml_gtx_upscale_load_src(desc, x1, y0, z, b); + const float d = ggml_gtx_upscale_load_src(desc, x1, y1, z, b); + value = a * (1.0f - dx) * (1.0f - dy) + + b0 * dx * (1.0f - dy) + + c * (1.0f - dx) * dy + + d * dx * dy; + } else { + const float fx = ((float)x + pixel_offset) / scale_x - pixel_offset; + const float fy = ((float)y + pixel_offset) / scale_y - pixel_offset; + const int32_t x1_i = ggml_gtx_upscale_floor_f32(fx); + const int32_t y1_i = ggml_gtx_upscale_floor_f32(fy); + const float dx = fx - (float)x1_i; + const float dy = fy - (float)y1_i; + + float rows[4]; + for (int32_t yy = -1; yy <= 2; ++yy) { + const uint32_t sy = ggml_gtx_upscale_clamp_i32(y1_i + yy, max_src_y); + const float p0 = ggml_gtx_upscale_load_src(desc, ggml_gtx_upscale_clamp_i32(x1_i - 1, max_src_x), sy, z, b); + const float p1 = ggml_gtx_upscale_load_src(desc, ggml_gtx_upscale_clamp_i32(x1_i, max_src_x), sy, z, b); + const float p2 = ggml_gtx_upscale_load_src(desc, ggml_gtx_upscale_clamp_i32(x1_i + 1, max_src_x), sy, z, b); + const float p3 = ggml_gtx_upscale_load_src(desc, ggml_gtx_upscale_clamp_i32(x1_i + 2, max_src_x), sy, z, b); + rows[yy + 1] = ggml_gtx_upscale_bicubic(p0, p1, p2, p3, dx); + } + value = ggml_gtx_upscale_bicubic(rows[0], rows[1], rows[2], rows[3], dy); + } + + const uint64_t dst_addr = desc->dst_addr + + (uint64_t)x * desc->nb[0] + + (uint64_t)y * desc->nb[1] + + (uint64_t)z * desc->nb[2] + + (uint64_t)b * desc->nb[3]; + ggml_gtx_upscale_store_f16(dst_addr, value); + } + } + } + } + + return 0; +} + +int ggml_gtx_kernel_upscale_f16(void * arg) { + const ggml_gtx_kernel_arg_t * karg = (const ggml_gtx_kernel_arg_t *) arg; + if (karg == 0 || + karg->magic != GGML_GTX_KERNEL_ARG_MAGIC || + karg->version != GGML_GTX_KERNEL_ARG_VERSION || + karg->size < sizeof(ggml_gtx_kernel_arg_t) || + karg->kernel_id != GGML_GTX_KERNEL_UPSCALE_F16 || + karg->desc.kernel_id != GGML_GTX_KERNEL_UPSCALE_F16) { + return -1; + } + + const ggml_gtx_kernel_launch_desc_t * desc = &karg->desc; + if (desc->dst_addr == 0 || + desc->src_addr[0] == 0 || + desc->dtype != GGML_GTX_TYPE_F16 || + desc->src_type[0] != GGML_GTX_TYPE_F16) { + return -2; + } + + const uint32_t src_width = desc->src_ne[0][0]; + const uint32_t src_rows = desc->src_ne[0][1]; + const uint32_t planes = desc->src_ne[0][2] * desc->src_ne[0][3]; + const uint32_t dst_width = desc->ne[0]; + const uint32_t dst_rows = desc->ne[1]; + + if (src_width == 0 || src_rows == 0 || planes == 0 || + dst_width == 0 || dst_rows == 0 || + desc->ne[2] != desc->src_ne[0][2] || + desc->ne[3] != desc->src_ne[0][3]) { + return -3; + } + + const uint32_t mode = (uint32_t)desc->op_params[0] & 0xffu; + if (mode != GGML_GTX_SCALE_MODE_NEAREST || + dst_width != src_width * SCALE_FACTOR || + dst_rows != src_rows * SCALE_FACTOR || + desc->src_ne[0][3] != 1 || + desc->ne[3] != 1) { + return ggml_gtx_kernel_upscale_generic_f16(desc); + } + + const uint32_t src_row_bytes = src_width * DTYPE; + const uint32_t dst_row_bytes = dst_width * DTYPE; + const uint32_t src_tile_bytes = src_rows * src_row_bytes; + const uint32_t dst_tile_bytes = dst_rows * dst_row_bytes; + const uint32_t src_rows_per_spu = (src_rows + SPU_NUM_PER_NEST - 1) / SPU_NUM_PER_NEST; + + if (src_tile_bytes > 0x2000 || dst_tile_bytes > 0x20000 || + src_row_bytes > UINT16_MAX || dst_row_bytes > UINT16_MAX || + src_tile_bytes > UINT16_MAX || dst_tile_bytes > UINT16_MAX) { + return ggml_gtx_kernel_upscale_generic_f16(desc); + } + + for (uint32_t plane = 0; plane < planes; ++plane) { + const uint64_t src_plane = desc->src_addr[0] + plane * desc->src_nb[0][2]; + const uint64_t dst_plane = desc->dst_addr + plane * desc->nb[2]; + + __split(); + + __start_plan(0); + __start_shared(); + __load_cr(src_plane, L2_A, + src_tile_bytes, + (uint16_t)src_tile_bytes, + 1, (uint16_t)src_tile_bytes, + 1, ACTIVE_TID_MASK, 0xBEEF); + + __credit_chk(ACTIVE_TID_MASK); + + __store_cr(L2_RESULT, dst_plane, + dst_tile_bytes, + (uint16_t)dst_tile_bytes, + 1, (uint16_t)dst_tile_bytes, + 1, ACTIVE_TID_MASK); + __end_shared(); + + for (uint8_t tid = 0; tid < SPU_NUM_PER_NEST; tid++) { + const uint16_t tid_mask = (uint16_t)(0x1u << tid); + const uint32_t src_row_start = (uint32_t)tid * src_rows_per_spu; + const uint32_t src_row_end = src_row_start + src_rows_per_spu; + + __start_thread(tid); + __set_spm_addr(BANK_R, BANK_C, BANK_B, BANK_A); + __credit_chk(0xBEEF); + + for (uint32_t src_row = src_row_start; src_row < src_row_end && src_row < src_rows; src_row++) { + const uint32_t src_off = src_row * src_row_bytes; + const uint32_t dst_off0 = (src_row * SCALE_FACTOR) * dst_row_bytes; + const uint32_t dst_off1 = dst_off0 + dst_row_bytes; + const int last_row = (src_row + 1 == src_row_end) || (src_row + 1 == src_rows); + + if (last_row) { + __load_cr(L2_A + src_off, BANK_A, + src_row_bytes, (uint16_t)src_row_bytes, + 1, (uint16_t)src_row_bytes, + 1, tid_mask, 0); + } else { + __load(L2_A + src_off, BANK_A, + src_row_bytes, (uint16_t)src_row_bytes, + 1, (uint16_t)src_row_bytes); + } + + for (uint32_t c = 0; c < src_width; c++) { + const uint32_t src_elem_off = c * DTYPE; + const uint32_t dst_elem_off = c * SCALE_FACTOR * DTYPE; + + __copy(BANK_A + src_elem_off, + BANK_R + dst_elem_off, + DTYPE, DTYPE, 1, DTYPE); + __copy(BANK_A + src_elem_off, + BANK_R + dst_elem_off + DTYPE, + DTYPE, DTYPE, 1, DTYPE); + } + __store(BANK_R, L2_RESULT + dst_off0, + dst_row_bytes, (uint16_t)dst_row_bytes, + 1, (uint16_t)dst_row_bytes); + + if (last_row) { + __store_cr(BANK_R, L2_RESULT + dst_off1, + dst_row_bytes, (uint16_t)dst_row_bytes, + 1, (uint16_t)dst_row_bytes, + 1, tid_mask); + } else { + __store(BANK_R, L2_RESULT + dst_off1, + dst_row_bytes, (uint16_t)dst_row_bytes, + 1, (uint16_t)dst_row_bytes); + } + } + __end_thread(tid); + } + + __end_plan(0); + __join(); + } + + return 0; +} + +int kernel_main(void * arg) { + return ggml_gtx_kernel_upscale_f16(arg); +} diff --git a/depend/ggml-gtx/src/ggml-gtx.cpp b/depend/ggml-gtx/src/ggml-gtx.cpp new file mode 100644 index 0000000..6f88042 --- /dev/null +++ b/depend/ggml-gtx/src/ggml-gtx.cpp @@ -0,0 +1,8504 @@ +/** + * @file ggml-gtx.cpp + * @brief GGML GTX NPU Backend Implementation + * + * Host-side GGML backend for GTX NPU accelerator. + * Communicates with the NPU firmware via PCIe. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "ggml-gtx.h" +#include "ggml-gtx-baremetal.h" +#include "ggml-gtx/ggml-gtx-kernel-args.h" +#include "ggml-gtx/ggml-gtx-protocol.h" +#include "ggml-gtx-server.h" +#include "ggml-impl.h" +#include "ggml-quants.h" +#include "ggml-backend-impl.h" +#include "pcie_irq_host.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +/* Memory barrier for Windows */ +#define MEMORY_BARRIER() _ReadWriteBarrier(); MemoryBarrier() +#else +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +/* Memory barrier for GCC/Clang */ +#define MEMORY_BARRIER() __sync_synchronize() +#endif + +/*============================================================================= + * Debug Logging + *============================================================================*/ + +static bool g_gtx_debug = false; + +static std::string gtx_default_vfio_socket_path() { + return "/tmp/gtx-ggml-cuda-vfio.sock"; +} + +static const char * gtx_selected_transport() { + const char * transport = getenv("GTX_TRANSPORT"); + return transport != nullptr && transport[0] != '\0' ? transport : "xdma"; +} + +static bool gtx_use_uart_vp_transport() { + const char * transport = gtx_selected_transport(); + return strcmp(transport, "uart-vp") == 0 || strcmp(transport, "xdma") == 0; +} + +static bool gtx_use_xdma_stream_transport() { + return strcmp(gtx_selected_transport(), "xdma") == 0; +} + +static const char * gtx_env_or_default(const char * name, const char * fallback) { + const char * value = getenv(name); + return value != nullptr && value[0] != '\0' ? value : fallback; +} + +static bool gtx_env_truthy(const char * name) { + const char * value = getenv(name); + return value != nullptr && value[0] != '\0' && + strcmp(value, "0") != 0 && + strcmp(value, "false") != 0 && + strcmp(value, "FALSE") != 0 && + strcmp(value, "off") != 0 && + strcmp(value, "OFF") != 0; +} + +static bool gtx_log_ops_enabled() { + return gtx_env_truthy("GTX_LOG_OPS"); +} + +static bool gtx_log_supports_ops_enabled() { + return gtx_env_truthy("GTX_LOG_SUPPORTS_OPS"); +} + +static bool gtx_log_chain_ops_enabled() { + return gtx_env_truthy("GTX_LOG_CHAIN_OPS"); +} + +static bool gtx_tiling_scheduler_enabled() { + return gtx_env_truthy("GTX_ENABLE_TILING_SCHED"); +} + +static bool gtx_dump_graph_types_enabled() { + return gtx_env_truthy("GTX_GRAPH_TYPES"); +} + +static bool gtx_log_softmax_enabled() { + return gtx_env_truthy("GTX_LOG_SOFTMAX"); +} + +static bool gtx_dump_op_samples_enabled() { + return gtx_env_truthy("GTX_DUMP_OP_SAMPLES"); +} + +static bool gtx_check_mul_mat_enabled() { + return gtx_env_truthy("GTX_CHECK_MUL_MAT"); +} + +static bool gtx_check_ops_enabled() { + return gtx_env_truthy("GTX_CHECK_OPS"); +} + +static uint64_t gtx_cpu_ref_graph_sequence = 0; +static uint64_t gtx_cpu_ref_current_graph = 0; +static bool gtx_cpu_ref_first_divergence_found = false; + +static bool gtx_cpu_ref_enabled() { + const char * dir = getenv("GTX_CPU_REF_DIR"); + return dir != nullptr && dir[0] != '\0'; +} + +static bool gtx_cpu_ref_path(char * path, size_t size, int node_index) { + const char * dir = getenv("GTX_CPU_REF_DIR"); + if (dir == nullptr || dir[0] == '\0' || path == nullptr || size == 0 || node_index < 0) { + return false; + } + const int n = snprintf(path, size, "%s/graph_%03llu_node_%04d.f32", + dir, (unsigned long long)gtx_cpu_ref_current_graph, node_index); + return n > 0 && (size_t)n < size; +} + +static bool gtx_cpu_ref_exists(int node_index) { + char path[1024]; + if (!gtx_cpu_ref_path(path, sizeof(path), node_index)) { + return false; + } + FILE * file = fopen(path, "rb"); + if (file == nullptr) { + return false; + } + fclose(file); + return true; +} + +static float gtx_float_from_bits(uint32_t bits) { + float value; + memcpy(&value, &bits, sizeof(value)); + return value; +} + +static const char * gtx_op_name_or_unknown(uint32_t raw_op) { + if (raw_op < (uint32_t) GGML_OP_COUNT) { + return ggml_op_name((ggml_op) raw_op); + } + if (raw_op == (uint32_t) GGML_OP_COUNT) { + return "HOST_OP_COUNT_OR_SHIFTED_GLU"; + } + return "UNKNOWN_HOST_OP"; +} + +static const char * gtx_normalized_op_name(uint32_t op) { + if (op == UINT32_MAX) { + return "UNSUPPORTED_HOST_OP"; + } + if (op == (uint32_t) GGML_OP_CONV_TRANSPOSE_2D) { + return "CONV_TRANSPOSE_2D"; + } + if (op == (uint32_t) GGML_OP_UPSCALE) { + return "UPSCALE"; + } + if (op == (uint32_t) GGML_OP_PAD) { + return "PAD"; + } + if (op == (uint32_t) GGML_OP_LEAKY_RELU) { + return "LEAKY_RELU"; + } + if (op == (uint32_t) GGML_OP_CLAMP) { + return "CLAMP"; + } + if (op == (uint32_t) GGML_OP_GLU) { + return "GLU"; + } + if (op == (uint32_t) GGML_OP_UNARY) { + return "UNARY"; + } + return gtx_op_name_or_unknown(op); +} + +static bool gtx_env_list_contains(const char * name, const char * needle) { + const char * value = getenv(name); + if (value == nullptr || value[0] == '\0' || needle == nullptr || needle[0] == '\0') { + return false; + } + + const std::string haystack(value); + size_t start = 0; + while (start <= haystack.size()) { + size_t end = haystack.find(',', start); + if (end == std::string::npos) { + end = haystack.size(); + } + const std::string item = haystack.substr(start, end - start); + if (item == needle) { + return true; + } + if (end == haystack.size()) { + break; + } + start = end + 1; + } + return false; +} + +static bool gtx_env_substr_matches(const char * name, const char * value_to_check) { + const char * value = getenv(name); + return value != nullptr && value[0] != '\0' && + value_to_check != nullptr && strstr(value_to_check, value) != nullptr; +} + +static size_t gtx_env_size_kib_or_default(const char * name, size_t fallback) { + const char * value = getenv(name); + if (value == nullptr || value[0] == '\0') { + return fallback; + } + + char * end = nullptr; + unsigned long long kib = strtoull(value, &end, 10); + if (end == value || kib == 0) { + return fallback; + } + return (size_t) kib * 1024ULL; +} + +static uint64_t gtx_env_u64_or_default(const char * name, uint64_t fallback) { + const char * value = getenv(name); + if (value == nullptr || value[0] == '\0') { + return fallback; + } + + errno = 0; + char * end = nullptr; + unsigned long long parsed = strtoull(value, &end, 0); + if (end == value || errno != 0) { + return fallback; + } + return (uint64_t) parsed; +} + +#define GTX_LOG(...) \ + do { if (g_gtx_debug) fprintf(stderr, "[GTX] " __VA_ARGS__); } while(0) + +#define GTX_LOG_WARN(...) \ + fprintf(stderr, "[GTX WARN] " __VA_ARGS__) + +#define GTX_LOG_ERROR(...) \ + fprintf(stderr, "[GTX ERROR] " __VA_ARGS__) + +static uint16_t gtx_fp32_to_fp16(float x) { + return GGML_FP32_TO_FP16(x); +} + +static float gtx_fp16_to_fp32(uint16_t h) { + return GGML_FP16_TO_FP32(h); +} + +static bool gtx_env_enabled(const char * name) { + const char * value = getenv(name); + return value != nullptr && value[0] != '\0' && strcmp(value, "0") != 0; +} + +static bool gtx_name_ends_with(const char * name, const char * suffix) { + if (name == nullptr || suffix == nullptr) { + return false; + } + const size_t name_len = strlen(name); + const size_t suffix_len = strlen(suffix); + return name_len >= suffix_len && strcmp(name + name_len - suffix_len, suffix) == 0; +} + +static bool gtx_name_equals(const char * name, const char * expected) { + return name != nullptr && expected != nullptr && strcmp(name, expected) == 0; +} + +static float gtx_env_float_or_default(const char * name, float fallback) { + const char * value = getenv(name); + if (value == nullptr || value[0] == '\0') { + return fallback; + } + + char * end = nullptr; + const float parsed = strtof(value, &end); + if (end == value || !std::isfinite(parsed) || parsed <= 0.0f) { + return fallback; + } + return parsed; +} + +static bool gtx_is_norm_weight_tensor(const ggml_tensor * tensor) { + return tensor != nullptr && gtx_name_ends_with(tensor->name, "norm.weight"); +} + +static bool gtx_is_bias_tensor(const ggml_tensor * tensor) { + return tensor != nullptr && strstr(tensor->name, "bias") != nullptr; +} + +static bool gtx_is_output_head_weight_tensor(const ggml_tensor * tensor) { + return tensor != nullptr && + (gtx_name_equals(tensor->name, "output_norm.weight") || + gtx_name_equals(tensor->name, "token_embd.weight")); +} + +static bool gtx_mul_norm_matmul_scale_enabled() { + static const bool enabled = gtx_env_enabled("GTX_MUL_NORM_MATMUL_SCALE"); + return enabled; +} + +static bool gtx_norm_weight_f32_upload_enabled() { + return gtx_env_truthy("GTX_NORM_WEIGHT_F32_UPLOAD"); +} + +static bool gtx_bias_f32_upload_enabled() { + return gtx_env_truthy("GTX_BIAS_F32_UPLOAD"); +} + +static float gtx_head_weight_scale() { + static const float scale = gtx_env_float_or_default("GTX_HEAD_WEIGHT_SCALE", 1.0f); + return scale; +} + +static bool gtx_is_mul_norm_result(const ggml_tensor * tensor) { + return tensor != nullptr && + tensor->op == GGML_OP_MUL && + gtx_is_norm_weight_tensor(tensor->src[1]); +} + +static bool gtx_is_mul_norm_matmul_result(const ggml_tensor * tensor) { + return tensor != nullptr && + tensor->op == GGML_OP_MUL_MAT && + gtx_is_mul_norm_result(tensor->src[1]); +} + +#define GGML_GTX_MUL_NORM_UPLOAD_SCALE 0.001f +#define GGML_GTX_MUL_NORM_HOST_RESCALE 1000.0f + +static float gtx_upload_f16_scale(const ggml_tensor * tensor) { + const float head_scale = gtx_head_weight_scale(); + if (head_scale != 1.0f && gtx_is_output_head_weight_tensor(tensor)) { + return head_scale; + } + + if (gtx_mul_norm_matmul_scale_enabled() && gtx_is_norm_weight_tensor(tensor)) { + return GGML_GTX_MUL_NORM_UPLOAD_SCALE; + } + + return 1.0f; +} + +static enum ggml_type gtx_remote_type(enum ggml_type type) { + switch (type) { + case GGML_TYPE_F32: + case GGML_TYPE_Q8_0: + return GGML_TYPE_F16; + default: + return type; + } +} + +static bool gtx_preserve_remote_f32(const ggml_tensor * tensor) { + if (tensor == nullptr || tensor->type != GGML_TYPE_F32) { + return false; + } + + return (gtx_is_norm_weight_tensor(tensor) && gtx_norm_weight_f32_upload_enabled()) || + (gtx_is_bias_tensor(tensor) && gtx_bias_f32_upload_enabled()); +} + +static enum ggml_type gtx_remote_tensor_type(const ggml_tensor * tensor) { + if (gtx_preserve_remote_f32(tensor)) { + return GGML_TYPE_F32; + } + return tensor != nullptr ? gtx_remote_type(tensor->type) : GGML_TYPE_F32; +} + +static size_t gtx_remote_size_bytes(const ggml_tensor * tensor) { + if (tensor == nullptr) { + return 0; + } + + if (gtx_preserve_remote_f32(tensor)) { + return ggml_nbytes(tensor); + } + + if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_Q8_0) { + return (size_t) ggml_nelements(tensor) * sizeof(uint16_t); + } + + return ggml_nbytes(tensor); +} + +static uint64_t gtx_remote_byte_offset(const ggml_tensor * tensor, uint64_t local_bytes) { + if (tensor != nullptr) { + if (gtx_preserve_remote_f32(tensor)) { + return local_bytes; + } + + if (tensor->type == GGML_TYPE_F32) { + return (local_bytes / sizeof(float)) * sizeof(uint16_t); + } + + if (tensor->type == GGML_TYPE_Q8_0) { + const uint64_t type_size = (uint64_t) ggml_type_size(tensor->type); + const uint64_t blck_size = (uint64_t) ggml_blck_size(tensor->type); + return (local_bytes / type_size) * blck_size * sizeof(uint16_t); + } + } + + return local_bytes; +} + +static uint64_t gtx_remote_data_ptr(const ggml_tensor * tensor); + +static std::vector gtx_pack_tensor_chunk(const ggml_tensor * tensor, const void * data, size_t size) { + if (tensor->type == GGML_TYPE_Q8_0) { + const size_t block_size = ggml_blck_size(tensor->type); + const size_t type_size = ggml_type_size(tensor->type); + const size_t n_blocks = size / type_size; + const size_t n = n_blocks * block_size; + + std::vector tmp(n); + dequantize_row_q8_0((const block_q8_0 *) data, tmp.data(), (int64_t) n); + + std::vector packed(n * sizeof(uint16_t)); + uint16_t * dst = (uint16_t *) packed.data(); + const float scale = gtx_upload_f16_scale(tensor); + for (size_t i = 0; i < n; ++i) { + dst[i] = gtx_fp32_to_fp16(tmp[i] * scale); + } + + return packed; + } + + if (tensor->type != GGML_TYPE_F32 || gtx_preserve_remote_f32(tensor)) { + const uint8_t * bytes = (const uint8_t *) data; + return std::vector(bytes, bytes + size); + } + + const size_t n = size / sizeof(float); + const float * src = (const float *) data; + std::vector packed(n * sizeof(uint16_t)); + uint16_t * dst = (uint16_t *) packed.data(); + const float scale = gtx_upload_f16_scale(tensor); + for (size_t i = 0; i < n; ++i) { + dst[i] = gtx_fp32_to_fp16(src[i] * scale); + } + return packed; +} + +static void gtx_unpack_tensor_chunk(const ggml_tensor * tensor, const uint8_t * src, void * data, size_t size) { + if (tensor->type != GGML_TYPE_F32 || gtx_preserve_remote_f32(tensor)) { + memcpy(data, src, size); + return; + } + + const size_t n = size / sizeof(float); + const uint16_t * src16 = (const uint16_t *) src; + float * dst = (float *) data; + const float scale = + gtx_mul_norm_matmul_scale_enabled() && gtx_is_mul_norm_matmul_result(tensor) ? + GGML_GTX_MUL_NORM_HOST_RESCALE : 1.0f; + for (size_t i = 0; i < n; ++i) { + dst[i] = gtx_fp16_to_fp32(src16[i]) * scale; + } +} + +static size_t gtx_upload_local_chunk_size(const ggml_tensor * tensor, size_t remaining, size_t max_chunk) { + if (tensor != nullptr && tensor->type == GGML_TYPE_Q8_0) { + const size_t type_size = ggml_type_size(tensor->type); + const size_t packed_block_size = (size_t) ggml_blck_size(tensor->type) * sizeof(uint16_t); + size_t chunk = std::min(remaining, (max_chunk / packed_block_size) * type_size); + if (chunk == 0) { + chunk = std::min(remaining, type_size); + } + return chunk; + } + + if (tensor != nullptr && tensor->type == GGML_TYPE_F32) { + const size_t packed_element_size = + gtx_preserve_remote_f32(tensor) ? sizeof(float) : sizeof(uint16_t); + const size_t max_elements = max_chunk / packed_element_size; + size_t chunk = std::min(remaining, max_elements * sizeof(float)); + if (chunk == 0) { + chunk = std::min(remaining, sizeof(float)); + } + return chunk; + } + + return std::min(remaining, max_chunk); +} + +static size_t ggml_backend_gtx_buffer_type_get_alloc_size( + ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor) { + (void) buft; + return std::max(gtx_remote_size_bytes(tensor), ggml_nbytes(tensor)); +} + +/*============================================================================= + * RPC Protocol (must match ggml_gtx_server.h) + *============================================================================*/ + +enum gtx_cmd { + GTX_CMD_ALLOC_BUFFER = 0, + GTX_CMD_GET_ALIGNMENT = 1, + GTX_CMD_GET_MAX_SIZE = 2, + GTX_CMD_BUFFER_GET_BASE = 3, + GTX_CMD_FREE_BUFFER = 4, + GTX_CMD_BUFFER_CLEAR = 5, + GTX_CMD_SET_TENSOR = 6, + GTX_CMD_SET_TENSOR_HASH = 7, + GTX_CMD_GET_TENSOR = 8, + GTX_CMD_COPY_TENSOR = 9, + GTX_CMD_GRAPH_COMPUTE = 10, + GTX_CMD_GET_DEVICE_MEMORY = 11, + GTX_CMD_INIT_TENSOR = 12, + GTX_CMD_GET_ALLOC_SIZE = 13, + GTX_CMD_HELLO = 14, + GTX_CMD_DEVICE_COUNT = 15, + GTX_CMD_GRAPH_RECOMPUTE = 16, + GTX_CMD_COUNT +}; + +static const char * gtx_cmd_name(uint8_t cmd) { + switch (cmd) { + case GTX_CMD_ALLOC_BUFFER: return "ALLOC_BUFFER"; + case GTX_CMD_GET_ALIGNMENT: return "GET_ALIGNMENT"; + case GTX_CMD_GET_MAX_SIZE: return "GET_MAX_SIZE"; + case GTX_CMD_BUFFER_GET_BASE: return "BUFFER_GET_BASE"; + case GTX_CMD_FREE_BUFFER: return "FREE_BUFFER"; + case GTX_CMD_BUFFER_CLEAR: return "BUFFER_CLEAR"; + case GTX_CMD_SET_TENSOR: return "SET_TENSOR"; + case GTX_CMD_SET_TENSOR_HASH: return "SET_TENSOR_HASH"; + case GTX_CMD_GET_TENSOR: return "GET_TENSOR"; + case GTX_CMD_COPY_TENSOR: return "COPY_TENSOR"; + case GTX_CMD_GRAPH_COMPUTE: return "GRAPH_COMPUTE"; + case GTX_CMD_GET_DEVICE_MEMORY: return "GET_DEVICE_MEMORY"; + case GTX_CMD_INIT_TENSOR: return "INIT_TENSOR"; + case GTX_CMD_GET_ALLOC_SIZE: return "GET_ALLOC_SIZE"; + case GTX_CMD_HELLO: return "HELLO"; + case GTX_CMD_DEVICE_COUNT: return "DEVICE_COUNT"; + case GTX_CMD_GRAPH_RECOMPUTE: return "GRAPH_RECOMPUTE"; + default: return "UNKNOWN"; + } +} + +#pragma pack(push, 1) + +/* 64-byte message structure (must match device) */ +struct gtx_message { + uint32_t seq_num; + uint32_t type; + uint32_t resource; + uint32_t size; + uint32_t flags; + uint32_t reserved; + uint64_t address; + uint64_t value; + uint8_t payload[24]; +}; +static_assert(sizeof(gtx_message) == 64, "gtx_message must be 64 bytes"); + +/* Response flag bit */ +#define GTX_FLAG_RESPONSE (1U << 31) + +/* RPC message structures */ +struct gtx_msg_hello_rsp { + uint8_t major; + uint8_t minor; + uint8_t patch; +}; + +struct gtx_msg_device_count_rsp { + uint32_t device_count; +}; + +struct gtx_msg_alloc_buffer_req { + uint32_t device; + uint64_t size; +}; + +struct gtx_msg_alloc_buffer_rsp { + uint64_t remote_ptr; + uint64_t remote_size; +}; + +struct gtx_msg_get_alignment_rsp { + uint64_t alignment; +}; + +struct gtx_msg_get_max_size_rsp { + uint64_t max_size; +}; + +struct gtx_msg_free_buffer_req { + uint64_t remote_ptr; +}; + +struct gtx_msg_buffer_clear_req { + uint64_t remote_ptr; + uint8_t value; +}; + +struct gtx_msg_get_device_memory_rsp { + uint64_t free_mem; + uint64_t total_mem; +}; + +/* RPC tensor (serialized ggml_tensor) */ +struct gtx_rpc_tensor { + uint64_t id; + uint32_t type; + uint64_t buffer; + uint32_t ne[GGML_MAX_DIMS]; + uint32_t nb[GGML_MAX_DIMS]; + uint32_t op; + int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)]; + int32_t flags; + uint64_t src[GGML_MAX_SRC]; + uint64_t view_src; + uint64_t view_offs; + uint64_t data; + // ⚠️ **와이어 포맷이라 `GGML_MAX_NAME` 를 따라가면 안 된다.** 펌웨어 쪽 + // `firmware/include/ggml_gtx_rpc_protocol.h` 가 `char name[64]` 로 하드코딩돼 있다. + // 정본 트리(`device/zephyr-host`)는 자기 `ggml.h` 의 `GGML_MAX_NAME` 이 64라 + // 그대로 두어도 맞지만, vision.cpp 의 ggml 은 **128** 이라 옮겨오면 구조체가 + // 어긋나 RPC 가 조용히 깨진다. 정본을 다시 가져올 때마다 이 자리를 다시 박는다. + char name[64]; + char padding[4]; +}; + +struct gtx_msg_get_tensor_req { + gtx_rpc_tensor tensor; + uint64_t offset; + uint64_t size; +}; + +struct gtx_msg_copy_tensor_req { + gtx_rpc_tensor src; + gtx_rpc_tensor dst; +}; + +struct gtx_msg_copy_tensor_rsp { + uint8_t result; +}; + +#pragma pack(pop) + +static gtx_rpc_tensor gtx_serialize_tensor(const ggml_tensor * tensor); + +/*============================================================================= + * PCIe Communication Layer (vfio-user based) + *============================================================================*/ + +/* PCIe BAR memory map offsets */ +#define GTX_BAR_ELBI_INT_GEN 0x0E40 /* Host -> Device interrupt (in BAR0) */ + +/* Message queue is accessed via BAR2 (maps to device memory 0x60000000) + * Layout matches ggml_baremetal/drivers/message.c: + * [0-3]: RX head (host writes, device reads) + * [4-7]: RX tail (device writes, host reads) + * [8-11]: TX head (device writes, host reads) + * [12-15]: TX tail (host writes, device reads) + * [64...]: RX queue data, then TX queue data + */ +#define GTX_MSG_QUEUE_BASE 0x00 /* Message queue at BAR2 offset 0 */ +#define GTX_MSG_QUEUE_RX_HEAD 0x00 /* RX head */ +#define GTX_MSG_QUEUE_RX_TAIL 0x04 /* RX tail */ +#define GTX_MSG_QUEUE_TX_HEAD 0x08 /* TX head */ +#define GTX_MSG_QUEUE_TX_TAIL 0x0C /* TX tail */ +#define GTX_MSG_QUEUE_HEADER 64 /* Header size (64 bytes aligned) */ +#define GTX_MSG_QUEUE_SIZE (64 * 1024) /* 64KB total */ +#define GTX_MSG_QUEUE_ENTRIES ((GTX_MSG_QUEUE_SIZE - GTX_MSG_QUEUE_HEADER) / 2 / 64) /* ~511 entries per queue */ +#define GTX_MSG_QUEUE_RX_DATA GTX_MSG_QUEUE_HEADER /* RX queue data starts at offset 64 */ +#define GTX_MSG_QUEUE_TX_DATA (GTX_MSG_QUEUE_RX_DATA + GTX_MSG_QUEUE_ENTRIES * 64) /* TX queue after RX */ + +/* BAR2 shared-memory staging area for large request/response payloads */ +#define GTX_BAR2_DEVICE_BASE 0x60000000ULL +#define GTX_BAR2_STAGING_OFFSET 0x00010000ULL +#define GTX_BAR2_STAGING_SIZE (8ULL * 1024ULL * 1024ULL) +#define GTX_BAR2_UPLOAD_CHUNK (512ULL * 1024ULL) +#define GTX_BAR2_DOWNLOAD_CHUNK (16ULL * 1024ULL) +#define GTX_XDMA_DRIVER_CHUNK (255ULL * 4096ULL) +#define GTX_CMD_TIMEOUT_MS 30000 +#define GTX_GRAPH_COMPUTE_TIMEOUT_MS 600000 +#define GTX_UART_TRANSFER_TIMEOUT_MS 600000 +#define GTX_UART_KERNEL_TIMEOUT_MS 600000 + +#define GTX_DIRECT_DDR_MAGIC 0x31524447u +#define GTX_DIRECT_DDR_OP_WRITE 1u +#define GTX_DIRECT_DDR_OP_READ 2u + +struct gtx_direct_ddr_header { + uint32_t magic; + uint32_t op; + uint64_t addr; + uint32_t size; + uint32_t status; +}; + +static_assert(GTX_BAR2_UPLOAD_CHUNK <= GTX_BAR2_STAGING_SIZE, + "BAR2 upload chunk must fit in staging area"); +static_assert(GTX_BAR2_DOWNLOAD_CHUNK <= GTX_BAR2_STAGING_SIZE, + "BAR2 download chunk must fit in staging area"); + +static size_t gtx_upload_chunk_size() { + const size_t default_chunk = gtx_use_xdma_stream_transport() ? + GTX_XDMA_DRIVER_CHUNK : GTX_BAR2_UPLOAD_CHUNK; + size_t chunk = gtx_env_size_kib_or_default("GTX_UPLOAD_CHUNK_KIB", default_chunk / 1024ULL); + if (chunk < 4ULL * 1024ULL) { + chunk = 4ULL * 1024ULL; + } + if (chunk > GTX_BAR2_STAGING_SIZE) { + chunk = GTX_BAR2_STAGING_SIZE; + } + return chunk; +} + +/* BAR region indices */ +#define GTX_BAR0_REGION 0 /* Registers (doorbell, etc.) */ +#define GTX_BAR2_REGION 2 /* Shared memory / Message queue */ + +class gtx_pcie_connection { +public: + gtx_pcie_connection() : m_irq_ctx(nullptr), m_seq_num(0) {} + ~gtx_pcie_connection() { close(); } + + bool open(const char* device_path, int bar_index = 0); + void close(); + bool is_open() const { return m_irq_ctx != nullptr; } + + /* Send command and wait for response */ + bool send_cmd(uint8_t cmd, const void* request, size_t req_size, + void* response, size_t rsp_size); + + bool send_cmd_staged(uint8_t cmd, uint64_t address, uint32_t size, + void* response, size_t rsp_size); + + /* Send data to device BAR2 (for large transfers) */ + bool send_data(uint64_t offset, const void* data, size_t size); + + /* Receive data from device BAR2 */ + bool recv_data(uint64_t offset, void* data, size_t size); + + /* Send doorbell interrupt to device */ + void send_doorbell(uint32_t source = 0); + + /* Wait for MSI from device */ + bool wait_msi(int vector, int timeout_ms); + + /* Get IRQ context (for advanced use) */ + gtx_pcie_irq_ctx* get_irq_ctx() const { return m_irq_ctx; } + +private: + /* vfio-user IRQ context */ + gtx_pcie_irq_ctx* m_irq_ctx; + + /* Sequence number for request matching */ + uint32_t m_seq_num; + + /* Mutex for thread safety */ + std::mutex m_mutex; +}; + +struct ggml_backend_gtx_buffer_context; + +#ifndef _WIN32 +struct gtx_uart_kernel_bundle { + uint32_t kernel_id; + const char * relative_path; + const char * entry_symbol; + const char * name; +}; + +static constexpr uint32_t GTX_KERNEL_WORK_SLOT_COUNT = 32; +static constexpr uint64_t GTX_FPGA_KERNEL_WORK_ADDR = 0x00040000ull; +static constexpr uint64_t GTX_FPGA_KERNEL_WORK_SIZE = 0x00010000ull; +static constexpr uint32_t GTX_FPGA_KERNEL_WORK_SLOT_COUNT = 28; +// Keep the first 256 MiB for the FPGA RPC rings, arguments, stats, and kernel +// staging. The CPU-visible DDR window continues from 0x380000000 through +// 0x3ffffffff, providing a separate contiguous 2 GiB tensor arena. +static constexpr uint64_t GTX_FPGA_TENSOR_ARENA_OFFSET = 0x10000000ull; +static constexpr uint64_t GTX_FPGA_TENSOR_ARENA_SIZE = 0x80000000ull; +static constexpr uint64_t GTX_STASH_L2_BYTES = 10ull * 1024ull * 1024ull; +static constexpr uint64_t GTX_STASH_MIN_SCRATCH_BYTES = GGML_GTX_L2_MIN_SCRATCH_BYTES; +static constexpr uint64_t GTX_STASH_SLOT_COUNT = 3ull; + +static std::string gtx_kernel_bundle_dir() { + return gtx_env_or_default("GTX_KERNEL_DIR", "build-kernels"); +} + +static std::string gtx_join_path(const std::string & dir, const char * file) { + if (file == nullptr || file[0] == '\0') { + return dir; + } + if (file[0] == '/') { + return file; + } + if (dir.empty() || dir == ".") { + return file; + } + if (dir.back() == '/') { + return dir + file; + } + return dir + "/" + file; +} + +static const gtx_uart_kernel_bundle * gtx_uart_kernel_bundle_for(uint32_t kernel_id) { + static const gtx_uart_kernel_bundle bundles[] = { + { + GGML_GTX_KERNEL_ADD_F16, + "add.llext", + "ggml_gtx_kernel_add_f16", + "add-f16", + }, + { + GGML_GTX_KERNEL_MUL_F16, + "mul.llext", + "ggml_gtx_kernel_mul_f16", + "mul-f16", + }, + { + GGML_GTX_KERNEL_SCALE_F16, + "scale.llext", + "ggml_gtx_kernel_scale_f16", + "scale-f16", + }, + { + GGML_GTX_KERNEL_RMS_NORM_F16, + "rms-norm.llext", + "ggml_gtx_kernel_rms_norm_f16", + "rms-norm-f16", + }, + { + GGML_GTX_KERNEL_NORM_F16, + "norm.llext", + "ggml_gtx_kernel_norm_f16", + "norm-f16", + }, + { + GGML_GTX_KERNEL_REPEAT_F16, + "repeat.llext", + "ggml_gtx_kernel_repeat_f16", + "repeat-f16", + }, + { + GGML_GTX_KERNEL_CONCAT_F16, + "concat.llext", + "ggml_gtx_kernel_concat_f16", + "concat-f16", + }, + { + GGML_GTX_KERNEL_SOFT_MAX_F16, + "soft-max.llext", + "ggml_gtx_kernel_soft_max_f16", + "soft-max-f16", + }, + { + GGML_GTX_KERNEL_GELU_F16, + "gelu.llext", + "ggml_gtx_kernel_gelu_f16", + "gelu-f16", + }, + { + GGML_GTX_KERNEL_MUL_MAT_F16, + "mul-mat.llext", + "ggml_gtx_kernel_mul_mat_f16", + "mul-mat-f16", + }, + { + GGML_GTX_KERNEL_ROPE_F16, + "rope.llext", + "ggml_gtx_kernel_rope_f16", + "rope-f16", + }, + { + GGML_GTX_KERNEL_GET_ROWS_F16, + "get-rows.llext", + "ggml_gtx_kernel_get_rows_f16", + "get-rows-f16", + }, + { + GGML_GTX_KERNEL_SET_ROWS_F16, + "set-rows.llext", + "ggml_gtx_kernel_set_rows_f16", + "set-rows-f16", + }, + { + GGML_GTX_KERNEL_IM2COL_F16, + "im2col.llext", + "ggml_gtx_kernel_im2col_f16", + "im2col-f16", + }, + { + GGML_GTX_KERNEL_SWIGLU_F16, + "swiglu.llext", + "ggml_gtx_kernel_swiglu_f16", + "swiglu-f16", + }, + { + GGML_GTX_KERNEL_CONT_F16, + "cont.llext", + "ggml_gtx_kernel_cont_f16", + "cont-f16", + }, + { + GGML_GTX_KERNEL_CPY_F16, + "cpy.llext", + "ggml_gtx_kernel_cpy_f16", + "cpy-f16", + }, + { + GGML_GTX_KERNEL_ADD_F16_F32_TO_F16, + "add-fp32.llext", + "ggml_gtx_kernel_add_f16_f32", + "add-f16-f32", + }, + { + GGML_GTX_KERNEL_CONV_2D_F16, + "conv-2d.llext", + "ggml_gtx_kernel_conv_2d_f16", + "conv-2d-f16", + }, + { + GGML_GTX_KERNEL_CONV_2D_DW_F16, + "conv-2d-dw.llext", + "ggml_gtx_kernel_conv_2d_dw_f16", + "conv-2d-dw-f16", + }, + { + GGML_GTX_KERNEL_CONV_TRANSPOSE_2D_F16, + "conv-transpose-2d.llext", + "kernel_main", + "conv-transpose-2d-f16", + }, + { + GGML_GTX_KERNEL_UPSCALE_F16, + "upscale.llext", + "ggml_gtx_kernel_upscale_f16", + "upscale-f16", + }, + { + GGML_GTX_KERNEL_SIN_F16, + "sin.llext", + "ggml_gtx_kernel_sin_f16", + "sin-f16", + }, + { + GGML_GTX_KERNEL_ABS_F16, + "abs.llext", + "ggml_gtx_kernel_abs_f16", + "abs-f16", + }, + { + GGML_GTX_KERNEL_NEG_F16, + "neg.llext", + "ggml_gtx_kernel_neg_f16", + "neg-f16", + }, + { + GGML_GTX_KERNEL_SGN_F16, + "sgn.llext", + "ggml_gtx_kernel_sgn_f16", + "sgn-f16", + }, + { + GGML_GTX_KERNEL_CEIL_F16, + "ceil.llext", + "ggml_gtx_kernel_ceil_f16", + "ceil-f16", + }, + { + GGML_GTX_KERNEL_FLOOR_F16, + "floor.llext", + "ggml_gtx_kernel_floor_f16", + "floor-f16", + }, + { + GGML_GTX_KERNEL_TANH_F16, + "tanh.llext", + "ggml_gtx_kernel_tanh_f16", + "tanh-f16", + }, + { + GGML_GTX_KERNEL_SILU_F16, + "silu.llext", + "ggml_gtx_kernel_silu_f16", + "silu-f16", + }, + { + GGML_GTX_KERNEL_RELU_F16, + "relu.llext", + "ggml_gtx_kernel_relu_f16", + "relu-f16", + }, + { + GGML_GTX_KERNEL_LEAKY_RELU_F16, + "leaky-relu.llext", + "ggml_gtx_kernel_leaky_relu_f16", + "leaky-relu-f16", + }, + { + GGML_GTX_KERNEL_CLAMP_F16, + "clamp.llext", + "ggml_gtx_kernel_clamp_f16", + "clamp-f16", + }, + { + GGML_GTX_KERNEL_COS_F16, + "cos.llext", + "ggml_gtx_kernel_cos_f16", + "cos-f16", + }, + { + GGML_GTX_KERNEL_PAD_F16, + "pad.llext", + "ggml_gtx_kernel_pad_f16", + "pad-f16", + }, + }; + + for (const gtx_uart_kernel_bundle & bundle : bundles) { + if (bundle.kernel_id == kernel_id) { + return &bundle; + } + } + return nullptr; +} + +static uint64_t gtx_kernel_work_size() { + return gtx_use_xdma_stream_transport() ? GTX_FPGA_KERNEL_WORK_SIZE + : GTX_FIXED_KERNEL_WORK_SIZE; +} + +static uint64_t gtx_kernel_work_base_addr() { + const uint64_t fallback = gtx_use_xdma_stream_transport() ? + GTX_FPGA_KERNEL_WORK_ADDR : + GTX_FIXED_KERNEL_WORK_ADDR; + return gtx_env_u64_or_default("GTX_KERNEL_WORK_ADDR", fallback); +} + +static uint64_t gtx_kernel_work_addr_for_slot(uint32_t slot) { + return gtx_kernel_work_base_addr() + (uint64_t) slot * gtx_kernel_work_size(); +} + +static uint64_t gtx_fixed_ddr_base() { + return gtx_env_u64_or_default("GTX_FIXED_DDR_BASE", GTX_DDR_BASE); +} + +static uint64_t gtx_fixed_arg_addr() { + return gtx_fixed_ddr_base() + + (gtx_use_xdma_stream_transport() ? 0x00300000ull : 0x01000000ull); +} + +static uint64_t gtx_fixed_tensor_arena_addr() { + return gtx_fixed_ddr_base() + + (gtx_use_xdma_stream_transport() ? GTX_FPGA_TENSOR_ARENA_OFFSET + : 0x01100000ull); +} + +static uint64_t gtx_fixed_tensor_arena_size() { + return gtx_use_xdma_stream_transport() ? GTX_FPGA_TENSOR_ARENA_SIZE + : GTX_FIXED_TENSOR_ARENA_SIZE; +} + +static bool gtx_range_overlaps(uint64_t a_addr, uint64_t a_size, + uint64_t b_addr, uint64_t b_size) { + if (a_size == 0 || b_size == 0) { + return false; + } + return a_addr < b_addr + b_size && b_addr < a_addr + a_size; +} + +static bool gtx_range_overlaps_tensor_arena(uint64_t addr, uint64_t size) { + return gtx_range_overlaps(addr, + size, + gtx_fixed_tensor_arena_addr(), + gtx_fixed_tensor_arena_size()); +} + +static uint64_t gtx_fixed_kernel_staging_addr() { + return gtx_fixed_ddr_base() + + (gtx_use_xdma_stream_transport() ? 0x0ff00000ull : 0x7ff00000ull); +} + +#ifndef _WIN32 +static constexpr uint64_t GTX_XDMA_HOST_DDR_BASE = 0x400000000ull; +static constexpr uint64_t GTX_XDMA_RX_OFFSET = 0x00100000ull; +static constexpr uint64_t GTX_XDMA_TX_OFFSET = 0x00200000ull; +static constexpr uint32_t GTX_XDMA_RING_SIZE = 0x00100000u; +static constexpr uint32_t GTX_XDMA_STREAM_MAGIC = 0x31534458u; + +struct gtx_xdma_stream_ctrl_host { + uint32_t magic; + uint32_t version; + uint32_t rx_head; + uint32_t rx_tail; + uint32_t tx_head; + uint32_t tx_tail; + uint32_t state; + uint32_t heartbeat; + uint64_t last_mcause; + uint64_t last_mepc; + uint64_t last_mtval; + uint8_t reserved[64]; +}; +#endif + +class gtx_uart_vp_connection { +public: + gtx_uart_vp_connection() = default; + ~gtx_uart_vp_connection() { close(); } + + bool open(); + void close(); + bool is_open() const { return m_open; } + + bool upload_bytes(uint64_t addr, const void * data, size_t size); + bool download_bytes(uint64_t addr, void * data, size_t size); + bool copy_tensor(const ggml_tensor * src, const ggml_tensor * dst); + bool stash_flush_reset(); + bool launch(const ggml_gtx_kernel_launch_desc_t & desc, + const void * kernel_args = nullptr, + size_t kernel_args_size = 0); + +private: + bool write_all(const void * data, size_t size); + bool read_byte(uint8_t & byte, int timeout_ms); + bool read_exact(void * data, size_t size, int timeout_ms); + bool direct_ddr_enabled() const; + bool direct_ddr_open(); + void direct_ddr_close(); + bool direct_ddr_write_all(const void * data, size_t size); + bool direct_ddr_read_all(void * data, size_t size); + bool direct_ddr_transfer(uint32_t op, uint64_t addr, void * data, size_t size); + bool stash_flush_range(uint64_t addr, size_t size); + bool stash_invalidate_range(uint64_t addr, size_t size); + bool read_frame(ggml_gtx_rpc_frame_t & frame, std::vector & payload, int timeout_ms); + bool rpc(uint16_t cmd, const void * payload, size_t payload_size, + std::vector * response_payload, int timeout_ms = 30000); + bool assign_kernel_work_addr(uint32_t kernel_id, uint64_t & work_addr); + bool ensure_kernel_loaded(const gtx_uart_kernel_bundle & bundle); + static std::vector read_file(const char * path); + + int m_fd = -1; + int m_direct_fd = -1; +#ifndef _WIN32 + int m_xdma_h2c_fd = -1; + int m_xdma_c2h_fd = -1; + int m_xdma_user_fd = -1; + volatile uint32_t * m_xdma_user_bar = nullptr; + uint32_t m_xdma_rx_head = 0; + uint32_t m_xdma_tx_tail = 0; + bool m_xdma = false; +#endif + uint32_t m_seq = 0; + bool m_open = false; + bool m_direct_disabled = false; + bool m_stash_may_be_resident = false; + uint32_t m_next_kernel_work_slot = 0; + std::unordered_map m_kernel_work_addrs; + std::unordered_set m_loaded_kernels; + std::string m_uart_log; +}; + +bool gtx_uart_vp_connection::open() { + if (m_open) { + return true; + } + + if (gtx_use_xdma_stream_transport()) { + const char * h2c = gtx_env_or_default("GTX_XDMA_H2C", "/dev/xdma0_h2c_0"); + const char * c2h = gtx_env_or_default("GTX_XDMA_C2H", "/dev/xdma0_c2h_0"); + const char * user = gtx_env_or_default("GTX_XDMA_USER", "/dev/xdma0_user"); + m_xdma_h2c_fd = ::open(h2c, O_WRONLY); + m_xdma_c2h_fd = ::open(c2h, O_RDONLY); + m_xdma_user_fd = ::open(user, O_RDWR | O_SYNC); + if (m_xdma_h2c_fd < 0 || m_xdma_c2h_fd < 0 || m_xdma_user_fd < 0) { + GTX_LOG_ERROR("failed to open XDMA stream devices: errno=%d (%s)\n", + errno, strerror(errno)); + close(); + return false; + } + void * bar = mmap(nullptr, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, + m_xdma_user_fd, 0); + if (bar == MAP_FAILED) { + m_xdma_user_bar = nullptr; + GTX_LOG_ERROR("failed to map XDMA user BAR: errno=%d (%s)\n", + errno, strerror(errno)); + close(); + return false; + } + m_xdma_user_bar = (volatile uint32_t *) bar; + + gtx_xdma_stream_ctrl_host ctrl = {}; + if (pread(m_xdma_c2h_fd, &ctrl, sizeof(ctrl), + (off_t) GTX_XDMA_HOST_DDR_BASE) != (ssize_t) sizeof(ctrl) || + ctrl.magic != GTX_XDMA_STREAM_MAGIC || ctrl.version != 1u) { + GTX_LOG_ERROR("XDMA firmware stream is not ready (magic=0x%08x version=%u)\n", + ctrl.magic, ctrl.version); + close(); + return false; + } + m_xdma_rx_head = ctrl.rx_head; + m_xdma_tx_tail = ctrl.tx_tail; + m_xdma = true; + m_open = true; + GTX_LOG("connected FPGA XDMA RPC stream\n"); + return true; + } + + const char * endpoint = gtx_env_or_default("GTX_RPC_ENDPOINT", "unix:/tmp/gtx-vp-uart.sock"); + const char * prefix = "unix:"; + if (strncmp(endpoint, prefix, strlen(prefix)) != 0) { + GTX_LOG_ERROR("unsupported GTX_RPC_ENDPOINT '%s' (expected unix:/path)\n", endpoint); + return false; + } + + const char * path = endpoint + strlen(prefix); + if (strlen(path) >= sizeof(sockaddr_un::sun_path)) { + GTX_LOG_ERROR("GTX_RPC_ENDPOINT path too long: %s\n", path); + return false; + } + + m_fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (m_fd < 0) { + GTX_LOG_ERROR("failed to create UART proxy socket\n"); + return false; + } + + sockaddr_un addr = {}; + addr.sun_family = AF_UNIX; + snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", path); + + if (connect(m_fd, (sockaddr *) &addr, sizeof(addr)) != 0) { + GTX_LOG_ERROR("failed to connect UART proxy endpoint: %s: errno=%d (%s)\n", + endpoint, errno, strerror(errno)); + close(); + return false; + } + + int flags = fcntl(m_fd, F_GETFL, 0); + if (flags >= 0) { + fcntl(m_fd, F_SETFL, flags | O_NONBLOCK); + } + + m_open = true; + return true; +} + +void gtx_uart_vp_connection::close() { + if (m_open) { + // XDMA has no socket EOF visible to firmware. Send a bounded explicit + // session marker so the physical UART trace has a reliable endpoint. + (void) rpc(GGML_GTX_RPC_DISCONNECT, nullptr, 0, nullptr, 1000); + } + direct_ddr_close(); +#ifndef _WIN32 + if (m_xdma_user_bar != nullptr) { + munmap((void *) m_xdma_user_bar, 4096); + m_xdma_user_bar = nullptr; + } + if (m_xdma_h2c_fd >= 0) ::close(m_xdma_h2c_fd); + if (m_xdma_c2h_fd >= 0) ::close(m_xdma_c2h_fd); + if (m_xdma_user_fd >= 0) ::close(m_xdma_user_fd); + m_xdma_h2c_fd = m_xdma_c2h_fd = m_xdma_user_fd = -1; + m_xdma = false; +#endif + if (m_fd >= 0) { + ::close(m_fd); + m_fd = -1; + } + m_open = false; +} + +bool gtx_uart_vp_connection::write_all(const void * data, size_t size) { +#ifndef _WIN32 + if (m_xdma) { + const uint8_t * bytes = (const uint8_t *) data; + while (size > 0) { + uint32_t rx_tail = 0; + if (pread(m_xdma_c2h_fd, &rx_tail, sizeof(rx_tail), + (off_t) (GTX_XDMA_HOST_DDR_BASE + + offsetof(gtx_xdma_stream_ctrl_host, rx_tail))) != + (ssize_t) sizeof(rx_tail)) { + return false; + } + uint32_t free_bytes = GTX_XDMA_RING_SIZE - (m_xdma_rx_head - rx_tail); + if (free_bytes == 0) { + usleep(100); + continue; + } + size_t chunk = std::min(size, free_bytes); + uint32_t pos = m_xdma_rx_head & (GTX_XDMA_RING_SIZE - 1u); + chunk = std::min(chunk, GTX_XDMA_RING_SIZE - pos); + chunk = std::min(chunk, GTX_XDMA_DRIVER_CHUNK); + if (pwrite(m_xdma_h2c_fd, bytes, chunk, + (off_t) (GTX_XDMA_HOST_DDR_BASE + GTX_XDMA_RX_OFFSET + pos)) != + (ssize_t) chunk) { + return false; + } + m_xdma_rx_head += (uint32_t) chunk; + if (pwrite(m_xdma_h2c_fd, &m_xdma_rx_head, sizeof(m_xdma_rx_head), + (off_t) (GTX_XDMA_HOST_DDR_BASE + + offsetof(gtx_xdma_stream_ctrl_host, rx_head))) != + (ssize_t) sizeof(m_xdma_rx_head)) { + return false; + } + MEMORY_BARRIER(); + m_xdma_user_bar[0x20 / sizeof(uint32_t)] = 1u; + bytes += chunk; + size -= chunk; + } + return true; + } +#endif + const uint8_t * bytes = (const uint8_t *) data; + while (size > 0) { + ssize_t n = write(m_fd, bytes, size); + if (n < 0 && errno == EINTR) { + continue; + } + if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) { + struct pollfd pfd = {}; + pfd.fd = m_fd; + pfd.events = POLLOUT; + if (poll(&pfd, 1, 30000) <= 0) { + return false; + } + continue; + } + if (n <= 0) { + return false; + } + bytes += n; + size -= (size_t) n; + } + return true; +} + + +// ⚠️ **direct-DDR 이 꺼지면 조용히 UART RPC 로 떨어진다.** 그러면 전송이 수십~수천배 +// 느려지는데 경고가 한 줄도 없다(위키 pitfall/15시간-로그없이-느림). +// 어느 자리에서 꺼졌는지 한 번만 알린다. +static void gtx_note_direct_ddr_off(int site) { + static bool told = false; + if (!told) { + told = true; + fprintf(stderr, + "[GTX WARN] direct-DDR disabled at site %d — falling back to UART RPC " + "(transfers get much slower)\n", site); + } +} + +bool gtx_uart_vp_connection::direct_ddr_enabled() const { + if (gtx_use_xdma_stream_transport()) { + return false; + } + const char * value = getenv("GTX_UART_DIRECT_DDR"); + return value == nullptr || strcmp(value, "0") != 0; +} + +bool gtx_uart_vp_connection::direct_ddr_open() { + if (!direct_ddr_enabled() || m_direct_disabled) { + return false; + } + if (m_direct_fd >= 0) { + return true; + } + + const char * endpoint = gtx_env_or_default("GTX_VP_DIRECT_ENDPOINT", "unix:/tmp/gtx-vp-ddr.sock"); + const char * prefix = "unix:"; + const char * path = endpoint; + if (strncmp(endpoint, prefix, strlen(prefix)) == 0) { + path = endpoint + strlen(prefix); + } + if (strlen(path) >= sizeof(sockaddr_un::sun_path)) { + GTX_LOG_ERROR("GTX_VP_DIRECT_ENDPOINT path too long: %s\n", path); + gtx_note_direct_ddr_off(1); + m_direct_disabled = true; + return false; + } + + m_direct_fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (m_direct_fd < 0) { + gtx_note_direct_ddr_off(2); + m_direct_disabled = true; + return false; + } + + sockaddr_un addr = {}; + addr.sun_family = AF_UNIX; + snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", path); + if (connect(m_direct_fd, (sockaddr *) &addr, sizeof(addr)) != 0) { + direct_ddr_close(); + gtx_note_direct_ddr_off(3); + m_direct_disabled = true; + GTX_LOG("direct DDR endpoint unavailable: %s\n", path); + return false; + } + + GTX_LOG("connected direct DDR endpoint: %s\n", path); + return true; +} + +void gtx_uart_vp_connection::direct_ddr_close() { + if (m_direct_fd >= 0) { + ::close(m_direct_fd); + m_direct_fd = -1; + } +} + +bool gtx_uart_vp_connection::direct_ddr_write_all(const void * data, size_t size) { + const uint8_t * bytes = (const uint8_t *) data; + while (size > 0) { + ssize_t n = write(m_direct_fd, bytes, size); + if (n < 0 && errno == EINTR) { + continue; + } + if (n <= 0) { + return false; + } + bytes += n; + size -= (size_t) n; + } + return true; +} + +bool gtx_uart_vp_connection::direct_ddr_read_all(void * data, size_t size) { + uint8_t * bytes = (uint8_t *) data; + while (size > 0) { + ssize_t n = read(m_direct_fd, bytes, size); + if (n < 0 && errno == EINTR) { + continue; + } + if (n <= 0) { + return false; + } + bytes += n; + size -= (size_t) n; + } + return true; +} + +bool gtx_uart_vp_connection::direct_ddr_transfer(uint32_t op, uint64_t addr, void * data, size_t size) { +#ifndef _WIN32 + if (m_xdma) { + const uint64_t ddr_base = gtx_fixed_ddr_base(); + if ((op != GTX_DIRECT_DDR_OP_WRITE && op != GTX_DIRECT_DDR_OP_READ) || + addr < ddr_base || size > UINT64_MAX - addr) { + return false; + } + + const uint64_t ddr_offset = addr - ddr_base; + if (ddr_offset > (uint64_t) std::numeric_limits::max() - + GTX_XDMA_HOST_DDR_BASE || + size > (uint64_t) std::numeric_limits::max() - + (GTX_XDMA_HOST_DDR_BASE + ddr_offset)) { + return false; + } + + uint8_t * bytes = (uint8_t *) data; + size_t done = 0; + while (done < size) { + const size_t chunk = + std::min(size - done, GTX_XDMA_DRIVER_CHUNK); + const off_t offset = + (off_t) (GTX_XDMA_HOST_DDR_BASE + ddr_offset + done); + const ssize_t transferred = + op == GTX_DIRECT_DDR_OP_WRITE ? + pwrite(m_xdma_h2c_fd, bytes + done, chunk, offset) : + pread(m_xdma_c2h_fd, bytes + done, chunk, offset); + if (transferred != (ssize_t) chunk) { + return false; + } + done += chunk; + } + return true; + } +#endif + + if (size > UINT32_MAX || !direct_ddr_open()) { + return false; + } + + gtx_direct_ddr_header hdr = {}; + hdr.magic = GTX_DIRECT_DDR_MAGIC; + hdr.op = op; + hdr.addr = addr; + hdr.size = (uint32_t) size; + hdr.status = 0; + + if (!direct_ddr_write_all(&hdr, sizeof(hdr)) || + (op == GTX_DIRECT_DDR_OP_WRITE && size > 0 && !direct_ddr_write_all(data, size))) { + direct_ddr_close(); + gtx_note_direct_ddr_off(4); + m_direct_disabled = true; + return false; + } + + gtx_direct_ddr_header rsp = {}; + if (!direct_ddr_read_all(&rsp, sizeof(rsp)) || + rsp.magic != GTX_DIRECT_DDR_MAGIC || + rsp.op != op || + rsp.addr != addr || + rsp.size != size || + rsp.status != 0) { + direct_ddr_close(); + gtx_note_direct_ddr_off(5); + m_direct_disabled = true; + return false; + } + + if (op == GTX_DIRECT_DDR_OP_READ && size > 0 && !direct_ddr_read_all(data, size)) { + direct_ddr_close(); + gtx_note_direct_ddr_off(6); + m_direct_disabled = true; + return false; + } + + return true; +} + +bool gtx_uart_vp_connection::stash_flush_range(uint64_t addr, size_t size) { + ggml_gtx_rpc_stash_sync_t req = {}; + req.op = GGML_GTX_RPC_STASH_FLUSH_RANGE; + req.addr = addr; + req.size = (uint64_t) size; + return rpc(GGML_GTX_RPC_STASH_SYNC, &req, sizeof(req), nullptr, + GTX_UART_KERNEL_TIMEOUT_MS); +} + +bool gtx_uart_vp_connection::stash_flush_reset() { + ggml_gtx_rpc_stash_sync_t req = {}; + req.op = GGML_GTX_RPC_STASH_FLUSH_RESET; + const bool ok = rpc(GGML_GTX_RPC_STASH_SYNC, &req, sizeof(req), nullptr, + GTX_UART_KERNEL_TIMEOUT_MS); + if (ok) { + m_stash_may_be_resident = false; + } + return ok; +} + +bool gtx_uart_vp_connection::stash_invalidate_range(uint64_t addr, size_t size) { + ggml_gtx_rpc_stash_sync_t req = {}; + req.op = GGML_GTX_RPC_STASH_INVALIDATE_RANGE; + req.addr = addr; + req.size = (uint64_t) size; + return rpc(GGML_GTX_RPC_STASH_SYNC, &req, sizeof(req), nullptr, + GTX_UART_KERNEL_TIMEOUT_MS); +} + +bool gtx_uart_vp_connection::read_byte(uint8_t & byte, int timeout_ms) { +#ifndef _WIN32 + if (m_xdma) { + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(timeout_ms); + for (;;) { + uint32_t tx_head = 0; + if (pread(m_xdma_c2h_fd, &tx_head, sizeof(tx_head), + (off_t) (GTX_XDMA_HOST_DDR_BASE + + offsetof(gtx_xdma_stream_ctrl_host, tx_head))) != + (ssize_t) sizeof(tx_head)) { + return false; + } + if (m_xdma_tx_tail != tx_head) { + uint32_t pos = m_xdma_tx_tail & (GTX_XDMA_RING_SIZE - 1u); + if (pread(m_xdma_c2h_fd, &byte, 1, + (off_t) (GTX_XDMA_HOST_DDR_BASE + GTX_XDMA_TX_OFFSET + pos)) != 1) { + return false; + } + ++m_xdma_tx_tail; + return pwrite(m_xdma_h2c_fd, &m_xdma_tx_tail, sizeof(m_xdma_tx_tail), + (off_t) (GTX_XDMA_HOST_DDR_BASE + + offsetof(gtx_xdma_stream_ctrl_host, tx_tail))) == + (ssize_t) sizeof(m_xdma_tx_tail); + } + if (std::chrono::steady_clock::now() >= deadline) return false; + usleep(100); + } + } +#endif + struct pollfd pfd = {}; + pfd.fd = m_fd; + pfd.events = POLLIN; + int ret = poll(&pfd, 1, timeout_ms); + if (ret <= 0) { + return false; + } + ssize_t n = read(m_fd, &byte, 1); + if (n < 0 && errno == EINTR) { + return read_byte(byte, timeout_ms); + } + return n == 1; +} + +bool gtx_uart_vp_connection::read_exact(void * data, size_t size, int timeout_ms) { + uint8_t * bytes = (uint8_t *) data; + while (size > 0) { + uint8_t ch = 0; + if (!read_byte(ch, timeout_ms)) { + return false; + } + *bytes++ = ch; + --size; + } + return true; +} + +bool gtx_uart_vp_connection::read_frame(ggml_gtx_rpc_frame_t & frame, + std::vector & payload, + int timeout_ms) { + uint32_t magic = 0; + while (true) { + while (magic != GGML_GTX_RPC_MAGIC) { + uint8_t ch = 0; + if (!read_byte(ch, timeout_ms)) { + return false; + } + if (ch == '\n' || (ch >= 0x20 && ch <= 0x7e)) { + m_uart_log.push_back((char) ch); + if (m_uart_log.size() > 8192) { + m_uart_log.erase(0, m_uart_log.size() - 8192); + } + } + magic = (magic >> 8) | ((uint32_t) ch << 24); + } + + frame.magic = GGML_GTX_RPC_MAGIC; + if (!read_exact(((uint8_t *) &frame) + sizeof(frame.magic), + sizeof(frame) - sizeof(frame.magic), timeout_ms)) { + return false; + } + + if (frame.version == GGML_GTX_RPC_VERSION && + frame.header_size == sizeof(frame) && + frame.payload_size <= (16u * 1024u * 1024u)) { + break; + } + + magic = 0; + m_uart_log.append("[discarded malformed GTX RPC candidate]\n"); + if (m_uart_log.size() > 8192) { + m_uart_log.erase(0, m_uart_log.size() - 8192); + } + } + + payload.resize(frame.payload_size); + return payload.empty() || read_exact(payload.data(), payload.size(), timeout_ms); +} + +bool gtx_uart_vp_connection::rpc(uint16_t cmd, const void * payload, size_t payload_size, + std::vector * response_payload, int timeout_ms) { + ggml_gtx_rpc_frame_t frame = {}; + frame.magic = GGML_GTX_RPC_MAGIC; + frame.version = GGML_GTX_RPC_VERSION; + frame.header_size = sizeof(frame); + frame.cmd = cmd; + frame.seq = ++m_seq; + frame.payload_size = (uint32_t) payload_size; + + if (!write_all(&frame, sizeof(frame)) || + (payload_size > 0 && !write_all(payload, payload_size))) { + return false; + } + + while (true) { + ggml_gtx_rpc_frame_t rsp = {}; + std::vector rsp_payload; + if (!read_frame(rsp, rsp_payload, timeout_ms)) { + GTX_LOG_ERROR("UART RPC cmd=%u seq=%u timed out or received malformed frame\n", + (unsigned) cmd, (unsigned) frame.seq); + if (!m_uart_log.empty()) { + GTX_LOG_ERROR("UART log tail:\n%s\n", m_uart_log.c_str()); + } + return false; + } + if (rsp.seq != frame.seq) { + continue; + } + if (rsp.status != GGML_GTX_RPC_STATUS_OK) { + GTX_LOG_ERROR("UART RPC cmd=%u seq=%u failed status=%u\n", + (unsigned) cmd, (unsigned) frame.seq, (unsigned) rsp.status); + if (!m_uart_log.empty()) { + GTX_LOG_ERROR("UART log tail:\n%s\n", m_uart_log.c_str()); + } + return false; + } + if (response_payload != nullptr) { + *response_payload = std::move(rsp_payload); + } + return true; + } +} + +std::vector gtx_uart_vp_connection::read_file(const char * path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + return {}; + } + input.seekg(0, std::ios::end); + const std::streamoff size = input.tellg(); + input.seekg(0, std::ios::beg); + std::vector data((size_t) size); + if (!data.empty()) { + input.read((char *) data.data(), size); + } + return data; +} + +bool gtx_uart_vp_connection::upload_bytes(uint64_t addr, const void * data, size_t size) { + const bool overlaps_stash = m_stash_may_be_resident && + gtx_range_overlaps_tensor_arena(addr, size); + + // A host write may partially replace a dirty resident tile. Preserve any + // untouched bytes before the direct-DDR write, then discard the stale L2 + // mapping after it. This is intentionally range-scoped: unrelated dirty + // stash outputs remain resident. + if (overlaps_stash && !stash_flush_range(addr, size)) { + return false; + } + + if (direct_ddr_transfer(GTX_DIRECT_DDR_OP_WRITE, addr, const_cast(data), size)) { + if (overlaps_stash && !stash_invalidate_range(addr, size)) { + return false; + } + return true; + } + + ggml_gtx_rpc_upload_bytes_t req = {}; + req.addr = addr; + req.size = (uint32_t) size; + + std::vector payload(sizeof(req) + size); + memcpy(payload.data(), &req, sizeof(req)); + if (size > 0) { + memcpy(payload.data() + sizeof(req), data, size); + } + return rpc(GGML_GTX_RPC_UPLOAD_BYTES, payload.data(), payload.size(), nullptr, + GTX_UART_TRANSFER_TIMEOUT_MS); +} + +bool gtx_uart_vp_connection::download_bytes(uint64_t addr, void * data, size_t size) { + if ((!m_stash_may_be_resident || stash_flush_range(addr, size)) && + direct_ddr_transfer(GTX_DIRECT_DDR_OP_READ, addr, data, size)) { + return true; + } + + ggml_gtx_rpc_download_bytes_t req = {}; + req.addr = addr; + req.size = (uint32_t) size; + + std::vector payload; + if (!rpc(GGML_GTX_RPC_DOWNLOAD_BYTES, &req, sizeof(req), &payload, + GTX_UART_TRANSFER_TIMEOUT_MS) || + payload.size() != size) { + return false; + } + if (size > 0) { + memcpy(data, payload.data(), size); + } + return true; +} + +bool gtx_uart_vp_connection::copy_tensor(const ggml_tensor * src, const ggml_tensor * dst) { + if (src == nullptr || dst == nullptr) { + return false; + } + + gtx_msg_copy_tensor_req req = {}; + req.src = gtx_serialize_tensor(src); + req.dst = gtx_serialize_tensor(dst); + + if (req.src.data == 0 || req.dst.data == 0) { + GTX_LOG_ERROR("UART COPY_TENSOR requires GTX-backed src and dst tensors\n"); + return false; + } + + std::vector payload; + if (!rpc(GGML_GTX_RPC_COPY_TENSOR, &req, sizeof(req), &payload, + GTX_UART_TRANSFER_TIMEOUT_MS) || + payload.size() != sizeof(gtx_msg_copy_tensor_rsp)) { + GTX_LOG_ERROR("UART COPY_TENSOR command failed\n"); + return false; + } + + gtx_msg_copy_tensor_rsp rsp = {}; + memcpy(&rsp, payload.data(), sizeof(rsp)); + return rsp.result != 0; +} + +bool gtx_uart_vp_connection::assign_kernel_work_addr(uint32_t kernel_id, uint64_t & work_addr) { + auto it = m_kernel_work_addrs.find(kernel_id); + if (it != m_kernel_work_addrs.end()) { + work_addr = it->second; + return true; + } + + const uint32_t default_slot_count = gtx_use_xdma_stream_transport() ? + GTX_FPGA_KERNEL_WORK_SLOT_COUNT : + GTX_KERNEL_WORK_SLOT_COUNT; + const uint32_t slot_count = (uint32_t) gtx_env_u64_or_default( + "GTX_KERNEL_WORK_SLOT_COUNT", default_slot_count); + if (m_next_kernel_work_slot >= slot_count) { + GTX_LOG_ERROR("no free LLEXT kernel work slot: kernel_id=%u slots=%u\n", + (unsigned) kernel_id, + (unsigned) slot_count); + return false; + } + + work_addr = gtx_kernel_work_addr_for_slot(m_next_kernel_work_slot++); + m_kernel_work_addrs[kernel_id] = work_addr; + return true; +} + +bool gtx_uart_vp_connection::ensure_kernel_loaded(const gtx_uart_kernel_bundle & bundle) { + if (m_loaded_kernels.find(bundle.kernel_id) != m_loaded_kernels.end()) { + if (gtx_log_ops_enabled()) { + fprintf(stderr, "[GTX KERNEL] reuse %s id=%u\n", + bundle.name, (unsigned) bundle.kernel_id); + } + return true; + } + + const std::string kernel_path = gtx_join_path(gtx_kernel_bundle_dir(), bundle.relative_path); + std::vector binary = read_file(kernel_path.c_str()); + if (binary.empty() || binary.size() > GTX_FIXED_KERNEL_STAGING_SIZE) { + GTX_LOG_ERROR("failed to read %s kernel: %s\n", bundle.name, kernel_path.c_str()); + return false; + } + const uint64_t staging_addr = gtx_fixed_kernel_staging_addr(); + if (!upload_bytes(staging_addr, binary.data(), binary.size())) { + return false; + } + + uint64_t work_addr = 0; + if (!assign_kernel_work_addr(bundle.kernel_id, work_addr)) { + return false; + } + + if (gtx_log_ops_enabled()) { + fprintf(stderr, "[GTX KERNEL] load begin %s id=%u path=%s size=%zu staging=0x%llx work=0x%llx\n", + bundle.name, + (unsigned) bundle.kernel_id, + kernel_path.c_str(), + binary.size(), + (unsigned long long) staging_addr, + (unsigned long long) work_addr); + } + + ggml_gtx_rpc_load_kernel_t req = {}; + req.kernel_id = bundle.kernel_id; + req.format = GGML_GTX_KERNEL_FORMAT_LLEXT; + req.staging_addr = staging_addr; + req.work_addr = work_addr; + req.work_size = gtx_kernel_work_size(); + req.binary_size = (uint32_t) binary.size(); + snprintf(req.entry_symbol, sizeof(req.entry_symbol), "%s", bundle.entry_symbol); + + if (!rpc(GGML_GTX_RPC_LOAD_KERNEL, &req, sizeof(req), nullptr, 120000)) { + if (gtx_log_ops_enabled()) { + fprintf(stderr, "[GTX KERNEL] load failed %s id=%u\n", + bundle.name, (unsigned) bundle.kernel_id); + } + m_kernel_work_addrs.erase(bundle.kernel_id); + const uint64_t work_base = gtx_kernel_work_base_addr(); + const uint32_t failed_slot = (uint32_t) ((work_addr - work_base) / + gtx_kernel_work_size()); + if (failed_slot + 1 == m_next_kernel_work_slot) { + m_next_kernel_work_slot = failed_slot; + } + return false; + } + + m_loaded_kernels.insert(bundle.kernel_id); + if (gtx_log_ops_enabled()) { + fprintf(stderr, "[GTX KERNEL] load done %s id=%u\n", + bundle.name, (unsigned) bundle.kernel_id); + } + return true; +} + +bool gtx_uart_vp_connection::launch(const ggml_gtx_kernel_launch_desc_t & desc, + const void * kernel_args, + size_t kernel_args_size) { + const gtx_uart_kernel_bundle * bundle = gtx_uart_kernel_bundle_for(desc.kernel_id); + if (bundle == nullptr || !ensure_kernel_loaded(*bundle)) { + return false; + } + + if ((kernel_args == nullptr) != (kernel_args_size == 0)) { + GTX_LOG_ERROR("kernel argument payload pointer/size mismatch: kernel_id=%u size=%zu\n", + (unsigned) desc.kernel_id, kernel_args_size); + return false; + } + if (kernel_args_size > UINT32_MAX) { + GTX_LOG_ERROR("kernel argument payload is too large: kernel_id=%u size=%zu\n", + (unsigned) desc.kernel_id, kernel_args_size); + return false; + } + + if (gtx_log_ops_enabled()) { + fprintf(stderr, + "[GTX LAUNCH] UART launch %s: kernel_id=%u flags=0x%x dtype=%u(%s) src_type=[%u(%s),%u(%s),%u(%s),%u(%s)] src0=0x%llx src1=0x%llx dst=0x%llx ne=%u,%u,%u,%u\n", + bundle->name, + (unsigned) desc.kernel_id, + (unsigned) desc.flags, + (unsigned) desc.dtype, + ggml_type_name((ggml_type) desc.dtype), + (unsigned) desc.src_type[0], + ggml_type_name((ggml_type) desc.src_type[0]), + (unsigned) desc.src_type[1], + ggml_type_name((ggml_type) desc.src_type[1]), + (unsigned) desc.src_type[2], + ggml_type_name((ggml_type) desc.src_type[2]), + (unsigned) desc.src_type[3], + ggml_type_name((ggml_type) desc.src_type[3]), + (unsigned long long) desc.src_addr[0], + (unsigned long long) desc.src_addr[1], + (unsigned long long) desc.dst_addr, + desc.ne[0], desc.ne[1], desc.ne[2], desc.ne[3]); + } + ggml_gtx_kernel_arg_t arg = {}; + arg.magic = GGML_GTX_KERNEL_ARG_MAGIC; + arg.version = GGML_GTX_KERNEL_ARG_VERSION; + arg.size = sizeof(arg); + arg.kernel_id = desc.kernel_id; + arg.desc = desc; + + const uint64_t arg_addr = gtx_fixed_arg_addr(); + const size_t payload_offset = (sizeof(arg) + 15u) & ~size_t(15u); + size_t upload_size = sizeof(arg); + if (kernel_args_size != 0) { + if (payload_offset > GTX_FIXED_ARG_REGION_SIZE || + kernel_args_size > GTX_FIXED_ARG_REGION_SIZE - payload_offset) { + GTX_LOG_ERROR("kernel argument payload exceeds fixed argument region: kernel_id=%u size=%zu\n", + (unsigned) desc.kernel_id, kernel_args_size); + return false; + } + arg.desc.args_addr = arg_addr + payload_offset; + arg.desc.args_size = (uint32_t) kernel_args_size; + upload_size = payload_offset + kernel_args_size; + } else if ((arg.desc.args_addr == 0u) != (arg.desc.args_size == 0u)) { + GTX_LOG_ERROR("remote kernel argument address/size mismatch: kernel_id=%u\n", + (unsigned) desc.kernel_id); + return false; + } + + std::vector arg_blob(upload_size, 0u); + memcpy(arg_blob.data(), &arg, sizeof(arg)); + if (kernel_args_size != 0) { + memcpy(arg_blob.data() + payload_offset, kernel_args, kernel_args_size); + } + + if (gtx_log_ops_enabled()) { + fprintf(stderr, "[GTX LAUNCH] arg upload begin %s id=%u arg=0x%llx\n", + bundle->name, + (unsigned) desc.kernel_id, + (unsigned long long) gtx_fixed_arg_addr()); + } + + if (!upload_bytes(arg_addr, arg_blob.data(), arg_blob.size())) { + if (gtx_log_ops_enabled()) { + fprintf(stderr, "[GTX LAUNCH] arg upload failed %s id=%u\n", + bundle->name, (unsigned) desc.kernel_id); + } + return false; + } + + ggml_gtx_rpc_launch_t req = {}; + req.kernel_id = desc.kernel_id; + req.arg_addr = arg_addr; + + if (gtx_log_ops_enabled()) { + fprintf(stderr, "[GTX LAUNCH] rpc begin %s id=%u\n", + bundle->name, (unsigned) desc.kernel_id); + } + + const bool ok = rpc(GGML_GTX_RPC_LAUNCH, &req, sizeof(req), nullptr, GTX_UART_KERNEL_TIMEOUT_MS); + if (ok) { + m_stash_may_be_resident = true; + } + if (gtx_log_ops_enabled()) { + fprintf(stderr, "[GTX LAUNCH] rpc %s %s id=%u\n", + ok ? "done" : "failed", bundle->name, (unsigned) desc.kernel_id); + } + return ok; +} +#else +class gtx_uart_vp_connection { +public: + bool open() { return false; } + bool is_open() const { return false; } + bool upload_bytes(uint64_t, const void *, size_t) { return false; } + bool download_bytes(uint64_t, void *, size_t) { return false; } + bool copy_tensor(const ggml_tensor *, const ggml_tensor *) { return false; } + bool stash_flush_reset() { return false; } + bool launch(const ggml_gtx_kernel_launch_desc_t &, const void * = nullptr, size_t = 0) { return false; } +}; +#endif + +static bool gtx_build_kernel_args(gtx_uart_vp_connection & connection, + const ggml_gtx_kernel_launch_desc_t & desc, + std::vector & args) { + args.clear(); + if ((desc.kernel_id == GGML_GTX_KERNEL_ADD_F16 || + desc.kernel_id == GGML_GTX_KERNEL_MUL_F16) && + desc.src_ne[1][0] == 1u) { + uint64_t scalar_count = 1u; + for (uint32_t d = 1u; d < GGML_MAX_DIMS; ++d) { + if (desc.src_ne[1][d] == 0u || + scalar_count > UINT32_MAX / desc.src_ne[1][d]) { + return false; + } + scalar_count *= desc.src_ne[1][d]; + } + + uint64_t expected_stride = sizeof(uint16_t); + for (uint32_t d = 0u; d < GGML_MAX_DIMS; ++d) { + if (desc.src_nb[1][d] != expected_stride) { + GTX_LOG_ERROR("ADD/MUL scalar payload requires compact RHS: dim=%u nb=%u expected=%llu\n", + d, desc.src_nb[1][d], + (unsigned long long) expected_stride); + return false; + } + expected_stride *= desc.src_ne[1][d]; + } + + const size_t scalar_bytes = (size_t) scalar_count * sizeof(uint16_t); + const size_t header_size = GGML_GTX_ADD_ARGS_HEADER_SIZE; + static_assert(GGML_GTX_ADD_ARGS_HEADER_SIZE == GGML_GTX_MUL_ARGS_HEADER_SIZE, + "ADD/MUL scalar payload headers must match"); + args.resize(header_size + scalar_bytes, 0u); + ggml_gtx_add_args_t params = {}; + params.version = desc.kernel_id == GGML_GTX_KERNEL_ADD_F16 ? + GGML_GTX_ADD_ARGS_VERSION : GGML_GTX_MUL_ARGS_VERSION; + params.size = (uint32_t) args.size(); + params.scalar_count = (uint32_t) scalar_count; + memcpy(args.data(), ¶ms, header_size); + if (!connection.download_bytes(desc.src_addr[1], + args.data() + header_size, + scalar_bytes)) { + GTX_LOG_ERROR("failed to download ADD/MUL scalar payload: addr=0x%llx size=%zu\n", + (unsigned long long) desc.src_addr[1], scalar_bytes); + args.clear(); + return false; + } + return true; + } + if (desc.kernel_id != GGML_GTX_KERNEL_MUL_MAT_F16) { + return true; + } + + ggml_gtx_mul_mat_args_t params = {}; + params.version = GGML_GTX_MUL_MAT_ARGS_VERSION; + params.size = sizeof(params); + params.m = desc.src_ne[0][1]; + params.k = desc.src_ne[0][0]; + params.n = desc.src_ne[1][1]; + args.resize(sizeof(params)); + memcpy(args.data(), ¶ms, sizeof(params)); + return true; +} + +static bool gtx_uart_launch_desc(gtx_uart_vp_connection & connection, + const ggml_gtx_kernel_launch_desc_t & desc) { + std::vector args; + if (!gtx_build_kernel_args(connection, desc, args)) { + return false; + } + return connection.launch(desc, + args.empty() ? nullptr : args.data(), + args.size()); +} + +static std::shared_ptr ggml_backend_gtx_shared_uart_connection(int device_id) { + static std::mutex s_mutex; + static std::vector> s_connections; + + std::lock_guard lock(s_mutex); + if ((int) s_connections.size() <= device_id) { + s_connections.resize(device_id + 1); + } + if (!s_connections[device_id]) { + s_connections[device_id] = std::make_shared(); + } + return s_connections[device_id]; +} + +static std::shared_ptr ggml_backend_gtx_shared_connection(int device_id) { + static std::mutex s_mutex; + static std::vector> s_connections; + + std::lock_guard lock(s_mutex); + if ((int) s_connections.size() <= device_id) { + s_connections.resize(device_id + 1); + } + if (!s_connections[device_id]) { + s_connections[device_id] = std::make_shared(); + } + return s_connections[device_id]; +} + +bool gtx_pcie_connection::open(const char* device_path, int bar_index) { + std::lock_guard lock(m_mutex); + (void)bar_index; + + if (m_irq_ctx != nullptr) { + return true; /* Already open */ + } + /* Try vfio-user socket paths in order */ + const std::string default_socket = gtx_default_vfio_socket_path(); + std::vector socket_paths = { + default_socket, + "./gtx-ggml-cuda-vfio.sock", + }; + if (device_path != nullptr && device_path[0] != '\0') { + socket_paths.emplace_back(device_path); + } + + for (const std::string & path_str : socket_paths) { + const char * path = path_str.c_str(); + +#ifdef __linux__ + struct stat st; + if (stat(path, &st) == 0 && S_ISSOCK(st.st_mode)) { + GTX_LOG("Trying vfio-user socket: %s\n", path); + m_irq_ctx = gtx_pcie_irq_create_vfio(path); + if (m_irq_ctx != nullptr) { + GTX_LOG("Connected to GTX device via vfio-user: %s\n", path); + return true; + } + } +#endif + } + + /* If no vfio-user socket found, try hardware VFIO */ + if (device_path && strncmp(device_path, "/dev/vfio/", 10) == 0) { + m_irq_ctx = gtx_pcie_irq_create_hw(device_path, 0); + if (m_irq_ctx != nullptr) { + GTX_LOG("Connected to GTX device via hardware VFIO: %s\n", device_path); + return true; + } + } + + GTX_LOG_ERROR("Failed to connect to GTX device\n"); + return false; +} + +void gtx_pcie_connection::close() { + std::lock_guard lock(m_mutex); + + if (m_irq_ctx != nullptr) { + gtx_pcie_irq_destroy(m_irq_ctx); + m_irq_ctx = nullptr; + GTX_LOG("GTX connection closed\n"); + } + +} + +void gtx_pcie_connection::send_doorbell(uint32_t source) { + if (m_irq_ctx == nullptr) return; + gtx_pcie_irq_send_doorbell(m_irq_ctx, source); +} + +bool gtx_pcie_connection::wait_msi(int vector, int timeout_ms) { + if (m_irq_ctx == nullptr) return false; + return gtx_pcie_irq_wait_msi(m_irq_ctx, vector, timeout_ms) >= 0; +} + +bool gtx_pcie_connection::send_data(uint64_t offset, const void* data, size_t size) { + if (m_irq_ctx == nullptr || data == nullptr) return false; + return gtx_pcie_irq_region_write(m_irq_ctx, GTX_BAR2_REGION, offset, data, size) == 0; +} + +bool gtx_pcie_connection::recv_data(uint64_t offset, void* data, size_t size) { + if (m_irq_ctx == nullptr || data == nullptr) return false; + return gtx_pcie_irq_region_read(m_irq_ctx, GTX_BAR2_REGION, offset, data, size) == 0; +} + +bool gtx_pcie_connection::send_cmd(uint8_t cmd, const void* request, size_t req_size, + void* response, size_t rsp_size) { + std::lock_guard lock(m_mutex); + + if (m_irq_ctx == nullptr) { + GTX_LOG_ERROR("send_cmd: not connected\n"); + return false; + } + + auto region_read = [&](uint64_t offset, void * data, size_t size) { + return gtx_pcie_irq_region_read(m_irq_ctx, GTX_BAR2_REGION, offset, data, size); + }; + + auto region_write = [&](uint64_t offset, const void * data, size_t size) { + return gtx_pcie_irq_region_write(m_irq_ctx, GTX_BAR2_REGION, offset, data, size); + }; + + /* Build 64-byte message */ + gtx_message msg = {}; + msg.seq_num = ++m_seq_num; + msg.type = cmd; + msg.size = (uint32_t)req_size; + + /* Copy request data to payload (up to 24 bytes) */ + if (request != nullptr && req_size > 0) { + size_t copy_size = req_size < sizeof(msg.payload) ? req_size : sizeof(msg.payload); + memcpy(msg.payload, request, copy_size); + } + + /* Message queue is in BAR2 (maps to device memory 0x60000000) */ + + /* Read current RX head from device */ + uint32_t rx_head = 0; + if (region_read(GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_RX_HEAD, + &rx_head, sizeof(rx_head)) < 0) { + GTX_LOG_ERROR("Failed to read RX head\n"); + return false; + } + + GTX_LOG("RX head before write: %u\n", rx_head); + + /* Write message to RX queue */ + uint64_t msg_offset = GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_RX_DATA + (rx_head * 64); + if (region_write(msg_offset, &msg, sizeof(msg)) < 0) { + GTX_LOG_ERROR("Failed to write message to RX queue\n"); + return false; + } + + /* Update RX head */ + uint32_t next_head = (rx_head + 1) % GTX_MSG_QUEUE_ENTRIES; + if (region_write(GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_RX_HEAD, + &next_head, sizeof(next_head)) < 0) { + GTX_LOG_ERROR("Failed to update RX head\n"); + return false; + } + + GTX_LOG("RX head after write: %u\n", next_head); + + GTX_LOG("send_cmd: cmd=%s(%u) seq=%u req_size=%zu\n", gtx_cmd_name(cmd), cmd, msg.seq_num, req_size); + + const bool log_alloc = (cmd == GTX_CMD_ALLOC_BUFFER); + auto log_alloc_rsp = [&](const char * tag, const gtx_message & rsp, uint32_t tx_head, uint32_t tx_tail) { + gtx_msg_alloc_buffer_rsp alloc_rsp = {}; + if (rsp.size >= sizeof(alloc_rsp)) { + memcpy(&alloc_rsp, rsp.payload, sizeof(alloc_rsp)); + } + + GTX_LOG("ALLOC_BUFFER %s: got_seq=%u expect_seq=%u tx_head=%u tx_tail=%u type=%u flags=0x%x size=%u rsp=%u remote_ptr=0x%llx remote_size=%llu\n", + tag, + rsp.seq_num, + msg.seq_num, + tx_head, + tx_tail, + rsp.type, + rsp.flags, + rsp.size, + (rsp.flags & GTX_FLAG_RESPONSE) != 0, + (unsigned long long) alloc_rsp.remote_ptr, + (unsigned long long) alloc_rsp.remote_size); + }; + + /* Send doorbell to wake device */ + send_doorbell(GTX_DOORBELL_CMD_READY); + + /* Wait for response in TX queue */ + auto start = std::chrono::steady_clock::now(); + const int timeout_ms = (cmd == GTX_CMD_GRAPH_COMPUTE) ? + GTX_GRAPH_COMPUTE_TIMEOUT_MS : GTX_CMD_TIMEOUT_MS; + + while (true) { + /* Read TX head and tail from BAR2 */ + uint32_t tx_head = 0, tx_tail = 0; + region_read(GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_TX_HEAD, + &tx_head, sizeof(tx_head)); + region_read(GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_TX_TAIL, + &tx_tail, sizeof(tx_tail)); + + if (tx_head != tx_tail) { + /* Read response message */ + gtx_message rsp = {}; + uint64_t rsp_offset = GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_TX_DATA + (tx_tail * 64); + region_read(rsp_offset, &rsp, sizeof(rsp)); + + if (log_alloc) { + log_alloc_rsp("queue rsp", rsp, tx_head, tx_tail); + } + + /* Update TX tail */ + uint32_t next_tail = (tx_tail + 1) % GTX_MSG_QUEUE_ENTRIES; + gtx_pcie_irq_region_write(m_irq_ctx, GTX_BAR2_REGION, + GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_TX_TAIL, + &next_tail, sizeof(next_tail)); + + /* Check sequence number */ + if (rsp.seq_num == msg.seq_num) { + /* Copy response data */ + if (response != nullptr && rsp_size > 0) { + size_t copy_size = rsp_size < sizeof(rsp.payload) ? rsp_size : sizeof(rsp.payload); + memcpy(response, rsp.payload, copy_size); + } + GTX_LOG("Response received: cmd=%s seq=%u\n", gtx_cmd_name(cmd), rsp.seq_num); + return true; + } else if (log_alloc) { + GTX_LOG_ERROR("ALLOC_BUFFER queue mismatch: got_seq=%u expect_seq=%u type=%u flags=0x%x size=%u rsp=%u tx_head=%u tx_tail=%u\n", + rsp.seq_num, + msg.seq_num, + rsp.type, + rsp.flags, + rsp.size, + (rsp.flags & GTX_FLAG_RESPONSE) != 0, + tx_head, + tx_tail); + } + } + + gtx_message rsp = {}; + uint64_t rsp_offset = GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_TX_DATA + (tx_tail * 64); + if (region_read(rsp_offset, &rsp, sizeof(rsp)) == 0 && + rsp.seq_num == msg.seq_num) { + if (log_alloc) { + log_alloc_rsp("fallback rsp", rsp, tx_head, tx_tail); + } else { + GTX_LOG("Response slot fallback hit: cmd=%s seq=%u\n", gtx_cmd_name(cmd), rsp.seq_num); + } + + uint32_t next_tail = (tx_tail + 1) % GTX_MSG_QUEUE_ENTRIES; + region_write(GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_TX_TAIL, + &next_tail, sizeof(next_tail)); + + if (response != nullptr && rsp_size > 0) { + size_t copy_size = rsp_size < sizeof(rsp.payload) ? rsp_size : sizeof(rsp.payload); + memcpy(response, rsp.payload, copy_size); + } + return true; + } + + /* Check timeout */ + auto now = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast(now - start).count(); + if (elapsed > timeout_ms) { + GTX_LOG_ERROR("Command timeout: cmd=%d, seq=%u\n", cmd, msg.seq_num); + return false; + } + + /* Small sleep to avoid busy-waiting */ +#ifdef _WIN32 + Sleep(1); +#else + usleep(1000); +#endif + } +} + +bool gtx_pcie_connection::send_cmd_staged(uint8_t cmd, uint64_t address, uint32_t size, + void* response, size_t rsp_size) { + std::lock_guard lock(m_mutex); + + if (m_irq_ctx == nullptr) { + GTX_LOG_ERROR("send_cmd_staged: not connected\n"); + return false; + } + + auto region_read = [&](uint64_t offset, void * data, size_t read_size) { + return gtx_pcie_irq_region_read(m_irq_ctx, GTX_BAR2_REGION, offset, data, read_size); + }; + + auto region_write = [&](uint64_t offset, const void * data, size_t write_size) { + return gtx_pcie_irq_region_write(m_irq_ctx, GTX_BAR2_REGION, offset, data, write_size); + }; + + gtx_message msg = {}; + msg.seq_num = ++m_seq_num; + msg.type = cmd; + msg.size = size; + msg.address = address; + + uint32_t rx_head = 0; + if (region_read(GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_RX_HEAD, + &rx_head, sizeof(rx_head)) < 0) { + GTX_LOG_ERROR("Failed to read RX head\n"); + return false; + } + + uint64_t msg_offset = GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_RX_DATA + (rx_head * 64); + if (region_write(msg_offset, &msg, sizeof(msg)) < 0) { + GTX_LOG_ERROR("Failed to write staged message to RX queue\n"); + return false; + } + + uint32_t next_head = (rx_head + 1) % GTX_MSG_QUEUE_ENTRIES; + if (region_write(GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_RX_HEAD, + &next_head, sizeof(next_head)) < 0) { + GTX_LOG_ERROR("Failed to update RX head\n"); + return false; + } + + send_doorbell(GTX_DOORBELL_CMD_READY); + + auto start = std::chrono::steady_clock::now(); + const int timeout_ms = 30000; + + while (true) { + uint32_t tx_head = 0, tx_tail = 0; + region_read(GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_TX_HEAD, + &tx_head, sizeof(tx_head)); + region_read(GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_TX_TAIL, + &tx_tail, sizeof(tx_tail)); + + if (tx_head != tx_tail) { + gtx_message rsp = {}; + uint64_t rsp_offset = GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_TX_DATA + (tx_tail * 64); + region_read(rsp_offset, &rsp, sizeof(rsp)); + + uint32_t next_tail = (tx_tail + 1) % GTX_MSG_QUEUE_ENTRIES; + region_write(GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_TX_TAIL, + &next_tail, sizeof(next_tail)); + + if (rsp.seq_num == msg.seq_num) { + if (response != nullptr && rsp_size > 0) { + size_t copy_size = rsp_size < sizeof(rsp.payload) ? rsp_size : sizeof(rsp.payload); + memcpy(response, rsp.payload, copy_size); + } + return true; + } + } + + gtx_message rsp = {}; + uint64_t rsp_offset = GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_TX_DATA + (tx_tail * 64); + if (region_read(rsp_offset, &rsp, sizeof(rsp)) == 0 && + rsp.seq_num == msg.seq_num) { + uint32_t next_tail = (tx_tail + 1) % GTX_MSG_QUEUE_ENTRIES; + region_write(GTX_MSG_QUEUE_BASE + GTX_MSG_QUEUE_TX_TAIL, + &next_tail, sizeof(next_tail)); + + if (response != nullptr && rsp_size > 0) { + size_t copy_size = rsp_size < sizeof(rsp.payload) ? rsp_size : sizeof(rsp.payload); + memcpy(response, rsp.payload, copy_size); + } + return true; + } + + auto now = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast(now - start).count(); + if (elapsed > timeout_ms) { + GTX_LOG_ERROR("Staged command timeout: cmd=%d, seq=%u\n", cmd, msg.seq_num); + return false; + } + +#ifdef _WIN32 + Sleep(1); +#else + usleep(1000); +#endif + } +} + +/*============================================================================= + * GTX Backend Context + *============================================================================*/ + +struct ggml_backend_gtx_context { + int device_id; + std::shared_ptr conn; + std::shared_ptr uart; + bool use_uart; + ggml_gtx_stats_t stats; + std::string last_error; + uint8_t firmware_version[3]; + + /* Buffer tracking */ + std::unordered_map allocated_buffers; /* ptr -> size */ + size_t uart_arena_next = 0; + + ggml_backend_gtx_context(int device) : device_id(device), use_uart(gtx_use_uart_vp_transport()) { + conn = ggml_backend_gtx_shared_connection(device); + if (use_uart) { + uart = ggml_backend_gtx_shared_uart_connection(device); + } + memset(&stats, 0, sizeof(stats)); + memset(firmware_version, 0, sizeof(firmware_version)); + } + + ~ggml_backend_gtx_context() = default; + + /* Get IRQ context from connection */ + gtx_pcie_irq_ctx* get_irq_ctx() const { + return conn ? conn->get_irq_ctx() : nullptr; + } +}; + +#if 0 +/* Materialize channel-wise ADD/MUL broadcasting as an ordinary dense remote + * tensor. The graph tensor does not know its consumer shape when set_tensor + * runs, so this is the consumer-side completion of the normal upload path. + * The temporary occupies the top of the fixed tensor arena and is reused + * after each synchronous descriptor launch. */ +static bool gtx_materialize_binary_rhs(ggml_backend_gtx_context * ctx, + ggml_gtx_kernel_launch_desc_t & desc) { + if (ctx == nullptr || !ctx->use_uart || ctx->uart == nullptr || + (desc.kernel_id != GGML_GTX_KERNEL_ADD_F16 && + desc.kernel_id != GGML_GTX_KERNEL_MUL_F16)) { + return true; + } + + bool same_shape = true; + for (uint32_t d = 0; d < GGML_MAX_DIMS; ++d) { + same_shape = same_shape && desc.src_ne[1][d] == desc.ne[d]; + } + if (same_shape) { + return true; + } + + /* This path intentionally covers scalar/channel-wise tensors such as + * [1,1,C,1]. Other broadcast layouts remain rejected by lowering. */ + if (desc.src_ne[1][0] != 1u || desc.src_ne[1][1] != 1u) { + GTX_LOG_ERROR("ADD/MUL remote materialization only supports channel-wise RHS\n"); + return false; + } + + uint64_t src_elems = 1u; + uint64_t dst_elems = 1u; + uint64_t dense_stride = sizeof(uint16_t); + for (uint32_t d = 0; d < GGML_MAX_DIMS; ++d) { + if (desc.ne[d] == 0u || desc.src_ne[1][d] == 0u || + (desc.src_ne[1][d] != 1u && desc.src_ne[1][d] != desc.ne[d]) || + desc.src_nb[1][d] != dense_stride || + src_elems > SIZE_MAX / desc.src_ne[1][d] || + dst_elems > SIZE_MAX / desc.ne[d]) { + GTX_LOG_ERROR("invalid ADD/MUL channel-wise RHS descriptor at dim=%u\n", d); + return false; + } + src_elems *= desc.src_ne[1][d]; + dst_elems *= desc.ne[d]; + dense_stride *= desc.src_ne[1][d]; + } + if (dst_elems > SIZE_MAX / sizeof(uint16_t)) { + return false; + } + + std::vector source((size_t) src_elems); + if (!ctx->uart->download_bytes(desc.src_addr[1], source.data(), + source.size() * sizeof(uint16_t))) { + GTX_LOG_ERROR("failed to download ADD/MUL channel-wise RHS\n"); + return false; + } + + std::vector expanded((size_t) dst_elems); + size_t out = 0; + for (uint32_t i3 = 0; i3 < desc.ne[3]; ++i3) { + const uint32_t j3 = desc.src_ne[1][3] == 1u ? 0u : i3; + for (uint32_t i2 = 0; i2 < desc.ne[2]; ++i2) { + const uint32_t j2 = desc.src_ne[1][2] == 1u ? 0u : i2; + const size_t src_index = + (size_t) j3 * desc.src_ne[1][2] * desc.src_ne[1][1] + + (size_t) j2 * desc.src_ne[1][1]; + const uint16_t value = source[src_index]; + for (uint32_t i1 = 0; i1 < desc.ne[1]; ++i1) { + for (uint32_t i0 = 0; i0 < desc.ne[0]; ++i0) { + expanded[out++] = value; + } + } + } + } + + const uint64_t bytes = dst_elems * sizeof(uint16_t); + const uint64_t aligned_bytes = (bytes + 63u) & ~63ull; + const uint64_t arena_base = gtx_fixed_tensor_arena_addr(); + const uint64_t arena_size = gtx_fixed_tensor_arena_size(); + if (aligned_bytes > arena_size || ctx->uart_arena_next > arena_size - aligned_bytes) { + GTX_LOG_ERROR("tensor arena has no room for broadcast RHS: bytes=%llu used=%zu arena=%llu\n", + (unsigned long long) bytes, ctx->uart_arena_next, + (unsigned long long) arena_size); + return false; + } + const uint64_t remote = arena_base + arena_size - aligned_bytes; + if (!ctx->uart->upload_bytes(remote, expanded.data(), (size_t) bytes)) { + GTX_LOG_ERROR("failed to upload materialized ADD/MUL RHS\n"); + return false; + } + + desc.src_addr[1] = remote; + uint64_t stride = sizeof(uint16_t); + for (uint32_t d = 0; d < GGML_MAX_DIMS; ++d) { + desc.src_ne[1][d] = desc.ne[d]; + desc.src_nb[1][d] = (uint32_t) stride; + stride *= desc.ne[d]; + } + if (gtx_log_ops_enabled()) { + fprintf(stderr, "[GTX BROADCAST] kernel=%u rhs=0x%llx bytes=%llu shape=[%u,%u,%u,%u]\n", + desc.kernel_id, (unsigned long long) remote, + (unsigned long long) bytes, desc.ne[0], desc.ne[1], + desc.ne[2], desc.ne[3]); + } + return true; +} + +static bool gtx_submit_uart_launch_desc(ggml_backend_gtx_context * ctx, + const ggml_gtx_kernel_launch_desc_t & input) { + ggml_gtx_kernel_launch_desc_t desc = input; + return gtx_materialize_binary_rhs(ctx, desc) && + gtx_uart_launch_desc(*ctx->uart, desc); +} +#endif + +struct ggml_backend_gtx_buffer_context { + int device_id; + uint64_t remote_ptr; + size_t size; + gtx_pcie_connection* conn; + ggml_backend_gtx_context* backend_ctx; + + /* Local shadow buffer for small tensors */ + std::vector shadow_buffer; +}; + +/*============================================================================= + * Buffer Type Implementation + *============================================================================*/ + +static const char* ggml_backend_gtx_buffer_type_get_name(ggml_backend_buffer_type_t buft) { + ggml_backend_gtx_context* ctx = (ggml_backend_gtx_context*)buft->context; + static char name[64]; + snprintf(name, sizeof(name), "GTX%d", ctx->device_id); + return name; +} + +static ggml_backend_buffer_t ggml_backend_gtx_buffer_type_alloc_buffer( + ggml_backend_buffer_type_t buft, size_t size); + +static void ggml_backend_gtx_buffer_free_buffer(ggml_backend_buffer_t buffer); + +static bool ggml_backend_gtx_open_device_connection(gtx_pcie_connection * conn, int device_id) { + if (conn == nullptr) { + return false; + } + if (conn->is_open()) { + return true; + } + + char device_path[64]; +#ifdef _WIN32 + snprintf(device_path, sizeof(device_path), "\\.\\gtx%d", device_id); +#else + snprintf(device_path, sizeof(device_path), "/dev/gtx%d", device_id); +#endif + + return conn->open(device_path); +} + +static size_t ggml_backend_gtx_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { + (void)buft; + return 64; /* 64-byte alignment for cache efficiency */ +} + +static size_t ggml_backend_gtx_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { + (void)buft; + return 16ULL * 1024 * 1024 * 1024; /* 16 GB max */ +} + +static size_t gtx_align_up(size_t value, size_t alignment) { + return (value + alignment - 1) & ~(alignment - 1); +} + +static bool ggml_backend_gtx_buffer_type_is_host(ggml_backend_buffer_type_t buft) { + (void)buft; + return false; /* Device memory, not host */ +} + +static bool ggml_backend_buffer_is_gtx(ggml_backend_buffer_t buffer) { + return buffer != nullptr && buffer->iface.free_buffer == ggml_backend_gtx_buffer_free_buffer; +} + +static uint64_t gtx_remote_data_ptr(const ggml_tensor * tensor) { + if (tensor == nullptr) { + return 0; + } + + if (tensor->view_src != nullptr) { + return gtx_remote_data_ptr(tensor->view_src) + + gtx_remote_byte_offset(tensor, tensor->view_offs); + } + + if (tensor->buffer == nullptr || !ggml_backend_buffer_is_gtx(tensor->buffer)) { + return 0; + } + + ggml_backend_gtx_buffer_context * ctx = + (ggml_backend_gtx_buffer_context *) tensor->buffer->context; + uint64_t base = ctx != nullptr ? ctx->remote_ptr : 0; + uintptr_t local_base = (uintptr_t) ggml_backend_buffer_get_base(tensor->buffer); + uintptr_t local_ptr = (uintptr_t) tensor->data; + return base + (local_ptr - local_base); +} + +static bool gtx_is_valid_unary_op(int32_t op) { + return op >= 0 && op < GGML_UNARY_OP_COUNT; +} + +static bool gtx_is_valid_glu_op(int32_t op) { + return op >= 0 && op < GGML_GLU_OP_COUNT; +} + +static constexpr uint32_t GGML_GTX_OP_UNSUPPORTED_HOST = UINT32_MAX; + +static uint32_t gtx_normalize_host_op(const ggml_tensor * tensor) { + const uint32_t raw_op = tensor != nullptr ? (uint32_t) tensor->op : (uint32_t) GGML_OP_NONE; + + if (tensor == nullptr) { + return raw_op; + } + + const char * host_op_name = ggml_op_name((ggml_op) raw_op); + + // 이 백엔드는 vision.cpp 안에서 호스트와 같은 ggml 헤더로 빌드된다. 예전에 있던 + // 「UNARY+1 이면 UNARY」·「COUNT 이면 GLU」 같은 번호 보정은 지웠다 — 같은 헤더에서는 + // UNARY+1 이 MAP_CUSTOM1 이라 그 노드를 UNARY 로 잘못 읽을 수 있었다. + + // Dynamic backends can be loaded by a newer ggml host whose op enum has + // drifted from this backend's bundled headers. Prefer the host's op name + // when it clearly identifies a parameterized op we can still execute. + if (raw_op != (uint32_t) GGML_OP_UNARY && + strcmp(host_op_name, "UNARY") == 0) { + const int32_t unary_op = tensor->op_params[0]; + if (tensor->src[1] == nullptr && gtx_is_valid_unary_op(unary_op)) { + return (uint32_t) GGML_OP_UNARY; + } + } + + if (raw_op != (uint32_t) GGML_OP_GLU && + strcmp(host_op_name, "GLU") == 0) { + const int32_t glu_op = tensor->op_params[0]; + if (tensor->src[1] != nullptr && gtx_is_valid_glu_op(glu_op)) { + return (uint32_t) GGML_OP_GLU; + } + } + + if (raw_op != (uint32_t) GGML_OP_UPSCALE && + strcmp(host_op_name, "UPSCALE") == 0) { + return (uint32_t) GGML_OP_UPSCALE; + } + + if (raw_op != (uint32_t) GGML_OP_PAD && + strcmp(host_op_name, "PAD") == 0) { + return (uint32_t) GGML_OP_PAD; + } + + if (raw_op != (uint32_t) GGML_OP_CONV_TRANSPOSE_2D && + strcmp(host_op_name, "CONV_TRANSPOSE_2D") == 0) { + return (uint32_t) GGML_OP_CONV_TRANSPOSE_2D; + } + + if (raw_op != (uint32_t) GGML_OP_LEAKY_RELU && + strcmp(host_op_name, "LEAKY_RELU") == 0) { + return (uint32_t) GGML_OP_LEAKY_RELU; + } + + if (raw_op != (uint32_t) GGML_OP_CLAMP && + strcmp(host_op_name, "CLAMP") == 0) { + return (uint32_t) GGML_OP_CLAMP; + } + + return raw_op; +} + +static const char * gtx_tensor_op_display_name(const ggml_tensor * tensor) { + if (tensor == nullptr) { + return "NULL"; + } + const uint32_t normalized_op = gtx_normalize_host_op(tensor); + if (normalized_op == GGML_GTX_OP_UNSUPPORTED_HOST) { + return ggml_op_name(tensor->op); + } + return gtx_normalized_op_name(normalized_op); +} + +static bool gtx_is_swiglu_node(const ggml_tensor * tensor) { + return tensor != nullptr && + gtx_normalize_host_op(tensor) == (uint32_t) GGML_OP_GLU && + tensor->op_params[0] == GGML_GLU_OP_SWIGLU; +} + +static bool gtx_is_gelu_node(const ggml_tensor * tensor) { + return tensor != nullptr && + gtx_normalize_host_op(tensor) == (uint32_t) GGML_OP_UNARY && + tensor->op_params[0] == GGML_UNARY_OP_GELU; +} + +static bool gtx_add_has_f32_operand(const ggml_tensor * tensor) { + return tensor != nullptr && + gtx_normalize_host_op(tensor) == (uint32_t) GGML_OP_ADD && + (tensor->type == GGML_TYPE_F32 || + (tensor->src[0] != nullptr && tensor->src[0]->type == GGML_TYPE_F32) || + (tensor->src[1] != nullptr && tensor->src[1]->type == GGML_TYPE_F32)); +} + +static bool gtx_is_add_f16_f32_vector_to_f16(const ggml_tensor * tensor) { + if (tensor == nullptr || + gtx_normalize_host_op(tensor) != (uint32_t) GGML_OP_ADD || + tensor->src[0] == nullptr || + tensor->src[1] == nullptr || + gtx_remote_tensor_type(tensor) != GGML_TYPE_F16 || + gtx_remote_tensor_type(tensor->src[0]) != GGML_TYPE_F16 || + gtx_remote_tensor_type(tensor->src[1]) != GGML_TYPE_F32 || + !gtx_preserve_remote_f32(tensor->src[1]) || + tensor->src[0]->ne[0] != tensor->ne[0] || + tensor->src[1]->ne[0] != tensor->ne[0]) { + return false; + } + + for (int i = 1; i < GGML_MAX_DIMS; ++i) { + if (tensor->src[0]->ne[i] != tensor->ne[i]) { + return false; + } + if (tensor->src[1]->ne[i] != 1) { + return false; + } + } + return true; +} + +static bool gtx_is_add_f16_bias_vector_basic(const ggml_tensor * tensor) { + if (tensor == nullptr || + gtx_normalize_host_op(tensor) != (uint32_t) GGML_OP_ADD || + tensor->type != GGML_TYPE_F32 || + tensor->src[0] == nullptr || + tensor->src[1] == nullptr || + tensor->src[0]->type != GGML_TYPE_F32 || + tensor->src[1]->type != GGML_TYPE_F32 || + strstr(tensor->src[1]->name, "bias") == nullptr || + tensor->src[0]->ne[0] != tensor->ne[0] || + tensor->src[1]->ne[0] != tensor->ne[0]) { + return false; + } + + for (int i = 1; i < GGML_MAX_DIMS; ++i) { + if (tensor->src[0]->ne[i] != tensor->ne[i]) { + return false; + } + if (tensor->src[1]->ne[i] != 1) { + return false; + } + } + + return gtx_remote_tensor_type(tensor) == GGML_TYPE_F16 && + gtx_remote_tensor_type(tensor->src[0]) == GGML_TYPE_F16 && + gtx_remote_tensor_type(tensor->src[1]) == GGML_TYPE_F16; +} + +static bool gtx_is_cont_same_shape(const ggml_tensor * tensor) { + if (tensor == nullptr || + gtx_normalize_host_op(tensor) != (uint32_t) GGML_OP_CONT || + tensor->src[0] == nullptr) { + return false; + } + + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + if (tensor->ne[i] != tensor->src[0]->ne[i]) { + return false; + } + } + return true; +} + +// ggml 연산 → 장치로 보내는 GTX 연산 번호. +// +// 와이어 번호는 ggml enum 에 묶지 않는다. depend/llama 는 upstream 을 따라가고, upstream 은 +// 새 연산을 enum 중간에 끼워 넣는다(vision-20260331 의 CONV_2D_DEFORM 이 그 뒤 39개를 1씩 밀었다). +// 그래서 번호는 2026-10-26 에 ISS 에서 검증한 판(GGML_OP_COUNT 97)으로 고정했고, 여기서 +// **이름으로** 짝짓는다. upstream 에 새 연산이 생기면 default 로 떨어진다. +static uint32_t gtx_wire_op_from_ggml_op(uint32_t op) { + switch ((ggml_op) op) { + case GGML_OP_NONE: return GGML_GTX_OP_NONE; + case GGML_OP_DUP: return GGML_GTX_OP_DUP; + case GGML_OP_ADD: return GGML_GTX_OP_ADD; + case GGML_OP_ADD_ID: return GGML_GTX_OP_ADD_ID; + case GGML_OP_ADD1: return GGML_GTX_OP_ADD1; + case GGML_OP_ACC: return GGML_GTX_OP_ACC; + case GGML_OP_SUB: return GGML_GTX_OP_SUB; + case GGML_OP_MUL: return GGML_GTX_OP_MUL; + case GGML_OP_DIV: return GGML_GTX_OP_DIV; + case GGML_OP_SQR: return GGML_GTX_OP_SQR; + case GGML_OP_SQRT: return GGML_GTX_OP_SQRT; + case GGML_OP_LOG: return GGML_GTX_OP_LOG; + case GGML_OP_SIN: return GGML_GTX_OP_SIN; + case GGML_OP_COS: return GGML_GTX_OP_COS; + case GGML_OP_SUM: return GGML_GTX_OP_SUM; + case GGML_OP_SUM_ROWS: return GGML_GTX_OP_SUM_ROWS; + case GGML_OP_CUMSUM: return GGML_GTX_OP_CUMSUM; + case GGML_OP_MEAN: return GGML_GTX_OP_MEAN; + case GGML_OP_ARGMAX: return 18; + case GGML_OP_COUNT_EQUAL: return 19; + case GGML_OP_REPEAT: return GGML_GTX_OP_REPEAT; + case GGML_OP_REPEAT_BACK: return 21; + case GGML_OP_CONCAT: return GGML_GTX_OP_CONCAT; + case GGML_OP_SILU_BACK: return 23; + case GGML_OP_NORM: return GGML_GTX_OP_NORM; + case GGML_OP_RMS_NORM: return GGML_GTX_OP_RMS_NORM; + case GGML_OP_RMS_NORM_BACK: return 26; + case GGML_OP_GROUP_NORM: return GGML_GTX_OP_GROUP_NORM; + case GGML_OP_L2_NORM: return GGML_GTX_OP_L2_NORM; + case GGML_OP_MUL_MAT: return GGML_GTX_OP_MUL_MAT; + case GGML_OP_MUL_MAT_ID: return GGML_GTX_OP_MUL_MAT_ID; + case GGML_OP_OUT_PROD: return GGML_GTX_OP_OUT_PROD; + case GGML_OP_SCALE: return GGML_GTX_OP_SCALE; + case GGML_OP_SET: return GGML_GTX_OP_SET; + case GGML_OP_CPY: return GGML_GTX_OP_CPY; + case GGML_OP_CONT: return GGML_GTX_OP_CONT; + case GGML_OP_RESHAPE: return GGML_GTX_OP_RESHAPE; + case GGML_OP_VIEW: return GGML_GTX_OP_VIEW; + case GGML_OP_PERMUTE: return GGML_GTX_OP_PERMUTE; + case GGML_OP_TRANSPOSE: return GGML_GTX_OP_TRANSPOSE; + case GGML_OP_GET_ROWS: return GGML_GTX_OP_GET_ROWS; + case GGML_OP_GET_ROWS_BACK: return 41; + case GGML_OP_SET_ROWS: return GGML_GTX_OP_SET_ROWS; + case GGML_OP_DIAG: return GGML_GTX_OP_DIAG; + case GGML_OP_DIAG_MASK_INF: return GGML_GTX_OP_DIAG_MASK_INF; + case GGML_OP_DIAG_MASK_ZERO: return GGML_GTX_OP_DIAG_MASK_ZERO; + case GGML_OP_SOFT_MAX: return GGML_GTX_OP_SOFTMAX; + case GGML_OP_SOFT_MAX_BACK: return 47; + case GGML_OP_ROPE: return GGML_GTX_OP_ROPE; + case GGML_OP_ROPE_BACK: return 49; + case GGML_OP_CLAMP: return GGML_GTX_OP_CLAMP; + case GGML_OP_CONV_TRANSPOSE_1D: return GGML_GTX_OP_CONV_TRANSPOSE_1D; + case GGML_OP_IM2COL: return GGML_GTX_OP_IM2COL; + case GGML_OP_IM2COL_BACK: return 53; + case GGML_OP_IM2COL_3D: return GGML_GTX_OP_IM2COL_3D; + case GGML_OP_CONV_2D: return GGML_GTX_OP_CONV_2D; + case GGML_OP_CONV_3D: return GGML_GTX_OP_CONV_3D; + case GGML_OP_CONV_2D_DW: return GGML_GTX_OP_CONV_2D_DW; + case GGML_OP_CONV_TRANSPOSE_2D: return GGML_GTX_OP_CONV_TRANSPOSE_2D; + case GGML_OP_POOL_1D: return GGML_GTX_OP_POOL_1D; + case GGML_OP_POOL_2D: return GGML_GTX_OP_POOL_2D; + case GGML_OP_POOL_2D_BACK: return 61; + case GGML_OP_UPSCALE: return GGML_GTX_OP_UPSCALE; + case GGML_OP_PAD: return GGML_GTX_OP_PAD; + case GGML_OP_PAD_REFLECT_1D: return GGML_GTX_OP_PAD_REFLECT_1D; + case GGML_OP_ROLL: return GGML_GTX_OP_ROLL; + case GGML_OP_ARANGE: return GGML_GTX_OP_ARANGE; + case GGML_OP_TIMESTEP_EMBEDDING: return GGML_GTX_OP_TIMESTEP_EMBEDDING; + case GGML_OP_ARGSORT: return GGML_GTX_OP_ARGSORT; + case GGML_OP_TOP_K: return GGML_GTX_OP_TOP_K; + case GGML_OP_LEAKY_RELU: return GGML_GTX_OP_LEAKY_RELU; + case GGML_OP_TRI: return GGML_GTX_OP_TRI; + case GGML_OP_FILL: return GGML_GTX_OP_FILL; + case GGML_OP_FLASH_ATTN_EXT: return GGML_GTX_OP_FLASH_ATTN_EXT; + case GGML_OP_FLASH_ATTN_BACK: return 74; + case GGML_OP_SSM_CONV: return GGML_GTX_OP_SSM_CONV; + case GGML_OP_SSM_SCAN: return GGML_GTX_OP_SSM_SCAN; + case GGML_OP_WIN_PART: return GGML_GTX_OP_WIN_PART; + case GGML_OP_WIN_UNPART: return GGML_GTX_OP_WIN_UNPART; + case GGML_OP_GET_REL_POS: return GGML_GTX_OP_GET_REL_POS; + case GGML_OP_ADD_REL_POS: return GGML_GTX_OP_ADD_REL_POS; + case GGML_OP_RWKV_WKV6: return 81; + case GGML_OP_GATED_LINEAR_ATTN: return GGML_GTX_OP_GATED_LINEAR_ATTN; + case GGML_OP_RWKV_WKV7: return 83; + case GGML_OP_SOLVE_TRI: return GGML_GTX_OP_SOLVE_TRI; + case GGML_OP_GATED_DELTA_NET: return 85; + case GGML_OP_UNARY: return 86; + case GGML_OP_MAP_CUSTOM1: return 87; + case GGML_OP_MAP_CUSTOM2: return 88; + case GGML_OP_MAP_CUSTOM3: return 89; + case GGML_OP_CUSTOM: return 90; + case GGML_OP_CROSS_ENTROPY_LOSS: return 91; + case GGML_OP_CROSS_ENTROPY_LOSS_BACK: return 92; + case GGML_OP_OPT_STEP_ADAMW: return 93; + case GGML_OP_OPT_STEP_SGD: return 94; + case GGML_OP_GLU: return 95; + case GGML_OP_CONV_2D_DEFORM: return 96; + default: return GGML_GTX_OP_UNSUPPORTED_HOST; + } +} + +static uint32_t gtx_wire_op_from_tensor(const ggml_tensor * tensor) { + const uint32_t op = gtx_normalize_host_op(tensor); + + if (tensor == nullptr) { + return gtx_wire_op_from_ggml_op(op); + } + + if (op == (uint32_t) GGML_OP_UNARY) { + switch ((ggml_unary_op) tensor->op_params[0]) { + case GGML_UNARY_OP_ABS: return GGML_GTX_OP_ABS; + case GGML_UNARY_OP_SGN: return GGML_GTX_OP_SGN; + case GGML_UNARY_OP_NEG: return GGML_GTX_OP_NEG; + case GGML_UNARY_OP_STEP: return GGML_GTX_OP_STEP; + case GGML_UNARY_OP_TANH: return GGML_GTX_OP_TANH; + case GGML_UNARY_OP_ELU: return GGML_GTX_OP_ELU; + case GGML_UNARY_OP_RELU: return GGML_GTX_OP_RELU; + case GGML_UNARY_OP_SIGMOID: return GGML_GTX_OP_SIGMOID; + case GGML_UNARY_OP_GELU: return GGML_GTX_OP_GELU; + case GGML_UNARY_OP_GELU_QUICK: return GGML_GTX_OP_GELU_QUICK; + case GGML_UNARY_OP_SILU: return GGML_GTX_OP_SILU; + case GGML_UNARY_OP_HARDSWISH: return GGML_GTX_OP_HARDSWISH; + case GGML_UNARY_OP_HARDSIGMOID: return GGML_GTX_OP_HARDSIGMOID; + case GGML_UNARY_OP_EXP: return GGML_GTX_OP_EXP; + case GGML_UNARY_OP_EXPM1: return GGML_GTX_OP_EXPM1; + case GGML_UNARY_OP_SOFTPLUS: return GGML_GTX_OP_SOFTPLUS; + case GGML_UNARY_OP_GELU_ERF: return GGML_GTX_OP_GELU_ERF; + case GGML_UNARY_OP_XIELU: return GGML_GTX_OP_XIELU; + case GGML_UNARY_OP_FLOOR: return GGML_GTX_OP_FLOOR; + case GGML_UNARY_OP_CEIL: return GGML_GTX_OP_CEIL; + case GGML_UNARY_OP_ROUND: return GGML_GTX_OP_ROUND; + case GGML_UNARY_OP_TRUNC: return GGML_GTX_OP_TRUNC; + default: return gtx_wire_op_from_ggml_op(op); + } + } + + if (op == (uint32_t) GGML_OP_GLU) { + return GGML_GTX_OP_GLU; + } + + return gtx_wire_op_from_ggml_op(op); +} + +static gtx_rpc_tensor gtx_serialize_tensor(const ggml_tensor * tensor) { + gtx_rpc_tensor result = {}; + + if (tensor == nullptr) { + return result; + } + + result.id = reinterpret_cast(tensor); + result.type = gtx_remote_type(tensor->type); + + if (tensor->buffer && ggml_backend_buffer_is_gtx(tensor->buffer)) { + ggml_backend_gtx_buffer_context * ctx = + (ggml_backend_gtx_buffer_context *) tensor->buffer->context; + result.buffer = ctx != nullptr ? ctx->remote_ptr : 0; + result.data = gtx_remote_data_ptr(tensor); + } + + for (uint32_t i = 0; i < GGML_MAX_DIMS; i++) { + result.ne[i] = tensor->ne[i]; + result.nb[i] = (uint32_t) gtx_remote_byte_offset(tensor, tensor->nb[i]); + } + + if (tensor->type == GGML_TYPE_Q8_0 && tensor->view_src == nullptr) { + result.nb[0] = sizeof(uint16_t); + for (uint32_t i = 1; i < GGML_MAX_DIMS; ++i) { + result.nb[i] = result.nb[i - 1] * result.ne[i - 1]; + } + } + + result.op = gtx_wire_op_from_tensor(tensor); + for (uint32_t i = 0; i < GGML_MAX_OP_PARAMS / sizeof(int32_t); i++) { + result.op_params[i] = tensor->op_params[i]; + } + result.flags = tensor->flags; + + for (uint32_t i = 0; i < GGML_MAX_SRC; i++) { + result.src[i] = reinterpret_cast(tensor->src[i]); + } + result.view_src = reinterpret_cast(tensor->view_src); + result.view_offs = gtx_remote_byte_offset(tensor, tensor->view_offs); + + memset(result.name, 0, sizeof(result.name)); + snprintf(result.name, 64, "%s", tensor->name); // 위 name[64] 와 같은 이유 + return result; +} + +static bool gtx_fill_tensor_desc(const ggml_tensor * tensor, + uint64_t * addr, + uint32_t * type, + uint32_t ne[GGML_MAX_DIMS], + uint32_t nb[GGML_MAX_DIMS]) { + if (tensor == nullptr) { + *addr = 0; + *type = 0; + memset(ne, 0, GGML_MAX_DIMS * sizeof(uint32_t)); + memset(nb, 0, GGML_MAX_DIMS * sizeof(uint32_t)); + return true; + } + + *addr = gtx_remote_data_ptr(tensor); + *type = (uint32_t) gtx_remote_tensor_type(tensor); + if (*addr == 0) { + return false; + } + + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + if (tensor->ne[i] < 0 || tensor->ne[i] > UINT32_MAX) { + return false; + } + const uint64_t rnb = gtx_remote_byte_offset(tensor, tensor->nb[i]); + if (rnb > UINT32_MAX) { + return false; + } + ne[i] = (uint32_t) tensor->ne[i]; + nb[i] = (uint32_t) rnb; + } + + return true; +} + +static uint32_t gtx_kernel_id_from_node(const ggml_tensor * node) { + if (node == nullptr) { + return GGML_GTX_KERNEL_INVALID; + } + + switch (gtx_normalize_host_op(node)) { + case GGML_OP_ADD: + return gtx_is_add_f16_f32_vector_to_f16(node) ? + GGML_GTX_KERNEL_ADD_F16_F32_TO_F16 : GGML_GTX_KERNEL_ADD_F16; + case GGML_OP_MUL: return GGML_GTX_KERNEL_MUL_F16; + case GGML_OP_SCALE: return GGML_GTX_KERNEL_SCALE_F16; + case GGML_OP_NORM: return GGML_GTX_KERNEL_NORM_F16; + case GGML_OP_RMS_NORM: return GGML_GTX_KERNEL_RMS_NORM_F16; + case GGML_OP_SOFT_MAX: return GGML_GTX_KERNEL_SOFT_MAX_F16; + case GGML_OP_MUL_MAT: return GGML_GTX_KERNEL_MUL_MAT_F16; + case GGML_OP_ROPE: return GGML_GTX_KERNEL_ROPE_F16; + case GGML_OP_GET_ROWS: return GGML_GTX_KERNEL_GET_ROWS_F16; + case GGML_OP_SET_ROWS: return GGML_GTX_KERNEL_SET_ROWS_F16; + case GGML_OP_IM2COL: return GGML_GTX_KERNEL_IM2COL_F16; + case GGML_OP_REPEAT: return GGML_GTX_KERNEL_REPEAT_F16; + case GGML_OP_CONCAT: return GGML_GTX_KERNEL_CONCAT_F16; + case GGML_OP_CONV_2D: return GGML_GTX_KERNEL_CONV_2D_F16; + case GGML_OP_CONV_2D_DW: return GGML_GTX_KERNEL_CONV_2D_DW_F16; + case GGML_OP_CONV_TRANSPOSE_2D: return GGML_GTX_KERNEL_CONV_TRANSPOSE_2D_F16; + case GGML_OP_UPSCALE: return GGML_GTX_KERNEL_UPSCALE_F16; + case GGML_OP_PAD: return GGML_GTX_KERNEL_PAD_F16; + case GGML_OP_SIN: return GGML_GTX_KERNEL_SIN_F16; + case GGML_OP_COS: return GGML_GTX_KERNEL_COS_F16; + case GGML_OP_CONT: return GGML_GTX_KERNEL_CONT_F16; + case GGML_OP_CPY: return GGML_GTX_KERNEL_CPY_F16; + case GGML_GTX_OP_GELU: return GGML_GTX_KERNEL_GELU_F16; + case GGML_GTX_OP_ABS: return GGML_GTX_KERNEL_ABS_F16; + case GGML_GTX_OP_NEG: return GGML_GTX_KERNEL_NEG_F16; + case GGML_GTX_OP_SGN: return GGML_GTX_KERNEL_SGN_F16; + case GGML_GTX_OP_CEIL: return GGML_GTX_KERNEL_CEIL_F16; + case GGML_GTX_OP_FLOOR: return GGML_GTX_KERNEL_FLOOR_F16; + case GGML_GTX_OP_TANH: return GGML_GTX_KERNEL_TANH_F16; + case GGML_GTX_OP_SILU: return GGML_GTX_KERNEL_SILU_F16; + case GGML_GTX_OP_RELU: return GGML_GTX_KERNEL_RELU_F16; + case GGML_OP_LEAKY_RELU: return GGML_GTX_KERNEL_LEAKY_RELU_F16; + case GGML_OP_CLAMP: return GGML_GTX_KERNEL_CLAMP_F16; + case GGML_OP_GLU: + return gtx_is_swiglu_node(node) ? GGML_GTX_KERNEL_SWIGLU_F16 : GGML_GTX_KERNEL_INVALID; + case GGML_OP_UNARY: + switch ((ggml_unary_op) node->op_params[0]) { + case GGML_UNARY_OP_GELU: return GGML_GTX_KERNEL_GELU_F16; + case GGML_UNARY_OP_ABS: return GGML_GTX_KERNEL_ABS_F16; + case GGML_UNARY_OP_NEG: return GGML_GTX_KERNEL_NEG_F16; + case GGML_UNARY_OP_SGN: return GGML_GTX_KERNEL_SGN_F16; + case GGML_UNARY_OP_CEIL: return GGML_GTX_KERNEL_CEIL_F16; + case GGML_UNARY_OP_FLOOR: return GGML_GTX_KERNEL_FLOOR_F16; + case GGML_UNARY_OP_TANH: return GGML_GTX_KERNEL_TANH_F16; + case GGML_UNARY_OP_SILU: return GGML_GTX_KERNEL_SILU_F16; + case GGML_UNARY_OP_RELU: return GGML_GTX_KERNEL_RELU_F16; + default: return GGML_GTX_KERNEL_INVALID; + } + default: + return GGML_GTX_KERNEL_INVALID; + } +} + +/* Kernel-owned resource metadata. Current L2 micro-kernels advertise zero; + * the function is the single host extension point for a future size formula. */ +static uint32_t gtx_kernel_l2_scratch_size(const ggml_tensor * node) { + if (node != nullptr && gtx_kernel_id_from_node(node) == GGML_GTX_KERNEL_ROPE_F16 && + node->ne[2] > 0 && node->ne[2] <= UINT32_MAX) { + return ggml_gtx_rope_scratch_size((uint32_t)node->ne[2], + (uint32_t)node->op_params[1]); + } + if (node != nullptr && + (gtx_kernel_id_from_node(node) == GGML_GTX_KERNEL_SIN_F16 || + gtx_kernel_id_from_node(node) == GGML_GTX_KERNEL_COS_F16)) { + return 0x40200u; + } + if (node != nullptr && + (gtx_kernel_id_from_node(node) == GGML_GTX_KERNEL_CONV_2D_F16 || + gtx_kernel_id_from_node(node) == GGML_GTX_KERNEL_CONV_2D_DW_F16)) { + return 8u * 1024u * 1024u; + } + return 0u; +} + +static uint64_t gtx_stash_data_budget(const ggml_tensor * node) { + const uint64_t required = gtx_kernel_l2_scratch_size(node); + const uint64_t scratch = std::max(required, GTX_STASH_MIN_SCRATCH_BYTES); + return scratch < GTX_STASH_L2_BYTES ? GTX_STASH_L2_BYTES - scratch : 0u; +} + +static uint64_t gtx_stash_data_budget(const std::vector & nodes) { + uint64_t required = 0u; + for (const ggml_tensor * node : nodes) { + required = std::max(required, gtx_kernel_l2_scratch_size(node)); + } + const uint64_t scratch = std::max(required, GTX_STASH_MIN_SCRATCH_BYTES); + return scratch < GTX_STASH_L2_BYTES ? GTX_STASH_L2_BYTES - scratch : 0u; +} + +enum class gtx_l2_chain_kind : uint32_t { + unsupported = 0, + native = 1, + producer = 2, + boundary = 3, +}; + +struct gtx_l2_chainability { + gtx_l2_chain_kind kind; + uint32_t kernel_id; + const char * reason; +}; + +static const char * gtx_l2_chain_kind_name(gtx_l2_chain_kind kind) { + switch (kind) { + case gtx_l2_chain_kind::native: return "native"; + case gtx_l2_chain_kind::producer: return "producer"; + case gtx_l2_chain_kind::boundary: return "boundary"; + case gtx_l2_chain_kind::unsupported: return "unsupported"; + } + return "unsupported"; +} + +static bool gtx_l2_chain_participates(gtx_l2_chain_kind kind) { + return kind == gtx_l2_chain_kind::native || + kind == gtx_l2_chain_kind::producer; +} + +static bool gtx_tensor_remote_f16_dense(const ggml_tensor * tensor) { + return tensor != nullptr && + gtx_remote_tensor_type(tensor) == GGML_TYPE_F16 && + ggml_is_contiguous(tensor); +} + +static bool gtx_tensor_remote_f16_vector_contiguous(const ggml_tensor * tensor) { + return tensor != nullptr && + gtx_remote_tensor_type(tensor) == GGML_TYPE_F16 && + tensor->ne[0] > 0 && + tensor->ne[0] <= UINT16_MAX && + gtx_remote_byte_offset(tensor, tensor->nb[0]) == sizeof(uint16_t); +} + +// A matrix's logical row count can be larger than a single kernel tile. This +// deliberately checks only the element stride; callers which expose a whole +// row to a kernel must still enforce their own `ne[0]` bound. +static bool gtx_tensor_remote_f16_element_contiguous(const ggml_tensor * tensor) { + return tensor != nullptr && + gtx_remote_tensor_type(tensor) == GGML_TYPE_F16 && + tensor->ne[0] > 0 && + gtx_remote_byte_offset(tensor, tensor->nb[0]) == sizeof(uint16_t); +} + +static bool gtx_tensor_remote_f16_c_fast_whc(const ggml_tensor * tensor) { + if (tensor == nullptr || gtx_remote_tensor_type(tensor) != GGML_TYPE_F16 || + tensor->ne[0] <= 0 || tensor->ne[1] <= 0 || tensor->ne[2] <= 0 || tensor->ne[3] != 1) { + return false; + } + const uint64_t c_bytes = (uint64_t)tensor->ne[2] * sizeof(uint16_t); + return gtx_remote_byte_offset(tensor, tensor->nb[2]) == sizeof(uint16_t) && + gtx_remote_byte_offset(tensor, tensor->nb[0]) == c_bytes && + gtx_remote_byte_offset(tensor, tensor->nb[1]) == c_bytes * (uint64_t)tensor->ne[0]; +} + +static bool gtx_tensor_remote_f16_c_fast_dw_weight(const ggml_tensor * tensor) { + if (tensor == nullptr || gtx_remote_tensor_type(tensor) != GGML_TYPE_F16 || + tensor->ne[0] <= 0 || tensor->ne[1] <= 0 || tensor->ne[2] != 1 || tensor->ne[3] <= 0) { + return false; + } + const uint64_t c_bytes = (uint64_t)tensor->ne[3] * sizeof(uint16_t); + return gtx_remote_byte_offset(tensor, tensor->nb[3]) == sizeof(uint16_t) && + gtx_remote_byte_offset(tensor, tensor->nb[0]) == c_bytes && + gtx_remote_byte_offset(tensor, tensor->nb[1]) == c_bytes * (uint64_t)tensor->ne[0]; +} + +static bool gtx_l2_conv_layout_ok(const ggml_tensor * node) { + if (node == nullptr || node->src[0] == nullptr || node->src[1] == nullptr) { + return false; + } + if (gtx_tensor_remote_f16_element_contiguous(node) && + gtx_tensor_remote_f16_element_contiguous(node->src[0]) && + gtx_tensor_remote_f16_element_contiguous(node->src[1])) { + return true; + } + return gtx_kernel_id_from_node(node) == GGML_GTX_KERNEL_CONV_2D_DW_F16 && + gtx_tensor_remote_f16_c_fast_whc(node) && + gtx_tensor_remote_f16_c_fast_dw_weight(node->src[0]) && + gtx_tensor_remote_f16_c_fast_whc(node->src[1]); +} + +static bool gtx_tensor_same_shape(const ggml_tensor * a, const ggml_tensor * b) { + if (a == nullptr || b == nullptr) { + return false; + } + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + if (a->ne[i] != b->ne[i]) { + return false; + } + } + return true; +} + +static bool gtx_l2_dense_unary_shape_ok(const ggml_tensor * node) { + return node != nullptr && + node->src[0] != nullptr && + gtx_tensor_remote_f16_dense(node) && + gtx_tensor_remote_f16_dense(node->src[0]) && + gtx_tensor_same_shape(node, node->src[0]); +} + +static bool gtx_l2_dense_binary_shape_ok(const ggml_tensor * node) { + return node != nullptr && + node->src[0] != nullptr && + node->src[1] != nullptr && + gtx_tensor_remote_f16_dense(node) && + gtx_tensor_remote_f16_dense(node->src[0]) && + gtx_tensor_remote_f16_dense(node->src[1]) && + gtx_tensor_same_shape(node, node->src[0]) && + gtx_tensor_same_shape(node, node->src[1]); +} + +static bool gtx_l2_rhs_broadcast_shape_ok(const ggml_tensor * node, bool allow_scalar_dim0) { + if (node == nullptr || + node->src[0] == nullptr || + node->src[1] == nullptr || + !gtx_tensor_remote_f16_dense(node) || + !gtx_tensor_remote_f16_dense(node->src[0]) || + !gtx_tensor_remote_f16_dense(node->src[1]) || + !gtx_tensor_same_shape(node, node->src[0])) { + return false; + } + + const ggml_tensor * rhs = node->src[1]; + if (rhs->ne[0] != node->ne[0] && + (!allow_scalar_dim0 || rhs->ne[0] != 1)) { + return false; + } + for (int i = 1; i < GGML_MAX_DIMS; ++i) { + if (rhs->ne[i] != 1 && rhs->ne[i] != node->ne[i]) { + return false; + } + } + return true; +} + +static bool gtx_l2_rhs_col_vector_broadcast_shape_ok(const ggml_tensor * node, + bool allow_scalar_dim0) { + if (!gtx_l2_rhs_broadcast_shape_ok(node, allow_scalar_dim0) || + node == nullptr || + node->src[1] == nullptr) { + return false; + } + + const ggml_tensor * rhs = node->src[1]; + if (rhs->ne[0] != node->ne[0] && + (!allow_scalar_dim0 || rhs->ne[0] != 1)) { + return false; + } + return rhs->ne[1] == 1 && + rhs->ne[2] == 1 && + rhs->ne[3] == 1; +} + +static bool gtx_l2_scale_shape_ok(const ggml_tensor * node) { + return node != nullptr && + node->src[0] != nullptr && + gtx_tensor_remote_f16_dense(node) && + gtx_tensor_remote_f16_dense(node->src[0]) && + gtx_tensor_same_shape(node, node->src[0]); +} + +static bool gtx_l2_add_shape_ok(const ggml_tensor * node) { + return gtx_l2_dense_binary_shape_ok(node) || + gtx_l2_rhs_broadcast_shape_ok(node, true) || + gtx_is_add_f16_bias_vector_basic(node); +} + +static bool gtx_l2_repeat_shape_ok(const ggml_tensor * node) { + if (node == nullptr || node->src[0] == nullptr || + !gtx_tensor_remote_f16_dense(node) || + !gtx_tensor_remote_f16_dense(node->src[0])) { + return false; + } + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + if (node->src[0]->ne[i] <= 0 || node->ne[i] < node->src[0]->ne[i] || + (node->ne[i] % node->src[0]->ne[i]) != 0 || + node->ne[i] > UINT16_MAX || node->src[0]->ne[i] > UINT16_MAX) { + return false; + } + } + return (uint64_t)node->ne[0] * sizeof(ggml_fp16_t) <= UINT16_MAX && + (uint64_t)node->src[0]->ne[0] * sizeof(ggml_fp16_t) <= UINT16_MAX; +} + +static bool gtx_l2_concat_shape_ok(const ggml_tensor * node) { + if (node == nullptr || node->src[0] == nullptr || node->src[1] == nullptr || + !gtx_tensor_remote_f16_dense(node) || + !gtx_tensor_remote_f16_dense(node->src[0]) || + !gtx_tensor_remote_f16_dense(node->src[1])) { + return false; + } + const int32_t dim = ((const int32_t *)node->op_params)[0]; + if (dim < 0 || dim >= GGML_MAX_DIMS) { + return false; + } + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + const int64_t expected = i == dim ? + node->src[0]->ne[i] + node->src[1]->ne[i] : node->src[0]->ne[i]; + if (node->ne[i] != expected || + (i != dim && node->src[0]->ne[i] != node->src[1]->ne[i]) || + node->ne[i] > UINT16_MAX || node->src[0]->ne[i] > UINT16_MAX || + node->src[1]->ne[i] > UINT16_MAX) { + return false; + } + } + const auto copy_shape_ok = [](const ggml_tensor * tensor) { + const uint64_t row_bytes = (uint64_t)tensor->ne[0] * sizeof(ggml_fp16_t); + const uint64_t rows = (uint64_t)tensor->ne[1] * tensor->ne[2] * tensor->ne[3]; + return row_bytes > 0 && row_bytes <= UINT16_MAX && rows > 0 && rows <= UINT16_MAX; + }; + return copy_shape_ok(node) && copy_shape_ok(node->src[0]) && copy_shape_ok(node->src[1]); +} + +static bool gtx_l2_pad_shape_ok(const ggml_tensor * node) { + if (node == nullptr || node->src[0] == nullptr || + !gtx_tensor_remote_f16_dense(node) || + !gtx_tensor_remote_f16_dense(node->src[0])) { + return false; + } + const int32_t * p = (const int32_t *)node->op_params; + if (p[0] != 0 || p[2] != 0 || p[4] != 0 || p[6] != 0 || + p[1] < 0 || p[3] < 0 || p[5] < 0 || p[7] != 0 || p[8] != 0) { + return false; + } + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + const int64_t expected = node->src[0]->ne[i] + (i < 3 ? p[i * 2 + 1] : 0); + if (node->ne[i] != expected || node->ne[i] > UINT16_MAX || + node->src[0]->ne[i] > UINT16_MAX) { + return false; + } + } + const uint64_t src_row_bytes = (uint64_t)node->src[0]->ne[0] * sizeof(ggml_fp16_t); + const uint64_t dst_row_bytes = (uint64_t)node->ne[0] * sizeof(ggml_fp16_t); + const uint64_t dst_rows = (uint64_t)node->ne[1] * node->ne[2] * node->ne[3]; + return src_row_bytes > 0 && src_row_bytes <= UINT16_MAX && + dst_row_bytes > 0 && dst_row_bytes <= UINT16_MAX && + dst_rows > 0 && dst_rows <= UINT16_MAX; +} + +static bool gtx_l2_conv_transpose_shape_ok(const ggml_tensor * node) { + if (node == nullptr || node->src[0] == nullptr || node->src[1] == nullptr || + !gtx_tensor_remote_f16_dense(node) || + !gtx_tensor_remote_f16_dense(node->src[0]) || + !gtx_tensor_remote_f16_dense(node->src[1])) { + return false; + } + const uint64_t slot_budget = (gtx_stash_data_budget(node) / GTX_STASH_SLOT_COUNT) & ~31ull; + const uint64_t dst_bytes = gtx_remote_size_bytes(node); + const uint64_t weight_bytes = gtx_remote_size_bytes(node->src[0]); + const uint64_t input_bytes = gtx_remote_size_bytes(node->src[1]); + return dst_bytes > 0 && dst_bytes <= slot_budget && + weight_bytes > 0 && weight_bytes <= slot_budget && + input_bytes > 0 && input_bytes <= slot_budget && + dst_bytes <= UINT64_MAX - weight_bytes && + dst_bytes + weight_bytes <= UINT64_MAX - input_bytes && + dst_bytes + weight_bytes + input_bytes <= gtx_stash_data_budget(node); +} + +static bool gtx_l2_conv_whole_shape_ok(const ggml_tensor * node) { + if (node == nullptr || node->src[0] == nullptr || node->src[1] == nullptr || + !gtx_l2_conv_layout_ok(node) || + node->ne[3] != 1 || node->src[1]->ne[3] != 1) { + return false; + } + return node->ne[0] > 0 && node->ne[1] > 0 && node->ne[2] > 0 && + node->src[0]->ne[0] > 0 && node->src[0]->ne[1] > 0 && + node->src[1]->ne[0] > 0 && node->src[1]->ne[1] > 0 && + node->src[1]->ne[2] > 0; +} + +static bool gtx_l2_softmax_basic_shape_ok(const ggml_tensor * node) { + if (node == nullptr || + node->src[0] == nullptr || + node->src[1] != nullptr || + !gtx_tensor_remote_f16_dense(node) || + !gtx_tensor_remote_f16_dense(node->src[0]) || + !gtx_tensor_same_shape(node, node->src[0]) || + node->ne[0] <= 0 || + node->ne[0] > UINT16_MAX) { + return false; + } + + const int32_t scale_bits = node->op_params[0]; + const int32_t max_bias_bits = node->op_params[1]; + return scale_bits == (int32_t)0x3f800000u && + max_bias_bits == 0; +} + +static bool gtx_l2_rms_norm_shape_ok(const ggml_tensor * node) { + return gtx_l2_dense_unary_shape_ok(node) && + node->ne[0] > 0 && + node->ne[0] <= UINT16_MAX && + (uint64_t)node->ne[0] * sizeof(uint16_t) <= 384ull * 1024ull; +} + +static bool gtx_l2_swiglu_shape_ok(const ggml_tensor * node) { + return node != nullptr && + node->src[0] != nullptr && + node->src[1] == nullptr && + gtx_is_swiglu_node(node) && + gtx_tensor_remote_f16_dense(node) && + gtx_tensor_remote_f16_dense(node->src[0]) && + node->ne[0] > 0 && + node->ne[0] <= UINT16_MAX && + node->src[0]->ne[0] == node->ne[0] * 2 && + node->src[0]->ne[0] <= UINT16_MAX && + node->src[0]->ne[1] == node->ne[1] && + node->src[0]->ne[2] == node->ne[2] && + node->src[0]->ne[3] == node->ne[3] && + (uint64_t)node->ne[0] * sizeof(uint16_t) <= 384ull * 1024ull; +} + +static bool gtx_l2_mul_mat_shape_ok(const ggml_tensor * node) { + if (node == nullptr || + node->src[0] == nullptr || + node->src[1] == nullptr || + !gtx_tensor_remote_f16_dense(node) || + !gtx_tensor_remote_f16_dense(node->src[0]) || + !gtx_tensor_remote_f16_dense(node->src[1])) { + return false; + } + + const ggml_tensor * src0 = node->src[0]; + const ggml_tensor * src1 = node->src[1]; + const int64_t k = src0->ne[0]; + const int64_t rows = src0->ne[1]; + const int64_t cols = src1->ne[1]; + const int64_t batch2 = node->ne[2]; + const int64_t batch3 = node->ne[3]; + + return k > 0 && rows > 0 && cols > 0 && + batch2 > 0 && batch3 > 0 && + k <= UINT16_MAX && + rows <= UINT16_MAX && + cols <= UINT16_MAX && + batch2 <= UINT16_MAX && + batch3 <= UINT16_MAX && + src1->ne[0] == k && + node->ne[0] == rows && + node->ne[1] == cols && + src1->ne[2] == batch2 && + src1->ne[3] == batch3 && + src0->ne[2] > 0 && + src0->ne[3] > 0 && + (batch2 % src0->ne[2]) == 0 && + (batch3 % src0->ne[3]) == 0; +} + +static bool gtx_l2_mul_mat_vector_shape_ok(const ggml_tensor * node) { + if (node == nullptr || + node->src[0] == nullptr || + node->src[1] == nullptr || + !gtx_tensor_remote_f16_vector_contiguous(node) || + !gtx_tensor_remote_f16_vector_contiguous(node->src[0]) || + !gtx_tensor_remote_f16_vector_contiguous(node->src[1])) { + return false; + } + + const ggml_tensor * src0 = node->src[0]; + const ggml_tensor * src1 = node->src[1]; + const int64_t k = src0->ne[0]; + const int64_t rows = src0->ne[1]; + const int64_t cols = src1->ne[1]; + const int64_t batch2 = node->ne[2]; + const int64_t batch3 = node->ne[3]; + + return k > 0 && rows > 0 && cols > 0 && + batch2 > 0 && batch3 > 0 && + k <= UINT16_MAX && + rows <= UINT16_MAX && + cols <= UINT16_MAX && + batch2 <= UINT16_MAX && + batch3 <= UINT16_MAX && + src1->ne[0] == k && + node->ne[0] == rows && + node->ne[1] == cols && + src1->ne[2] == batch2 && + src1->ne[3] == batch3 && + src0->ne[2] > 0 && + src0->ne[3] > 0 && + (batch2 % src0->ne[2]) == 0 && + (batch3 % src0->ne[3]) == 0; +} + +static bool gtx_l2_materialize_producer_shape_ok(const ggml_tensor * node) { + constexpr uint64_t gtx_host_stash_slot_bytes = 5ull * 1024ull * 1024ull; + return node != nullptr && + node->src[0] != nullptr && + gtx_tensor_remote_f16_dense(node) && + gtx_remote_tensor_type(node->src[0]) == GGML_TYPE_F16 && + gtx_tensor_same_shape(node, node->src[0]) && + node->ne[0] > 0 && + node->ne[0] <= UINT16_MAX && + (uint64_t) gtx_remote_size_bytes(node) > 0 && + (uint64_t) gtx_remote_size_bytes(node) <= gtx_host_stash_slot_bytes; +} + +static bool gtx_l2_materialize_producer_tile_shape_ok(const ggml_tensor * node) { + return node != nullptr && + node->src[0] != nullptr && + gtx_tensor_remote_f16_dense(node) && + gtx_remote_tensor_type(node->src[0]) == GGML_TYPE_F16 && + gtx_tensor_same_shape(node, node->src[0]) && + node->ne[0] > 0 && + node->ne[0] <= UINT16_MAX; +} + +static bool gtx_l2_get_rows_producer_shape_ok(const ggml_tensor * node) { + constexpr uint64_t gtx_host_stash_slot_bytes = 5ull * 1024ull * 1024ull; + if (node == nullptr || + node->src[0] == nullptr || + node->src[1] == nullptr || + !gtx_tensor_remote_f16_dense(node) || + gtx_remote_tensor_type(node->src[0]) != GGML_TYPE_F16 || + (node->src[1]->type != GGML_TYPE_I32 && node->src[1]->type != GGML_TYPE_I64) || + node->src[0]->ne[0] <= 0 || + node->src[0]->ne[0] > UINT16_MAX) { + return false; + } + + const uint64_t row_bytes = (uint64_t)node->src[0]->ne[0] * sizeof(uint16_t); + const uint64_t dst_bytes = (uint64_t)gtx_remote_size_bytes(node); + return row_bytes > 0 && + row_bytes <= UINT16_MAX && + dst_bytes > 0 && + dst_bytes <= gtx_host_stash_slot_bytes; +} + +// Initial L2 SET_ROWS support is intentionally narrow: src2 already aliases +// the full destination backing region, so the device can keep that resident +// slot and overwrite only the indexed rows from src0. +static bool gtx_l2_set_rows_producer_shape_ok(const ggml_tensor * node) { + constexpr uint64_t gtx_host_stash_slot_bytes = 5ull * 1024ull * 1024ull; + if (node == nullptr || + node->src[0] == nullptr || node->src[1] == nullptr || node->src[2] == nullptr || + !gtx_tensor_remote_f16_dense(node) || + !gtx_tensor_remote_f16_dense(node->src[0]) || + !gtx_tensor_remote_f16_dense(node->src[2]) || + gtx_remote_tensor_type(node->src[0]) != GGML_TYPE_F16 || + gtx_remote_tensor_type(node->src[2]) != GGML_TYPE_F16 || + (node->src[1]->type != GGML_TYPE_I32 && node->src[1]->type != GGML_TYPE_I64) || + gtx_remote_data_ptr(node) != gtx_remote_data_ptr(node->src[2]) || + !gtx_tensor_same_shape(node, node->src[2]) || + node->ne[0] <= 0 || node->ne[0] > UINT16_MAX || + node->src[0]->ne[0] != node->ne[0] || + node->src[0]->ne[1] <= 0 || node->src[0]->ne[1] > node->ne[1] || + node->src[0]->ne[2] != node->ne[2] || + node->src[0]->ne[3] != node->ne[3]) { + return false; + } + + const uint64_t row_bytes = (uint64_t)node->ne[0] * sizeof(uint16_t); + const uint64_t data_bytes = row_bytes * (uint64_t)node->src[0]->ne[1] * + (uint64_t)node->ne[2] * (uint64_t)node->ne[3]; + const uint64_t dst_bytes = (uint64_t)gtx_remote_size_bytes(node); + return row_bytes > 0 && row_bytes <= UINT16_MAX && + data_bytes > 0 && data_bytes <= gtx_host_stash_slot_bytes && + dst_bytes > 0 && dst_bytes <= gtx_host_stash_slot_bytes; +} + +static bool gtx_l2_rope_shape_ok(const ggml_tensor * node) { + if (node == nullptr || node->src[0] == nullptr || node->src[1] == nullptr || + !gtx_tensor_remote_f16_dense(node) || + !gtx_tensor_remote_f16_dense(node->src[0]) || + !gtx_tensor_same_shape(node, node->src[0]) || + gtx_remote_tensor_type(node->src[1]) != GGML_TYPE_I32 || + !ggml_is_contiguous(node->src[1]) || + node->op_params[2] != GGML_ROPE_TYPE_NORMAL || + node->ne[0] <= 0 || node->ne[0] > UINT16_MAX || + node->ne[1] <= 0 || node->ne[1] > UINT16_MAX || + node->ne[2] <= 0 || node->ne[2] > UINT16_MAX || + node->ne[3] <= 0 || node->ne[3] > UINT16_MAX || + node->op_params[1] < 2 || node->op_params[1] > node->ne[0] || + (node->op_params[1] & 1) != 0 || + node->src[1]->ne[0] < node->ne[2]) { + return false; + } + + const uint64_t rows = (uint64_t)node->ne[1] * node->ne[2] * node->ne[3]; + const uint64_t padded_pairs = (((uint64_t)node->op_params[1] / 2u) + 15u) & ~15ull; + const uint64_t table_bytes = (uint64_t)node->ne[2] * padded_pairs * sizeof(uint16_t); + return (uint64_t)node->ne[0] * sizeof(uint16_t) <= UINT16_MAX && + (uint64_t)node->ne[2] * sizeof(int32_t) <= UINT16_MAX && + rows <= UINT16_MAX && table_bytes <= 64u * 1024u && + ggml_gtx_rope_scratch_size((uint32_t)node->ne[2], + (uint32_t)node->op_params[1]) != 0u; +} + +static gtx_l2_chainability gtx_l2_chainability_from_node(const ggml_tensor * node) { + const uint32_t kernel_id = gtx_kernel_id_from_node(node); + if (node == nullptr || kernel_id == GGML_GTX_KERNEL_INVALID) { + return { gtx_l2_chain_kind::unsupported, kernel_id, "unsupported_kernel" }; + } + + switch (kernel_id) { + case GGML_GTX_KERNEL_ADD_F16: + return { gtx_l2_add_shape_ok(node) ? gtx_l2_chain_kind::native : gtx_l2_chain_kind::boundary, + kernel_id, + gtx_l2_add_shape_ok(node) ? "add_l2_native" : "add_shape_or_layout_boundary" }; + case GGML_GTX_KERNEL_MUL_F16: + return { (gtx_l2_dense_binary_shape_ok(node) || + gtx_l2_rhs_broadcast_shape_ok(node, true)) ? gtx_l2_chain_kind::native : gtx_l2_chain_kind::boundary, + kernel_id, + (gtx_l2_dense_binary_shape_ok(node) || + gtx_l2_rhs_broadcast_shape_ok(node, true)) ? "mul_l2_native" : "mul_shape_or_layout_boundary" }; + case GGML_GTX_KERNEL_SCALE_F16: + return { gtx_l2_scale_shape_ok(node) ? gtx_l2_chain_kind::native : gtx_l2_chain_kind::boundary, + kernel_id, + gtx_l2_scale_shape_ok(node) ? "scale_l2_native" : "scale_shape_or_layout_boundary" }; + case GGML_GTX_KERNEL_GELU_F16: + case GGML_GTX_KERNEL_ABS_F16: + case GGML_GTX_KERNEL_NEG_F16: + case GGML_GTX_KERNEL_SGN_F16: + case GGML_GTX_KERNEL_CEIL_F16: + case GGML_GTX_KERNEL_FLOOR_F16: + case GGML_GTX_KERNEL_TANH_F16: + case GGML_GTX_KERNEL_SILU_F16: + case GGML_GTX_KERNEL_RELU_F16: + case GGML_GTX_KERNEL_LEAKY_RELU_F16: + case GGML_GTX_KERNEL_CLAMP_F16: + return { gtx_l2_dense_unary_shape_ok(node) ? gtx_l2_chain_kind::native : gtx_l2_chain_kind::boundary, + kernel_id, + gtx_l2_dense_unary_shape_ok(node) ? "unary_l2_native" : "unary_shape_or_layout_boundary" }; + case GGML_GTX_KERNEL_SOFT_MAX_F16: + return { gtx_l2_softmax_basic_shape_ok(node) ? gtx_l2_chain_kind::native : gtx_l2_chain_kind::boundary, + kernel_id, + gtx_l2_softmax_basic_shape_ok(node) ? "softmax_basic_l2_native" : "softmax_mask_or_param_boundary" }; + case GGML_GTX_KERNEL_MUL_MAT_F16: + if (gtx_l2_mul_mat_shape_ok(node)) { + return { gtx_l2_chain_kind::native, kernel_id, "mul_mat_l2_native" }; + } + return { gtx_l2_mul_mat_vector_shape_ok(node) ? gtx_l2_chain_kind::native : gtx_l2_chain_kind::boundary, + kernel_id, + gtx_l2_mul_mat_vector_shape_ok(node) ? "mul_mat_vector_l2_native" : "mul_mat_shape_or_layout_boundary" }; + case GGML_GTX_KERNEL_RMS_NORM_F16: + return { gtx_l2_rms_norm_shape_ok(node) ? gtx_l2_chain_kind::native : gtx_l2_chain_kind::boundary, + kernel_id, + gtx_l2_rms_norm_shape_ok(node) ? "rms_norm_l2_native" : "rms_norm_shape_or_layout_boundary" }; + case GGML_GTX_KERNEL_SWIGLU_F16: + return { gtx_l2_swiglu_shape_ok(node) ? gtx_l2_chain_kind::native : gtx_l2_chain_kind::boundary, + kernel_id, + gtx_l2_swiglu_shape_ok(node) ? "swiglu_l2_native" : "swiglu_shape_or_layout_boundary" }; + case GGML_GTX_KERNEL_REPEAT_F16: + return { gtx_l2_repeat_shape_ok(node) ? gtx_l2_chain_kind::native : gtx_l2_chain_kind::boundary, + kernel_id, + gtx_l2_repeat_shape_ok(node) ? "repeat_l2_native" : "repeat_shape_or_layout_boundary" }; + case GGML_GTX_KERNEL_CONCAT_F16: + return { gtx_l2_concat_shape_ok(node) ? gtx_l2_chain_kind::native : gtx_l2_chain_kind::boundary, + kernel_id, + gtx_l2_concat_shape_ok(node) ? "concat_l2_native" : "concat_shape_or_layout_boundary" }; + case GGML_GTX_KERNEL_PAD_F16: + return { gtx_l2_pad_shape_ok(node) ? gtx_l2_chain_kind::native : gtx_l2_chain_kind::boundary, + kernel_id, + gtx_l2_pad_shape_ok(node) ? "pad_l2_native" : "pad_shape_or_layout_boundary" }; + case GGML_GTX_KERNEL_CONV_TRANSPOSE_2D_F16: + return { gtx_l2_conv_transpose_shape_ok(node) ? gtx_l2_chain_kind::native : gtx_l2_chain_kind::boundary, + kernel_id, + gtx_l2_conv_transpose_shape_ok(node) ? "conv_transpose_l2_native" : "conv_transpose_shape_or_budget_boundary" }; + case GGML_GTX_KERNEL_CONV_2D_F16: + case GGML_GTX_KERNEL_CONV_2D_DW_F16: + return { gtx_l2_conv_whole_shape_ok(node) ? gtx_l2_chain_kind::native : gtx_l2_chain_kind::boundary, + kernel_id, + gtx_l2_conv_whole_shape_ok(node) ? "conv_l2_native" : "conv_needs_host_spatial_tile" }; + case GGML_GTX_KERNEL_CONT_F16: + case GGML_GTX_KERNEL_CPY_F16: + return { gtx_l2_chain_kind::boundary, kernel_id, + "layout_materialize_boundary" }; + case GGML_GTX_KERNEL_GET_ROWS_F16: + return { gtx_l2_chain_kind::boundary, kernel_id, + "gather_boundary" }; + case GGML_GTX_KERNEL_SET_ROWS_F16: + return { gtx_l2_chain_kind::boundary, kernel_id, + "scatter_alias_boundary" }; + case GGML_GTX_KERNEL_ROPE_F16: + return { gtx_l2_rope_shape_ok(node) ? gtx_l2_chain_kind::native : gtx_l2_chain_kind::boundary, + kernel_id, + gtx_l2_rope_shape_ok(node) ? "rope_l2_native" : "rope_shape_or_mode_boundary" }; + case GGML_GTX_KERNEL_SIN_F16: + case GGML_GTX_KERNEL_COS_F16: + return { gtx_l2_dense_unary_shape_ok(node) && (ggml_nelements(node) % 16) == 0 ? + gtx_l2_chain_kind::native : gtx_l2_chain_kind::boundary, + kernel_id, + gtx_l2_dense_unary_shape_ok(node) && (ggml_nelements(node) % 16) == 0 ? + "sincos_l2_native" : "sincos_shape_boundary" }; + case GGML_GTX_KERNEL_NORM_F16: + case GGML_GTX_KERNEL_ADD_F16_F32_TO_F16: + return { gtx_l2_chain_kind::boundary, kernel_id, "coherence_boundary" }; + default: + return { gtx_l2_chain_kind::unsupported, kernel_id, "not_chain_advertised" }; + } +} + +static void gtx_log_chainability_decision(const ggml_tensor * node) { + if (!gtx_log_chain_ops_enabled() || node == nullptr) { + return; + } + const gtx_l2_chainability chain = gtx_l2_chainability_from_node(node); + fprintf(stderr, + "[GTX CHAIN] op=%s tensor=%s kernel_id=%u kind=%s participates=%s reason=%s shape=[%lld,%lld,%lld,%lld]\n", + gtx_tensor_op_display_name(node), + node->name[0] != '\0' ? node->name : "(anon)", + (unsigned) chain.kernel_id, + gtx_l2_chain_kind_name(chain.kind), + gtx_l2_chain_participates(chain.kind) ? "yes" : "no", + chain.reason, + (long long) node->ne[0], + (long long) node->ne[1], + (long long) node->ne[2], + (long long) node->ne[3]); + if ((chain.kernel_id == GGML_GTX_KERNEL_ADD_F16 || + chain.kernel_id == GGML_GTX_KERNEL_MUL_F16) && + node->src[0] != nullptr && node->src[1] != nullptr) { + fprintf(stderr, + "[GTX BINARY SHAPE] op=%s dst=[%lld,%lld,%lld,%lld] " + "src0=[%lld,%lld,%lld,%lld] src1=[%lld,%lld,%lld,%lld]\n", + gtx_tensor_op_display_name(node), + (long long)node->ne[0], (long long)node->ne[1], + (long long)node->ne[2], (long long)node->ne[3], + (long long)node->src[0]->ne[0], (long long)node->src[0]->ne[1], + (long long)node->src[0]->ne[2], (long long)node->src[0]->ne[3], + (long long)node->src[1]->ne[0], (long long)node->src[1]->ne[1], + (long long)node->src[1]->ne[2], (long long)node->src[1]->ne[3]); + } + if ((chain.kind == gtx_l2_chain_kind::boundary && + (chain.kernel_id == GGML_GTX_KERNEL_CONV_2D_F16 || + chain.kernel_id == GGML_GTX_KERNEL_CONV_2D_DW_F16)) || + chain.kernel_id == GGML_GTX_KERNEL_ADD_F16 || + chain.kernel_id == GGML_GTX_KERNEL_MUL_F16) { + const auto log_layout = [](const char * label, const ggml_tensor * tensor) { + if (tensor == nullptr) { + fprintf(stderr, "[GTX CHAIN] %s=-\n", label); + return; + } + fprintf(stderr, + "[GTX CHAIN] %s remote_type=%s ne=[%lld,%lld,%lld,%lld] " + "remote_nb=[%llu,%llu,%llu,%llu]\n", + label, ggml_type_name(gtx_remote_tensor_type(tensor)), + (long long)tensor->ne[0], (long long)tensor->ne[1], + (long long)tensor->ne[2], (long long)tensor->ne[3], + (unsigned long long)gtx_remote_byte_offset(tensor, tensor->nb[0]), + (unsigned long long)gtx_remote_byte_offset(tensor, tensor->nb[1]), + (unsigned long long)gtx_remote_byte_offset(tensor, tensor->nb[2]), + (unsigned long long)gtx_remote_byte_offset(tensor, tensor->nb[3])); + }; + log_layout("dst", node); + log_layout("src0", node->src[0]); + log_layout("src1", node->src[1]); + } +} + +static void gtx_log_graph_chainability(const ggml_cgraph * cgraph) { + if (!gtx_log_chain_ops_enabled() || cgraph == nullptr) { + return; + } + fprintf(stderr, "[GTX CHAIN] n_nodes=%d\n", cgraph->n_nodes); + for (int i = 0; i < cgraph->n_nodes; ++i) { + gtx_log_chainability_decision(cgraph->nodes[i]); + } +} + +static bool gtx_build_kernel_launch_desc(const ggml_tensor * node, + ggml_gtx_kernel_launch_desc_t & desc) { + const uint32_t kernel_id = gtx_kernel_id_from_node(node); + if (kernel_id == GGML_GTX_KERNEL_INVALID) { + if (node != nullptr) { + GTX_LOG_ERROR("unsupported descriptor op: %s (%d)\n", + gtx_tensor_op_display_name(node), (int) node->op); + } + return false; + } + + desc = {}; + desc.kernel_id = kernel_id; + desc.abi_version = GGML_GTX_GRAPH_V2_VERSION; + // A normal, untiled launch is an explicit coherent DDR boundary. The + // scheduler replaces this with L2_REQUIRED only after it has lowered a + // complete contiguous raw-view tile. + desc.flags = GGML_GTX_DESC_FLAG_DDR_ALLOWED; + + if (!gtx_fill_tensor_desc(node, &desc.dst_addr, &desc.dtype, desc.ne, desc.nb)) { + GTX_LOG_ERROR("Failed to describe destination tensor for op: %s\n", + gtx_tensor_op_display_name(node)); + return false; + } + + for (int i = 0; i < 4; ++i) { + if (!gtx_fill_tensor_desc(node->src[i], + &desc.src_addr[i], + &desc.src_type[i], + desc.src_ne[i], + desc.src_nb[i])) { + GTX_LOG_ERROR("Failed to describe source %d for op: %s\n", + i, gtx_tensor_op_display_name(node)); + return false; + } + } + + const int n_params = GGML_MAX_OP_PARAMS / sizeof(int32_t); + for (int i = 0; i < n_params && i < 16; ++i) { + desc.op_params[i] = node->op_params[i]; + } + + desc.l2_scratch_size = gtx_kernel_l2_scratch_size(node); + + return desc.dst_addr != 0; +} + +static bool gtx_build_launch_payload(uint32_t device, + const ggml_gtx_kernel_launch_desc_t * descs, + size_t n_desc, + std::vector & output) { + if (descs == nullptr || n_desc == 0 || n_desc > UINT32_MAX) { + return false; + } + + ggml_gtx_graph_v2_header_t hdr = {}; + hdr.magic = GGML_GTX_GRAPH_V2_MAGIC; + hdr.version = GGML_GTX_GRAPH_V2_VERSION; + hdr.header_size = sizeof(hdr); + hdr.device = device; + hdr.n_desc = (uint32_t) n_desc; + hdr.desc_size = sizeof(ggml_gtx_kernel_launch_desc_t); + + output.clear(); + output.reserve(sizeof(hdr) + n_desc * sizeof(ggml_gtx_kernel_launch_desc_t)); + const uint8_t * hdr_bytes = reinterpret_cast(&hdr); + output.insert(output.end(), hdr_bytes, hdr_bytes + sizeof(hdr)); + const uint8_t * desc_bytes = reinterpret_cast(descs); + output.insert(output.end(), desc_bytes, desc_bytes + n_desc * sizeof(ggml_gtx_kernel_launch_desc_t)); + return true; +} + +static bool gtx_build_single_launch_payload(uint32_t device, + const ggml_gtx_kernel_launch_desc_t & desc, + std::vector & output) { + return gtx_build_launch_payload(device, &desc, 1, output); +} + +static bool gtx_submit_launch_desc(ggml_backend_gtx_context * ctx, + const ggml_gtx_kernel_launch_desc_t & desc) { + std::vector graph_blob; + if (!gtx_build_single_launch_payload((uint32_t) ctx->device_id, desc, graph_blob)) { + return false; + } + + if (!ctx->conn->send_data(GTX_BAR2_STAGING_OFFSET, graph_blob.data(), graph_blob.size()) || + !ctx->conn->send_cmd_staged(GTX_CMD_GRAPH_COMPUTE, + GTX_BAR2_DEVICE_BASE + GTX_BAR2_STAGING_OFFSET, + (uint32_t) graph_blob.size(), nullptr, 0)) { + return false; + } + + gtx_pcie_irq_ctx * irq_ctx = ctx->get_irq_ctx(); + return irq_ctx == nullptr || gtx_wait_infer_done(irq_ctx, GTX_GRAPH_COMPUTE_TIMEOUT_MS) >= 0; +} + +static bool gtx_submit_launch_descs(ggml_backend_gtx_context * ctx, + const std::vector & descs) { + if (ctx == nullptr || descs.empty()) { + return false; + } + + if (ctx->use_uart) { + for (size_t i = 0; i < descs.size(); ++i) { + const ggml_gtx_kernel_launch_desc_t & desc = descs[i]; + if (!gtx_uart_launch_desc(*ctx->uart, desc)) { + GTX_LOG_ERROR("descriptor launch failed: index=%zu/%zu kernel=%u flags=0x%x " + "dst=[%u,%u,%u,%u] src0=[%u,%u,%u,%u] " + "src1=[%u,%u,%u,%u] scratch=%u\n", + i, descs.size(), desc.kernel_id, desc.flags, + desc.ne[0], desc.ne[1], desc.ne[2], desc.ne[3], + desc.src_ne[0][0], desc.src_ne[0][1], + desc.src_ne[0][2], desc.src_ne[0][3], + desc.src_ne[1][0], desc.src_ne[1][1], + desc.src_ne[1][2], desc.src_ne[1][3], + desc.l2_scratch_size); + return false; + } + } + return true; + } + + std::vector graph_blob; + if (!gtx_build_launch_payload((uint32_t) ctx->device_id, descs.data(), descs.size(), graph_blob)) { + return false; + } + + if (!ctx->conn->send_data(GTX_BAR2_STAGING_OFFSET, graph_blob.data(), graph_blob.size()) || + !ctx->conn->send_cmd_staged(GTX_CMD_GRAPH_COMPUTE, + GTX_BAR2_DEVICE_BASE + GTX_BAR2_STAGING_OFFSET, + (uint32_t) graph_blob.size(), nullptr, 0)) { + return false; + } + + gtx_pcie_irq_ctx * irq_ctx = ctx->get_irq_ctx(); + return irq_ctx == nullptr || gtx_wait_infer_done(irq_ctx, GTX_GRAPH_COMPUTE_TIMEOUT_MS) >= 0; +} + +static bool gtx_copy_tensor_rpc(gtx_pcie_connection * conn, + const ggml_tensor * src, + ggml_tensor * dst) { + if (conn == nullptr || !conn->is_open() || src == nullptr || dst == nullptr) { + return false; + } + + gtx_msg_copy_tensor_req req = {}; + req.src = gtx_serialize_tensor(src); + req.dst = gtx_serialize_tensor(dst); + + if (req.src.data == 0 || req.dst.data == 0) { + GTX_LOG_ERROR("COPY_TENSOR requires GTX-backed src and dst tensors\n"); + return false; + } + + gtx_msg_copy_tensor_rsp rsp = {}; + if (!conn->send_data(GTX_BAR2_STAGING_OFFSET, &req, sizeof(req)) || + !conn->send_cmd_staged(GTX_CMD_COPY_TENSOR, + GTX_BAR2_DEVICE_BASE + GTX_BAR2_STAGING_OFFSET, + (uint32_t) sizeof(req), &rsp, sizeof(rsp))) { + GTX_LOG_ERROR("COPY_TENSOR command failed\n"); + return false; + } + + return rsp.result != 0; +} + +static bool gtx_is_metadata_op(enum ggml_op op) { + return op == GGML_OP_NONE || + op == GGML_OP_RESHAPE || + op == GGML_OP_VIEW || + op == GGML_OP_PERMUTE || + op == GGML_OP_TRANSPOSE; +} + +static void gtx_format_shape(const ggml_tensor * tensor, char * buf, size_t buf_size) { + if (tensor == nullptr) { + snprintf(buf, buf_size, "-"); + return; + } + + snprintf(buf, buf_size, "[%lld,%lld,%lld,%lld]", + (long long) tensor->ne[0], + (long long) tensor->ne[1], + (long long) tensor->ne[2], + (long long) tensor->ne[3]); +} + +static void gtx_format_tensor_detail(const ggml_tensor * tensor, char * buf, size_t buf_size) { + if (tensor == nullptr) { + snprintf(buf, buf_size, "-"); + return; + } + + snprintf(buf, buf_size, + "%s type=%s->%s shape=[%lld,%lld,%lld,%lld] nb=[%zu,%zu,%zu,%zu] addr=0x%llx", + tensor->name[0] != '\0' ? tensor->name : "(anon)", + ggml_type_name(tensor->type), + ggml_type_name(gtx_remote_tensor_type(tensor)), + (long long) tensor->ne[0], + (long long) tensor->ne[1], + (long long) tensor->ne[2], + (long long) tensor->ne[3], + (size_t) tensor->nb[0], + (size_t) tensor->nb[1], + (size_t) tensor->nb[2], + (size_t) tensor->nb[3], + (unsigned long long) gtx_remote_data_ptr(tensor)); +} + +static void gtx_log_op_shapes(const ggml_tensor * node) { + if (!gtx_log_ops_enabled()) { + return; + } + + char dst[256]; + char src0[256]; + char src1[256]; + char src2[256]; + + gtx_format_tensor_detail(node, dst, sizeof(dst)); + gtx_format_tensor_detail(node != nullptr ? node->src[0] : nullptr, src0, sizeof(src0)); + gtx_format_tensor_detail(node != nullptr ? node->src[1] : nullptr, src1, sizeof(src1)); + gtx_format_tensor_detail(node != nullptr ? node->src[2] : nullptr, src2, sizeof(src2)); + + if (node != nullptr && + gtx_normalize_host_op(node) == (uint32_t) GGML_OP_UNARY) { + const int32_t unary_op = node->op_params[0]; + const char * unary_name = + unary_op >= 0 && unary_op < GGML_UNARY_OP_COUNT ? + ggml_unary_op_name((ggml_unary_op) unary_op) : "unknown"; + fprintf(stderr, + "[GTX OP] %s unary=%s(%d)\n dst=%s\n src0=%s\n src1=%s\n src2=%s\n", + gtx_tensor_op_display_name(node), + unary_name, unary_op, + dst, src0, src1, src2); + return; + } + + fprintf(stderr, "[GTX OP] %s\n dst=%s\n src0=%s\n src1=%s\n src2=%s\n", + gtx_tensor_op_display_name(node), + dst, src0, src1, src2); +} + +static void gtx_dump_graph_node_types(const ggml_cgraph * cgraph) { + if (!gtx_dump_graph_types_enabled() || cgraph == nullptr) { + return; + } + + const char * path = getenv("GTX_GRAPH_TYPES_FILE"); + FILE * out = stderr; + if (path != nullptr && path[0] != '\0') { + out = fopen(path, "a"); + if (out == nullptr) { + out = stderr; + fprintf(stderr, "[GTX WARN] failed to open GTX_GRAPH_TYPES_FILE=%s\n", path); + } + } + + fprintf(out, "[GTX GRAPH] n_nodes=%d\n", cgraph->n_nodes); + for (int i = 0; i < cgraph->n_nodes; ++i) { + const ggml_tensor * node = cgraph->nodes[i]; + if (node == nullptr) { + fprintf(out, "[GTX GRAPH] node=%d op=NULL dst=-\n", i); + continue; + } + + fprintf(out, + "[GTX GRAPH] node=%d op=%s dst=%s:%s->%s", + i, + gtx_tensor_op_display_name(node), + node->name[0] != '\0' ? node->name : "(anon)", + ggml_type_name(node->type), + ggml_type_name(gtx_remote_tensor_type(node))); + + for (int j = 0; j < GGML_MAX_SRC; ++j) { + const ggml_tensor * src = node->src[j]; + fprintf(out, + " src%d=%s", + j, + src != nullptr ? (src->name[0] != '\0' ? src->name : "(anon)") : "-"); + if (src != nullptr) { + fprintf(out, ":%s->%s", + ggml_type_name(src->type), + ggml_type_name(gtx_remote_tensor_type(src))); + } + } + fprintf(out, "\n"); + } + + if (out != stderr) { + fclose(out); + } +} + +static void gtx_log_supports_op_decision(const ggml_tensor * node, bool supported, const char * reason) { + if (!gtx_log_supports_ops_enabled() || node == nullptr) { + return; + } + + char dst[256]; + char src0[256]; + char src1[256]; + char src2[256]; + + gtx_format_tensor_detail(node, dst, sizeof(dst)); + gtx_format_tensor_detail(node->src[0], src0, sizeof(src0)); + gtx_format_tensor_detail(node->src[1], src1, sizeof(src1)); + gtx_format_tensor_detail(node->src[2], src2, sizeof(src2)); + + const uint32_t raw_op = (uint32_t) node->op; + const uint32_t normalized_op = gtx_normalize_host_op(node); + const char * display_op = gtx_tensor_op_display_name(node); + + fprintf(stderr, + "[GTX SUPPORT] %s op=%s raw_op=%u(%s) normalized_op=%u(%s) reason=%s\n" + " dst=%s\n" + " src0=%s\n" + " src1=%s\n" + " src2=%s\n", + supported ? "yes" : "no", + display_op, + raw_op, gtx_op_name_or_unknown(raw_op), + normalized_op, gtx_normalized_op_name(normalized_op), + reason != nullptr ? reason : "-", + dst, src0, src1, src2); +} + +static bool gtx_download_tensor_prefix_as_f32(ggml_backend_gtx_context * ctx, + const ggml_tensor * tensor, + size_t max_elems, + std::vector & out) { + out.clear(); + if (ctx == nullptr || tensor == nullptr || max_elems == 0) { + return false; + } + + const size_t elems = (size_t) std::min((int64_t) max_elems, ggml_nelements(tensor)); + if (elems == 0) { + return false; + } + + if (tensor->type == GGML_TYPE_F32) { + std::vector tmp(elems); + const size_t local_bytes = elems * sizeof(float); + const size_t remote_bytes = (size_t) gtx_remote_byte_offset(tensor, local_bytes); + std::vector packed(remote_bytes); + const uint64_t remote = gtx_remote_data_ptr(tensor); + + bool ok = false; + if (ctx->use_uart) { + ok = ctx->uart != nullptr && + ctx->uart->download_bytes(remote, packed.data(), packed.size()); + } else if (ctx->conn != nullptr && ctx->conn->is_open()) { + gtx_msg_get_tensor_req req = {}; + req.tensor = gtx_serialize_tensor(tensor); + req.offset = 0; + req.size = (uint32_t) packed.size(); + ok = ctx->conn->send_data(GTX_BAR2_STAGING_OFFSET, &req, sizeof(req)) && + ctx->conn->send_cmd_staged(GTX_CMD_GET_TENSOR, + GTX_BAR2_DEVICE_BASE + GTX_BAR2_STAGING_OFFSET, + (uint32_t) sizeof(req), nullptr, 0) && + ctx->conn->recv_data(GTX_BAR2_STAGING_OFFSET, packed.data(), packed.size()); + } + if (!ok) { + return false; + } + + gtx_unpack_tensor_chunk(tensor, packed.data(), tmp.data(), local_bytes); + out.swap(tmp); + return true; + } + + if (tensor->type == GGML_TYPE_F16) { + std::vector tmp(elems); + const size_t bytes = elems * sizeof(ggml_fp16_t); + const uint64_t remote = gtx_remote_data_ptr(tensor); + + bool ok = false; + if (ctx->use_uart) { + ok = ctx->uart != nullptr && + ctx->uart->download_bytes(remote, tmp.data(), bytes); + } else if (ctx->conn != nullptr && ctx->conn->is_open()) { + gtx_msg_get_tensor_req req = {}; + req.tensor = gtx_serialize_tensor(tensor); + req.offset = 0; + req.size = (uint32_t) bytes; + ok = ctx->conn->send_data(GTX_BAR2_STAGING_OFFSET, &req, sizeof(req)) && + ctx->conn->send_cmd_staged(GTX_CMD_GET_TENSOR, + GTX_BAR2_DEVICE_BASE + GTX_BAR2_STAGING_OFFSET, + (uint32_t) sizeof(req), nullptr, 0) && + ctx->conn->recv_data(GTX_BAR2_STAGING_OFFSET, tmp.data(), bytes); + } + if (!ok) { + return false; + } + + out.resize(elems); + ggml_fp16_to_fp32_row(tmp.data(), out.data(), (int64_t) elems); + return true; + } + + return false; +} + +static void gtx_log_softmax_tensor_stats(ggml_backend_gtx_context * ctx, + const ggml_tensor * tensor, + const char * tag) { + if (!gtx_log_softmax_enabled() || tensor == nullptr) { + return; + } + + const size_t row = tensor->ne[0] > 0 ? (size_t) tensor->ne[0] : 0; + if (row == 0) { + return; + } + + const uint64_t env_limit = gtx_env_u64_or_default("GTX_LOG_SOFTMAX_ELEMS", 65536); + const size_t elems = (size_t) std::min((uint64_t) ggml_nelements(tensor), + std::max(row, env_limit)); + std::vector values; + if (!gtx_download_tensor_prefix_as_f32(ctx, tensor, elems, values) || values.empty()) { + fprintf(stderr, "[GTX SOFTMAX] %s tensor=%s download failed\n", + tag, tensor->name[0] != '\0' ? tensor->name : "(anon)"); + return; + } + + float min_v = std::numeric_limits::infinity(); + float max_v = -std::numeric_limits::infinity(); + size_t nan_count = 0; + size_t inf_count = 0; + size_t neg_count = 0; + size_t gt1_count = 0; + for (float v : values) { + if (std::isnan(v)) { + nan_count++; + continue; + } + if (!std::isfinite(v)) { + inf_count++; + continue; + } + min_v = std::min(min_v, v); + max_v = std::max(max_v, v); + if (v < 0.0f) { + neg_count++; + } + if (v > 1.0f) { + gt1_count++; + } + } + + const size_t row_elems = std::min(row, values.size()); + double row_sum = 0.0; + float row_min = std::numeric_limits::infinity(); + float row_max = -std::numeric_limits::infinity(); + for (size_t i = 0; i < row_elems; ++i) { + const float v = values[i]; + if (std::isfinite(v)) { + row_sum += (double) v; + row_min = std::min(row_min, v); + row_max = std::max(row_max, v); + } + } + + const size_t head = std::min(8, values.size()); + fprintf(stderr, + "[GTX SOFTMAX] %s tensor=%s graph_type=%s remote_type=%s shape=[%lld,%lld,%lld,%lld] elems=%zu/%lld row0_sum=%.9g row0_min=%.9g row0_max=%.9g all_min=%.9g all_max=%.9g nan=%zu inf=%zu neg=%zu gt1=%zu head=[", + tag, + tensor->name[0] != '\0' ? tensor->name : "(anon)", + ggml_type_name(tensor->type), + ggml_type_name(gtx_remote_tensor_type(tensor)), + (long long) tensor->ne[0], + (long long) tensor->ne[1], + (long long) tensor->ne[2], + (long long) tensor->ne[3], + values.size(), + (long long) ggml_nelements(tensor), + row_sum, + row_min, + row_max, + min_v, + max_v, + nan_count, + inf_count, + neg_count, + gt1_count); + for (size_t i = 0; i < head; ++i) { + fprintf(stderr, "%s%.9g", i == 0 ? "" : " ", values[i]); + } + fprintf(stderr, "]\n"); +} + +static bool gtx_dump_op_sample_should_dump(const ggml_tensor * node) { + if (!gtx_dump_op_samples_enabled() || node == nullptr) { + return false; + } + + const uint32_t normalized_op = gtx_normalize_host_op(node); + const char * op_name = gtx_normalized_op_name(normalized_op); + const char * value = getenv("GTX_DUMP_OP_SAMPLES"); + const bool all_default = value != nullptr && + (strcmp(value, "1") == 0 || + strcmp(value, "true") == 0 || + strcmp(value, "TRUE") == 0 || + strcmp(value, "on") == 0 || + strcmp(value, "ON") == 0 || + strcmp(value, "all") == 0 || + strcmp(value, "ALL") == 0); + const bool op_selected = all_default ? + (normalized_op == (uint32_t) GGML_OP_MUL || + normalized_op == (uint32_t) GGML_OP_MUL_MAT) : + gtx_env_list_contains("GTX_DUMP_OP_SAMPLES", op_name); + if (!op_selected) { + return false; + } + + const char * filter = getenv("GTX_DUMP_OP_SAMPLE_FILTER"); + if (filter != nullptr && filter[0] != '\0') { + const char * name = node->name[0] != '\0' ? node->name : ""; + bool matched = strstr(name, filter) != nullptr; + for (int i = 0; i < GGML_MAX_SRC && !matched; ++i) { + const ggml_tensor * src = node->src[i]; + const char * src_name = src != nullptr && src->name[0] != '\0' ? src->name : ""; + matched = strstr(src_name, filter) != nullptr; + } + if (!matched) { + return false; + } + } + + return true; +} + +static void gtx_dump_op_sample_tensor(ggml_backend_gtx_context * ctx, + const ggml_tensor * tensor, + const char * tag) { + if (ctx == nullptr || tensor == nullptr) { + return; + } + + const uint64_t max_elems_env = gtx_env_u64_or_default("GTX_DUMP_OP_SAMPLE_ELEMS", 64); + const size_t max_elems = max_elems_env == 0 ? + (size_t) ggml_nelements(tensor) : + (size_t) std::min(max_elems_env, (uint64_t) ggml_nelements(tensor)); + std::vector values; + if (!gtx_download_tensor_prefix_as_f32(ctx, tensor, max_elems, values) || values.empty()) { + fprintf(stderr, "[GTX SAMPLE] %s tensor=%s download failed\n", + tag != nullptr ? tag : "-", + tensor->name[0] != '\0' ? tensor->name : "(anon)"); + return; + } + + float min_v = std::numeric_limits::infinity(); + float max_v = -std::numeric_limits::infinity(); + double sum = 0.0; + size_t finite_count = 0; + size_t nan_count = 0; + size_t inf_count = 0; + for (float v : values) { + if (std::isnan(v)) { + nan_count++; + continue; + } + if (!std::isfinite(v)) { + inf_count++; + continue; + } + min_v = std::min(min_v, v); + max_v = std::max(max_v, v); + sum += (double) v; + finite_count++; + } + const double mean = finite_count > 0 ? sum / (double) finite_count : 0.0; + const size_t top_k = std::min((size_t) gtx_env_u64_or_default("GTX_DUMP_OP_SAMPLE_TOPK", 8), + values.size()); + std::vector top_indices(values.size()); + for (size_t i = 0; i < top_indices.size(); ++i) { + top_indices[i] = i; + } + if (top_k > 0) { + std::partial_sort(top_indices.begin(), + top_indices.begin() + top_k, + top_indices.end(), + [&values](size_t a, size_t b) { + const float va = values[a]; + const float vb = values[b]; + if (std::isnan(va)) { + return false; + } + if (std::isnan(vb)) { + return true; + } + return va > vb; + }); + } + + FILE * out = stderr; + const char * path = getenv("GTX_DUMP_OP_SAMPLE_FILE"); + if (path != nullptr && path[0] != '\0') { + out = fopen(path, "a"); + if (out == nullptr) { + out = stderr; + fprintf(stderr, "[GTX WARN] failed to open GTX_DUMP_OP_SAMPLE_FILE=%s\n", path); + } + } + + const size_t head = std::min(16, values.size()); + fprintf(out, + "[GTX SAMPLE] %s op=%s tensor=%s graph_type=%s remote_type=%s shape=[%lld,%lld,%lld,%lld] elems=%zu/%lld finite=%zu nan=%zu inf=%zu min=%.9g max=%.9g mean=%.9g head=[", + tag != nullptr ? tag : "-", + ggml_op_name(tensor->op), + tensor->name[0] != '\0' ? tensor->name : "(anon)", + ggml_type_name(tensor->type), + ggml_type_name(gtx_remote_tensor_type(tensor)), + (long long) tensor->ne[0], + (long long) tensor->ne[1], + (long long) tensor->ne[2], + (long long) tensor->ne[3], + values.size(), + (long long) ggml_nelements(tensor), + finite_count, + nan_count, + inf_count, + finite_count > 0 ? min_v : 0.0f, + finite_count > 0 ? max_v : 0.0f, + mean); + for (size_t i = 0; i < head; ++i) { + fprintf(out, "%s%.9g", i == 0 ? "" : " ", values[i]); + } + fprintf(out, "] top=["); + for (size_t i = 0; i < top_k; ++i) { + const size_t idx = top_indices[i]; + fprintf(out, "%s%zu:%.9g", i == 0 ? "" : " ", idx, values[idx]); + } + fprintf(out, "]\n"); + + if (out != stderr) { + fclose(out); + } +} + +static bool gtx_tensor_is_remote_compact_f16(const ggml_tensor * tensor) { + if (tensor == nullptr || gtx_remote_tensor_type(tensor) != GGML_TYPE_F16) { + return false; + } + + const gtx_rpc_tensor rpc = gtx_serialize_tensor(tensor); + uint64_t stride = sizeof(uint16_t); + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + if (rpc.nb[i] != stride) { + return false; + } + stride *= (uint64_t) std::max(tensor->ne[i], 1); + } + return true; +} + +static bool gtx_download_compact_f16_tensor_as_f32(ggml_backend_gtx_context * ctx, + const ggml_tensor * tensor, + std::vector & out) { + out.clear(); + if (ctx == nullptr || tensor == nullptr || !gtx_tensor_is_remote_compact_f16(tensor)) { + return false; + } + + const size_t elems = (size_t) ggml_nelements(tensor); + if (elems == 0) { + return false; + } + + std::vector tmp(elems); + const uint64_t remote = gtx_remote_data_ptr(tensor); + const size_t bytes = tmp.size() * sizeof(ggml_fp16_t); + bool ok = false; + if (ctx->use_uart) { + ok = ctx->uart != nullptr && ctx->uart->download_bytes(remote, tmp.data(), bytes); + } else if (ctx->conn != nullptr && ctx->conn->is_open()) { + gtx_msg_get_tensor_req req = {}; + req.tensor = gtx_serialize_tensor(tensor); + req.offset = 0; + req.size = (uint32_t) bytes; + ok = ctx->conn->send_data(GTX_BAR2_STAGING_OFFSET, &req, sizeof(req)) && + ctx->conn->send_cmd_staged(GTX_CMD_GET_TENSOR, + GTX_BAR2_DEVICE_BASE + GTX_BAR2_STAGING_OFFSET, + (uint32_t) sizeof(req), nullptr, 0) && + ctx->conn->recv_data(GTX_BAR2_STAGING_OFFSET, tmp.data(), bytes); + } + if (!ok) { + return false; + } + + out.resize(elems); + ggml_fp16_to_fp32_row(tmp.data(), out.data(), (int64_t) elems); + return true; +} + +static bool gtx_download_f16_bytes_as_f32(ggml_backend_gtx_context * ctx, + uint64_t remote, + size_t elems, + float * out) { + if (ctx == nullptr || remote == 0 || elems == 0 || out == nullptr) { + return false; + } + + std::vector tmp(elems); + const size_t bytes = tmp.size() * sizeof(ggml_fp16_t); + bool ok = false; + if (ctx->use_uart) { + ok = ctx->uart != nullptr && ctx->uart->download_bytes(remote, tmp.data(), bytes); + } else if (ctx->conn != nullptr && ctx->conn->is_open()) { + ok = ctx->conn->recv_data(remote, tmp.data(), bytes); + } + if (!ok) { + return false; + } + + ggml_fp16_to_fp32_row(tmp.data(), out, (int64_t) elems); + return true; +} + +static bool gtx_download_tensor_strided_f16_as_f32(ggml_backend_gtx_context * ctx, + const ggml_tensor * tensor, + std::vector & out) { + out.clear(); + if (ctx == nullptr || + tensor == nullptr || + gtx_remote_tensor_type(tensor) != GGML_TYPE_F16 || + tensor->ne[0] <= 0 || + tensor->ne[1] <= 0 || + tensor->ne[2] <= 0 || + tensor->ne[3] <= 0) { + return false; + } + + const uint64_t max_elems = gtx_cpu_ref_enabled() ? + gtx_env_u64_or_default("GTX_CPU_REF_MAX_ELEMS", 32ull * 1024ull * 1024ull) : + gtx_env_u64_or_default("GTX_CHECK_MUL_MAT_MAX_ELEMS", 4ull * 1024ull * 1024ull); + const uint64_t elems64 = (uint64_t) ggml_nelements(tensor); + if (max_elems != 0 && elems64 > max_elems) { + return false; + } + + const uint64_t base = gtx_remote_data_ptr(tensor); + const gtx_rpc_tensor rpc = gtx_serialize_tensor(tensor); + out.resize((size_t) elems64); + + for (int64_t i3 = 0; i3 < tensor->ne[3]; ++i3) { + for (int64_t i2 = 0; i2 < tensor->ne[2]; ++i2) { + for (int64_t i1 = 0; i1 < tensor->ne[1]; ++i1) { + const uint64_t remote = base + + (uint64_t) i3 * rpc.nb[3] + + (uint64_t) i2 * rpc.nb[2] + + (uint64_t) i1 * rpc.nb[1]; + const size_t dst_off = + (((size_t) i3 * (size_t) tensor->ne[2] + (size_t) i2) * + (size_t) tensor->ne[1] + (size_t) i1) * + (size_t) tensor->ne[0]; + if (!gtx_download_f16_bytes_as_f32(ctx, remote, (size_t) tensor->ne[0], out.data() + dst_off)) { + out.clear(); + return false; + } + } + } + } + return true; +} + +static inline float gtx_tensor4_value(const std::vector & data, + const ggml_tensor * tensor, + int64_t i0, + int64_t i1, + int64_t i2, + int64_t i3) { + const size_t idx = + (((size_t) i3 * (size_t) tensor->ne[2] + (size_t) i2) * + (size_t) tensor->ne[1] + (size_t) i1) * + (size_t) tensor->ne[0] + (size_t) i0; + return data[idx]; +} + +static void gtx_check_mul_mat_after_launch(ggml_backend_gtx_context * ctx, + const ggml_tensor * node) { + if (!gtx_check_mul_mat_enabled() || + ctx == nullptr || + node == nullptr || + gtx_normalize_host_op(node) != (uint32_t) GGML_OP_MUL_MAT || + node->src[0] == nullptr || + node->src[1] == nullptr) { + return; + } + + const char * filter = getenv("GTX_CHECK_MUL_MAT_FILTER"); + if (filter != nullptr && filter[0] != '\0') { + const char * node_name = node->name[0] != '\0' ? node->name : ""; + const char * src0_name = node->src[0]->name[0] != '\0' ? node->src[0]->name : ""; + const char * src1_name = node->src[1]->name[0] != '\0' ? node->src[1]->name : ""; + if (strstr(node_name, filter) == nullptr && + strstr(src0_name, filter) == nullptr && + strstr(src1_name, filter) == nullptr) { + return; + } + } + + static uint64_t checked_nodes = 0; + const uint64_t max_nodes = gtx_env_u64_or_default("GTX_CHECK_MUL_MAT_MAX_NODES", 256); + if (max_nodes != 0 && checked_nodes >= max_nodes) { + return; + } + checked_nodes++; + + const ggml_tensor * src0 = node->src[0]; + const ggml_tensor * src1 = node->src[1]; + const int64_t k = src0->ne[0]; + const int64_t rows = src0->ne[1]; + const int64_t cols = src1->ne[1]; + const int64_t batch2 = node->ne[2]; + const int64_t batch3 = node->ne[3]; + if (k <= 0 || rows <= 0 || cols <= 0 || + batch2 <= 0 || batch3 <= 0 || + src1->ne[0] != k || + node->ne[0] != rows || + node->ne[1] != cols || + src1->ne[2] != batch2 || src1->ne[3] != batch3 || + src0->ne[2] <= 0 || src0->ne[3] <= 0 || + (batch2 % src0->ne[2]) != 0 || + (batch3 % src0->ne[3]) != 0) { + fprintf(stderr, + "[GTX MUL_MAT CHECK] skip tensor=%s reason=unsupported-shape dst=[%lld,%lld,%lld,%lld] src0=[%lld,%lld,%lld,%lld] src1=[%lld,%lld,%lld,%lld]\n", + node->name[0] != '\0' ? node->name : "(anon)", + (long long) node->ne[0], (long long) node->ne[1], (long long) node->ne[2], (long long) node->ne[3], + (long long) src0->ne[0], (long long) src0->ne[1], (long long) src0->ne[2], (long long) src0->ne[3], + (long long) src1->ne[0], (long long) src1->ne[1], (long long) src1->ne[2], (long long) src1->ne[3]); + return; + } + + std::vector a; + std::vector b; + std::vector dst; + if (!gtx_download_tensor_strided_f16_as_f32(ctx, src0, a) || + !gtx_download_tensor_strided_f16_as_f32(ctx, src1, b) || + !gtx_download_tensor_strided_f16_as_f32(ctx, node, dst)) { + fprintf(stderr, + "[GTX MUL_MAT CHECK] skip tensor=%s reason=download-failed-or-too-large\n", + node->name[0] != '\0' ? node->name : "(anon)"); + return; + } + + const int64_t r2 = batch2 / src0->ne[2]; + const int64_t r3 = batch3 / src0->ne[3]; + const int64_t max_rows = (int64_t) gtx_env_u64_or_default("GTX_CHECK_MUL_MAT_ROWS", 128); + const int64_t row_limit = max_rows <= 0 ? rows : std::min(rows, max_rows); + const int64_t col_limit = cols; + float max_diff_f32 = 0.0f; + float max_diff_f16 = 0.0f; + int64_t max_row = 0; + int64_t max_col = 0; + float max_ref = 0.0f; + float max_ref_f16 = 0.0f; + float max_got = 0.0f; + + for (int64_t i3 = 0; i3 < batch3; ++i3) { + for (int64_t i2 = 0; i2 < batch2; ++i2) { + const int64_t src0_i2 = i2 / r2; + const int64_t src0_i3 = i3 / r3; + for (int64_t s = 0; s < row_limit; ++s) { + const int64_t r = row_limit == rows || row_limit <= 1 ? + s : + (s * (rows - 1)) / (row_limit - 1); + for (int64_t c = 0; c < col_limit; ++c) { + float acc = 0.0f; + for (int64_t i = 0; i < k; ++i) { + acc += gtx_tensor4_value(a, src0, i, r, src0_i2, src0_i3) * + gtx_tensor4_value(b, src1, i, c, i2, i3); + } + const float acc_f16 = gtx_fp16_to_fp32(gtx_fp32_to_fp16(acc)); + const float got = gtx_tensor4_value(dst, node, r, c, i2, i3); + const float diff_f32 = fabsf(got - acc); + const float diff_f16 = fabsf(got - acc_f16); + if (diff_f16 > max_diff_f16) { + max_diff_f16 = diff_f16; + max_diff_f32 = diff_f32; + max_row = r; + max_col = c; + max_ref = acc; + max_ref_f16 = acc_f16; + max_got = got; + } + } + } + } + } + + fprintf(stderr, + "[GTX MUL_MAT CHECK] tensor=%s src0=%s src1=%s dst=[%lld,%lld,%lld,%lld] k=%lld checked_rows=%lld cols=%lld max_diff_f16=%.9g max_diff_f32=%.9g at=[%lld,%lld] ref=%.9g ref_f16=%.9g got=%.9g\n", + node->name[0] != '\0' ? node->name : "(anon)", + src0->name[0] != '\0' ? src0->name : "(anon)", + src1->name[0] != '\0' ? src1->name : "(anon)", + (long long) node->ne[0], (long long) node->ne[1], + (long long) node->ne[2], (long long) node->ne[3], + (long long) k, + (long long) row_limit, + (long long) col_limit, + max_diff_f16, + max_diff_f32, + (long long) max_row, + (long long) max_col, + max_ref, + max_ref_f16, + max_got); +} + +static bool gtx_check_ops_should_check(const ggml_tensor * node) { + if (!gtx_check_ops_enabled() || node == nullptr) { + return false; + } + + const uint32_t normalized_op = gtx_normalize_host_op(node); + const char * op_name = gtx_normalized_op_name(normalized_op); + const char * value = getenv("GTX_CHECK_OPS"); + const bool all_default = value != nullptr && + (strcmp(value, "1") == 0 || + strcmp(value, "true") == 0 || + strcmp(value, "TRUE") == 0 || + strcmp(value, "on") == 0 || + strcmp(value, "ON") == 0 || + strcmp(value, "all") == 0 || + strcmp(value, "ALL") == 0); + const bool supported = normalized_op == (uint32_t) GGML_OP_NORM || + normalized_op == (uint32_t) GGML_OP_SOFT_MAX; + return supported && (all_default || gtx_env_list_contains("GTX_CHECK_OPS", op_name)); +} + +static bool gtx_check_ops_download(ggml_backend_gtx_context * ctx, + const ggml_tensor * tensor, + std::vector & out, + const char * tag, + const char * node_name) { + const uint64_t max_elems = gtx_env_u64_or_default("GTX_CHECK_OP_MAX_ELEMS", 4ull * 1024ull * 1024ull); + const uint64_t elems = tensor != nullptr ? (uint64_t) ggml_nelements(tensor) : 0; + if (max_elems != 0 && elems > max_elems) { + fprintf(stderr, + "[GTX OP CHECK] skip tensor=%s reason=%s-too-large elems=%llu max=%llu\n", + node_name != nullptr ? node_name : "(anon)", + tag != nullptr ? tag : "tensor", + (unsigned long long) elems, + (unsigned long long) max_elems); + return false; + } + return gtx_download_tensor_strided_f16_as_f32(ctx, tensor, out); +} + +struct gtx_op_check_pre_state { + bool valid = false; + uint32_t normalized_op = 0; + std::vector src0; + std::vector src1; +}; + +static gtx_op_check_pre_state gtx_check_ops_before_launch(ggml_backend_gtx_context * ctx, + const ggml_tensor * node) { + gtx_op_check_pre_state pre; + if (!gtx_check_ops_should_check(node) || node == nullptr || node->src[0] == nullptr) { + return pre; + } + + pre.normalized_op = gtx_normalize_host_op(node); + const char * name = node->name[0] != '\0' ? node->name : "(anon)"; + if (!gtx_check_ops_download(ctx, node->src[0], pre.src0, "src0", name)) { + fprintf(stderr, "[GTX OP CHECK] skip %s tensor=%s reason=pre-src0-download-failed-or-too-large\n", + gtx_normalized_op_name(pre.normalized_op), name); + pre.src0.clear(); + return pre; + } + if (pre.normalized_op == (uint32_t) GGML_OP_SOFT_MAX && node->src[1] != nullptr && + !gtx_check_ops_download(ctx, node->src[1], pre.src1, "mask", name)) { + fprintf(stderr, "[GTX OP CHECK] skip SOFT_MAX tensor=%s reason=pre-mask-download-failed-or-too-large\n", + name); + pre.src0.clear(); + pre.src1.clear(); + return pre; + } + pre.valid = true; + return pre; +} + +static void gtx_check_norm_after_launch(ggml_backend_gtx_context * ctx, + const ggml_tensor * node, + const gtx_op_check_pre_state & pre) { + if (node == nullptr || node->src[0] == nullptr) { + return; + } + + std::vector dst; + const char * name = node->name[0] != '\0' ? node->name : "(anon)"; + if (!pre.valid || + pre.normalized_op != (uint32_t) GGML_OP_NORM || + !gtx_check_ops_download(ctx, node, dst, "dst", name)) { + fprintf(stderr, "[GTX OP CHECK] skip NORM tensor=%s reason=download-failed-or-too-large\n", name); + return; + } + const std::vector & src = pre.src0; + + const int64_t cols = node->ne[0]; + const int64_t rows = node->ne[1] * node->ne[2] * node->ne[3]; + if (cols <= 0 || rows <= 0 || src.size() != dst.size()) { + return; + } + + const float eps = gtx_float_from_bits((uint32_t) node->op_params[0]); + float max_diff = 0.0f; + float max_diff_f16 = 0.0f; + size_t max_idx = 0; + float max_ref = 0.0f; + float max_ref_f16 = 0.0f; + float max_got = 0.0f; + + for (int64_t row = 0; row < rows; ++row) { + const size_t base = (size_t) row * (size_t) cols; + double mean = 0.0; + for (int64_t col = 0; col < cols; ++col) { + mean += (double) src[base + (size_t) col]; + } + mean /= (double) cols; + + double var = 0.0; + for (int64_t col = 0; col < cols; ++col) { + const double d = (double) src[base + (size_t) col] - mean; + var += d * d; + } + var /= (double) cols; + const float inv_std = 1.0f / sqrtf((float) var + eps); + + for (int64_t col = 0; col < cols; ++col) { + const size_t idx = base + (size_t) col; + const float ref = (src[idx] - (float) mean) * inv_std; + const float ref_f16 = gtx_fp16_to_fp32(gtx_fp32_to_fp16(ref)); + const float got = dst[idx]; + const float diff = fabsf(got - ref); + const float diff_f16 = fabsf(got - ref_f16); + if (diff_f16 > max_diff_f16) { + max_diff = diff; + max_diff_f16 = diff_f16; + max_idx = idx; + max_ref = ref; + max_ref_f16 = ref_f16; + max_got = got; + } + } + } + + fprintf(stderr, + "[GTX OP CHECK] NORM tensor=%s shape=[%lld,%lld,%lld,%lld] eps=%.9g max_diff_f16=%.9g max_diff=%.9g idx=%zu ref=%.9g ref_f16=%.9g got=%.9g\n", + name, + (long long) node->ne[0], (long long) node->ne[1], + (long long) node->ne[2], (long long) node->ne[3], + eps, + max_diff_f16, + max_diff, + max_idx, + max_ref, + max_ref_f16, + max_got); +} + +static void gtx_check_softmax_after_launch(ggml_backend_gtx_context * ctx, + const ggml_tensor * node, + const gtx_op_check_pre_state & pre) { + if (node == nullptr || node->src[0] == nullptr) { + return; + } + + std::vector dst; + const char * name = node->name[0] != '\0' ? node->name : "(anon)"; + if (!pre.valid || + pre.normalized_op != (uint32_t) GGML_OP_SOFT_MAX || + !gtx_check_ops_download(ctx, node, dst, "dst", name)) { + fprintf(stderr, "[GTX OP CHECK] skip SOFT_MAX tensor=%s reason=download-failed-or-too-large\n", name); + return; + } + const std::vector & src = pre.src0; + const std::vector & mask = pre.src1; + + const int64_t ne0 = node->ne[0]; + const int64_t ne1 = node->ne[1]; + const int64_t ne2 = node->ne[2]; + const int64_t ne3 = node->ne[3]; + if (ne0 <= 0 || ne1 <= 0 || ne2 <= 0 || ne3 <= 0 || src.size() != dst.size()) { + return; + } + + const float scale = gtx_float_from_bits((uint32_t) node->op_params[0]); + const int64_t mne2 = node->src[1] != nullptr ? node->src[1]->ne[2] : 1; + const int64_t mne3 = node->src[1] != nullptr ? node->src[1]->ne[3] : 1; + float max_diff = 0.0f; + float max_diff_f16 = 0.0f; + size_t max_idx = 0; + float max_ref = 0.0f; + float max_ref_f16 = 0.0f; + float max_got = 0.0f; + + std::vector tmp((size_t) ne0); + for (int64_t i3 = 0; i3 < ne3; ++i3) { + for (int64_t i2 = 0; i2 < ne2; ++i2) { + for (int64_t i1 = 0; i1 < ne1; ++i1) { + const size_t base = (((size_t) i3 * (size_t) ne2 + (size_t) i2) * + (size_t) ne1 + (size_t) i1) * + (size_t) ne0; + const int64_t mi2 = mne2 > 0 ? i2 % mne2 : 0; + const int64_t mi3 = mne3 > 0 ? i3 % mne3 : 0; + const size_t mask_base = node->src[1] != nullptr ? + (((size_t) mi3 * (size_t) node->src[1]->ne[2] + (size_t) mi2) * + (size_t) node->src[1]->ne[1] + (size_t) i1) * + (size_t) node->src[1]->ne[0] : 0; + + float row_max = -std::numeric_limits::infinity(); + for (int64_t i0 = 0; i0 < ne0; ++i0) { + float v = src[base + (size_t) i0] * scale; + if (node->src[1] != nullptr) { + v += mask[mask_base + (size_t) i0]; + } + tmp[(size_t) i0] = v; + row_max = std::max(row_max, v); + } + + float sum = 0.0f; + for (int64_t i0 = 0; i0 < ne0; ++i0) { + const float e = expf(tmp[(size_t) i0] - row_max); + tmp[(size_t) i0] = e; + sum += e; + } + const float inv_sum = sum > 0.0f ? 1.0f / sum : 0.0f; + for (int64_t i0 = 0; i0 < ne0; ++i0) { + const size_t idx = base + (size_t) i0; + const float ref = tmp[(size_t) i0] * inv_sum; + const float ref_f16 = gtx_fp16_to_fp32(gtx_fp32_to_fp16(ref)); + const float got = dst[idx]; + const float diff = fabsf(got - ref); + const float diff_f16 = fabsf(got - ref_f16); + if (diff_f16 > max_diff_f16) { + max_diff = diff; + max_diff_f16 = diff_f16; + max_idx = idx; + max_ref = ref; + max_ref_f16 = ref_f16; + max_got = got; + } + } + } + } + } + + fprintf(stderr, + "[GTX OP CHECK] SOFT_MAX tensor=%s shape=[%lld,%lld,%lld,%lld] scale=%.9g mask=%d max_diff_f16=%.9g max_diff=%.9g idx=%zu ref=%.9g ref_f16=%.9g got=%.9g\n", + name, + (long long) ne0, (long long) ne1, (long long) ne2, (long long) ne3, + scale, + node->src[1] != nullptr ? 1 : 0, + max_diff_f16, + max_diff, + max_idx, + max_ref, + max_ref_f16, + max_got); +} + +static void gtx_check_ops_after_launch(ggml_backend_gtx_context * ctx, + const ggml_tensor * node, + const gtx_op_check_pre_state & pre) { + if (!gtx_check_ops_should_check(node)) { + return; + } + + static uint64_t checked_nodes = 0; + const uint64_t max_nodes = gtx_env_u64_or_default("GTX_CHECK_OP_MAX_NODES", 256); + if (max_nodes != 0 && checked_nodes >= max_nodes) { + return; + } + checked_nodes++; + + const uint32_t normalized_op = gtx_normalize_host_op(node); + if (normalized_op == (uint32_t) GGML_OP_NORM) { + gtx_check_norm_after_launch(ctx, node, pre); + } else if (normalized_op == (uint32_t) GGML_OP_SOFT_MAX) { + gtx_check_softmax_after_launch(ctx, node, pre); + } +} + +static void gtx_dump_op_sample_after_launch(ggml_backend_gtx_context * ctx, + const ggml_tensor * node) { + if (!gtx_dump_op_sample_should_dump(node)) { + return; + } + + gtx_dump_op_sample_tensor(ctx, node, "post-dst"); +} + +static void gtx_dump_op_sample_before_launch(ggml_backend_gtx_context * ctx, + const ggml_tensor * node) { + if (!gtx_dump_op_sample_should_dump(node)) { + return; + } + + gtx_dump_op_sample_tensor(ctx, node->src[0], "pre-src0"); + const uint32_t normalized_op = gtx_normalize_host_op(node); + if ((normalized_op == (uint32_t) GGML_OP_MUL_MAT || + normalized_op == (uint32_t) GGML_OP_ADD || + normalized_op == (uint32_t) GGML_OP_MUL) && + node->src[1] != nullptr) { + gtx_dump_op_sample_tensor(ctx, node->src[1], "pre-src1"); + } +} + +static void gtx_diagnostic_report_first_nonfinite(ggml_backend_gtx_context * ctx, + const ggml_tensor * node) { + if (!gtx_env_enabled("GTX_DIAGNOSTIC_FIRST_NONFINITE") || + ctx == nullptr || node == nullptr) { + return; + } + + static bool reported = false; + if (reported) { + return; + } + + const uint64_t limit_env = + gtx_env_u64_or_default("GTX_DIAGNOSTIC_NONFINITE_ELEMS", 4096); + const size_t limit = limit_env == 0 ? + (size_t) ggml_nelements(node) : + (size_t) std::min(limit_env, (uint64_t) ggml_nelements(node)); + std::vector values; + if (limit == 0 || + !gtx_download_tensor_prefix_as_f32(ctx, node, limit, values)) { + return; + } + + size_t first = values.size(); + for (size_t i = 0; i < values.size(); ++i) { + if (!std::isfinite(values[i])) { + first = i; + break; + } + } + if (first == values.size()) { + return; + } + + reported = true; + fprintf(stderr, + "[GTX FIRST NONFINITE] op=%s tensor=%s index=%zu value=%.9g " + "shape=[%lld,%lld,%lld,%lld] scanned=%zu/%lld\n", + gtx_tensor_op_display_name(node), + node->name[0] != '\0' ? node->name : "(anon)", + first, values[first], + (long long) node->ne[0], (long long) node->ne[1], + (long long) node->ne[2], (long long) node->ne[3], + values.size(), (long long) ggml_nelements(node)); + for (int i = 0; i < GGML_MAX_SRC; ++i) { + if (node->src[i] != nullptr) { + char tag[32]; + snprintf(tag, sizeof(tag), "first-bad-src%d", i); + gtx_dump_op_sample_tensor(ctx, node->src[i], tag); + } + } + gtx_dump_op_sample_tensor(ctx, node, "first-bad-dst"); + + const char * capture_prefix = getenv("GTX_DIAGNOSTIC_CAPTURE_PREFIX"); + if (capture_prefix == nullptr || capture_prefix[0] == '\0') { + return; + } + + char meta_path[1024]; + snprintf(meta_path, sizeof(meta_path), "%s-meta.txt", capture_prefix); + FILE * meta = fopen(meta_path, "w"); + if (meta != nullptr) { + fprintf(meta, + "op=%s\ntensor=%s\nshape=%lld,%lld,%lld,%lld\nfirst_nonfinite=%zu\n", + gtx_tensor_op_display_name(node), + node->name[0] != '\0' ? node->name : "(anon)", + (long long) node->ne[0], (long long) node->ne[1], + (long long) node->ne[2], (long long) node->ne[3], first); + } + + for (int i = 0; i < GGML_MAX_SRC; ++i) { + const ggml_tensor * src = node->src[i]; + if (src == nullptr) { + continue; + } + const size_t elems = (size_t) ggml_nelements(src); + std::vector captured; + if (!gtx_download_tensor_prefix_as_f32(ctx, src, elems, captured) || + captured.size() != elems) { + fprintf(stderr, + "[GTX CAPTURE] failed src%d tensor=%s elems=%zu\n", + i, src->name[0] != '\0' ? src->name : "(anon)", elems); + if (meta != nullptr) { + fprintf(meta, "src%d_status=download_failed\n", i); + } + continue; + } + + char data_path[1024]; + snprintf(data_path, sizeof(data_path), "%s-src%d.f32", capture_prefix, i); + FILE * data = fopen(data_path, "wb"); + const bool wrote = data != nullptr && + fwrite(captured.data(), sizeof(float), captured.size(), data) == captured.size(); + if (data != nullptr) { + fclose(data); + } + fprintf(stderr, + "[GTX CAPTURE] %s src%d tensor=%s shape=[%lld,%lld,%lld,%lld] " + "elems=%zu path=%s\n", + wrote ? "saved" : "failed", + i, src->name[0] != '\0' ? src->name : "(anon)", + (long long) src->ne[0], (long long) src->ne[1], + (long long) src->ne[2], (long long) src->ne[3], + captured.size(), data_path); + if (meta != nullptr) { + fprintf(meta, + "src%d_name=%s\nsrc%d_shape=%lld,%lld,%lld,%lld\n" + "src%d_graph_type=%s\nsrc%d_remote_type=%s\n" + "src%d_elems=%zu\nsrc%d_file=%s\nsrc%d_status=%s\n", + i, src->name[0] != '\0' ? src->name : "(anon)", + i, (long long) src->ne[0], (long long) src->ne[1], + (long long) src->ne[2], (long long) src->ne[3], + i, ggml_type_name(src->type), + i, ggml_type_name(gtx_remote_tensor_type(src)), + i, captured.size(), i, data_path, i, wrote ? "saved" : "write_failed"); + } + } + if (meta != nullptr) { + fclose(meta); + } +} + +static bool gtx_tiling_scheduler_diagnostics_active() { + return gtx_check_ops_enabled() || + gtx_check_mul_mat_enabled() || + gtx_dump_op_samples_enabled() || + gtx_env_enabled("GTX_DIAGNOSTIC_FIRST_NONFINITE") || + gtx_log_softmax_enabled(); +} + +static bool gtx_diagnostic_materialize_stash(ggml_backend_gtx_context * ctx, + const ggml_tensor * node) { + if (!gtx_env_enabled("GTX_DIAGNOSTIC_STASH_FLUSH_RESET")) { + return true; + } + if (ctx == nullptr || !ctx->use_uart || ctx->uart == nullptr) { + return true; + } + const bool ok = ctx->uart->stash_flush_reset(); + if (!ok || gtx_log_ops_enabled()) { + fprintf(stderr, "[GTX DIAG STASH] flush_reset %s tensor=%s op=%s\n", + ok ? "done" : "failed", + node != nullptr && node->name[0] != '\0' ? node->name : "(anon)", + node != nullptr ? gtx_tensor_op_display_name(node) : "-"); + } + return ok; +} + +static bool gtx_scheduler_node_skippable(const ggml_tensor * node) { + return node == nullptr || + gtx_is_metadata_op(node->op) || + ggml_nelements(node) == 0; +} + +static bool gtx_scheduler_dense_f16_2d(const ggml_tensor * tensor) { + return gtx_tensor_remote_f16_dense(tensor) && + tensor->ne[0] > 0 && + tensor->ne[1] > 0 && + tensor->ne[2] == 1 && + tensor->ne[3] == 1 && + tensor->ne[0] <= UINT32_MAX && + tensor->ne[1] <= UINT32_MAX; +} + +static bool gtx_scheduler_same_2d_shape(const ggml_tensor * a, const ggml_tensor * b) { + return a != nullptr && b != nullptr && + a->ne[0] == b->ne[0] && + a->ne[1] == b->ne[1] && + a->ne[2] == 1 && + a->ne[3] == 1 && + b->ne[2] == 1 && + b->ne[3] == 1; +} + +struct gtx_scheduler_col_tile_shape { + uint32_t i1; + uint32_t i2; + uint32_t i3; + uint32_t ne1; + uint32_t ne2; + uint32_t ne3; +}; + +static bool gtx_scheduler_make_col_tile_shape(const ggml_tensor * node, + uint32_t col0, + uint32_t cols, + gtx_scheduler_col_tile_shape & shape) { + if (node == nullptr || + cols == 0 || + node->ne[1] <= 0 || + node->ne[2] <= 0 || + node->ne[3] <= 0 || + node->ne[1] > UINT32_MAX || + node->ne[2] > UINT32_MAX || + node->ne[3] > UINT32_MAX) { + return false; + } + + const uint32_t ne1 = (uint32_t)node->ne[1]; + const uint32_t ne2 = (uint32_t)node->ne[2]; + const uint32_t i1 = col0 % ne1; + const uint32_t tmp = col0 / ne1; + const uint32_t i2 = tmp % ne2; + const uint32_t i3 = tmp / ne2; + + if (i1 + cols <= ne1) { + shape = { i1, i2, i3, cols, 1u, 1u }; + return true; + } + + if (i1 == 0 && cols % ne1 == 0) { + const uint32_t tile_ne2 = cols / ne1; + if (tile_ne2 > 0 && i2 + tile_ne2 <= ne2) { + shape = { i1, i2, i3, ne1, tile_ne2, 1u }; + return true; + } + } + + return false; +} + +static uint64_t gtx_scheduler_col_offset(const ggml_tensor * tensor, + const gtx_scheduler_col_tile_shape & shape) { + return (uint64_t)shape.i1 * gtx_remote_byte_offset(tensor, tensor->nb[1]) + + (uint64_t)shape.i2 * gtx_remote_byte_offset(tensor, tensor->nb[2]) + + (uint64_t)shape.i3 * gtx_remote_byte_offset(tensor, tensor->nb[3]); +} + +static bool gtx_scheduler_col_span_contiguous(const ggml_tensor * tensor, + uint32_t col0, + uint32_t cols) { + gtx_scheduler_col_tile_shape shape = {}; + if (!gtx_scheduler_make_col_tile_shape(tensor, col0, cols, shape) || + !gtx_tensor_remote_f16_vector_contiguous(tensor)) { + return false; + } + + const uint64_t row_bytes = (uint64_t)tensor->ne[0] * sizeof(uint16_t); + return (shape.ne1 <= 1u || + gtx_remote_byte_offset(tensor, tensor->nb[1]) == row_bytes) && + (shape.ne2 <= 1u || + gtx_remote_byte_offset(tensor, tensor->nb[2]) == row_bytes * (uint64_t)tensor->ne[1]) && + (shape.ne3 <= 1u || + gtx_remote_byte_offset(tensor, tensor->nb[3]) == row_bytes * (uint64_t)tensor->ne[1] * (uint64_t)tensor->ne[2]); +} + +static bool gtx_scheduler_col_span_strided_rows_supported(const ggml_tensor * tensor, + uint32_t col0, + uint32_t cols) { + gtx_scheduler_col_tile_shape shape = {}; + if (!gtx_scheduler_make_col_tile_shape(tensor, col0, cols, shape) || + !gtx_tensor_remote_f16_element_contiguous(tensor)) { + return false; + } + + const uint64_t nb1 = gtx_remote_byte_offset(tensor, tensor->nb[1]); + const uint64_t nb2 = gtx_remote_byte_offset(tensor, tensor->nb[2]); + const uint64_t nb3 = gtx_remote_byte_offset(tensor, tensor->nb[3]); + return nb1 > 0 && nb1 <= UINT32_MAX && + nb2 <= UINT32_MAX && + nb3 <= UINT32_MAX; +} + +static void gtx_scheduler_make_dense_tile_shape_4d(ggml_gtx_kernel_launch_desc_t & desc, + uint32_t ne0, + uint32_t ne1, + uint32_t ne2, + uint32_t ne3) { + desc.ne[0] = ne0; + desc.ne[1] = ne1; + desc.ne[2] = ne2; + desc.ne[3] = ne3; + desc.nb[0] = sizeof(uint16_t); + desc.nb[1] = ne0 * sizeof(uint16_t); + desc.nb[2] = desc.nb[1] * ne1; + desc.nb[3] = desc.nb[2] * ne2; +} + +static void gtx_scheduler_make_strided_dst_tile_shape_4d( + ggml_gtx_kernel_launch_desc_t & desc, + const ggml_tensor * dst, + uint32_t ne0, + uint32_t ne1, + uint32_t ne2, + uint32_t ne3) { + desc.ne[0] = ne0; + desc.ne[1] = ne1; + desc.ne[2] = ne2; + desc.ne[3] = ne3; + if (dst != nullptr) { + desc.nb[0] = (uint32_t)gtx_remote_byte_offset(dst, dst->nb[0]); + desc.nb[1] = (uint32_t)gtx_remote_byte_offset(dst, dst->nb[1]); + desc.nb[2] = (uint32_t)gtx_remote_byte_offset(dst, dst->nb[2]); + desc.nb[3] = (uint32_t)gtx_remote_byte_offset(dst, dst->nb[3]); + } +} + +static void gtx_scheduler_make_dense_src_tile_shape_4d(ggml_gtx_kernel_launch_desc_t & desc, + uint32_t src_idx, + uint32_t ne0, + uint32_t ne1, + uint32_t ne2, + uint32_t ne3) { + desc.src_ne[src_idx][0] = ne0; + desc.src_ne[src_idx][1] = ne1; + desc.src_ne[src_idx][2] = ne2; + desc.src_ne[src_idx][3] = ne3; + desc.src_nb[src_idx][0] = sizeof(uint16_t); + desc.src_nb[src_idx][1] = ne0 * sizeof(uint16_t); + desc.src_nb[src_idx][2] = desc.src_nb[src_idx][1] * ne1; + desc.src_nb[src_idx][3] = desc.src_nb[src_idx][2] * ne2; +} + +static void gtx_scheduler_make_strided_src_tile_shape_4d(ggml_gtx_kernel_launch_desc_t & desc, + uint32_t src_idx, + const ggml_tensor * src, + uint32_t ne0, + uint32_t ne1, + uint32_t ne2, + uint32_t ne3) { + desc.src_ne[src_idx][0] = ne0; + desc.src_ne[src_idx][1] = ne1; + desc.src_ne[src_idx][2] = ne2; + desc.src_ne[src_idx][3] = ne3; + if (src != nullptr) { + desc.src_nb[src_idx][0] = (uint32_t)gtx_remote_byte_offset(src, src->nb[0]); + desc.src_nb[src_idx][1] = (uint32_t)gtx_remote_byte_offset(src, src->nb[1]); + desc.src_nb[src_idx][2] = (uint32_t)gtx_remote_byte_offset(src, src->nb[2]); + desc.src_nb[src_idx][3] = (uint32_t)gtx_remote_byte_offset(src, src->nb[3]); + } +} + +static bool gtx_scheduler_tileable_row_node(const ggml_tensor * node) { + if (!gtx_scheduler_dense_f16_2d(node)) { + return false; + } + + const gtx_l2_chainability chain = gtx_l2_chainability_from_node(node); + if (chain.kind == gtx_l2_chain_kind::producer) { + // Materialization requires a host-provided strided-copy access plan. + return false; + } + if (chain.kind != gtx_l2_chain_kind::native) { + return false; + } + + const uint32_t kernel_id = chain.kernel_id; + if (kernel_id == GGML_GTX_KERNEL_MUL_MAT_F16) { + return false; + } + + if (kernel_id == GGML_GTX_KERNEL_SOFT_MAX_F16) { + return false; + } + if (kernel_id == GGML_GTX_KERNEL_RMS_NORM_F16) { + return gtx_l2_rms_norm_shape_ok(node) && + gtx_scheduler_same_2d_shape(node, node->src[0]); + } + if (kernel_id == GGML_GTX_KERNEL_SWIGLU_F16) { + return false; + } + + if (node->src[0] == nullptr || + !gtx_scheduler_same_2d_shape(node, node->src[0])) { + return false; + } + + if (kernel_id == GGML_GTX_KERNEL_ADD_F16 || + kernel_id == GGML_GTX_KERNEL_MUL_F16) { + if (node->src[1] == nullptr) { + return false; + } + if (gtx_scheduler_same_2d_shape(node, node->src[1])) { + return true; + } + return false; + } + + return kernel_id == GGML_GTX_KERNEL_SCALE_F16 || + kernel_id == GGML_GTX_KERNEL_GELU_F16 || + kernel_id == GGML_GTX_KERNEL_ABS_F16 || + kernel_id == GGML_GTX_KERNEL_NEG_F16 || + kernel_id == GGML_GTX_KERNEL_SGN_F16 || + kernel_id == GGML_GTX_KERNEL_CEIL_F16 || + kernel_id == GGML_GTX_KERNEL_FLOOR_F16 || + kernel_id == GGML_GTX_KERNEL_TANH_F16 || + kernel_id == GGML_GTX_KERNEL_SILU_F16 || + kernel_id == GGML_GTX_KERNEL_RELU_F16 || + kernel_id == GGML_GTX_KERNEL_LEAKY_RELU_F16 || + kernel_id == GGML_GTX_KERNEL_CLAMP_F16; +} + +static uint64_t gtx_scheduler_row_offset(const ggml_tensor * tensor, uint32_t row0) { + return (uint64_t)row0 * gtx_remote_byte_offset(tensor, tensor->nb[1]); +} + +static void gtx_scheduler_make_dense_tile_shape(ggml_gtx_kernel_launch_desc_t & desc, + uint32_t ne0, + uint32_t rows) { + gtx_scheduler_make_dense_tile_shape_4d(desc, ne0, rows, 1u, 1u); +} + +static void gtx_scheduler_make_dense_src_tile_shape(ggml_gtx_kernel_launch_desc_t & desc, + uint32_t src_idx, + uint32_t ne0, + uint32_t rows) { + gtx_scheduler_make_dense_src_tile_shape_4d(desc, src_idx, ne0, rows, 1u, 1u); +} + +// These uploaded L2 kernels consume a single flat vector. Keep this ABI +// adaptation host-side: firmware receives an already-final raw tile and never +// derives it from the ggml operation. +static bool gtx_scheduler_row_tile_kernel_is_flat(uint32_t kernel_id) { + switch (kernel_id) { + case GGML_GTX_KERNEL_ADD_F16: + case GGML_GTX_KERNEL_MUL_F16: + case GGML_GTX_KERNEL_SCALE_F16: + case GGML_GTX_KERNEL_GELU_F16: + case GGML_GTX_KERNEL_ABS_F16: + case GGML_GTX_KERNEL_NEG_F16: + case GGML_GTX_KERNEL_SGN_F16: + case GGML_GTX_KERNEL_CEIL_F16: + case GGML_GTX_KERNEL_FLOOR_F16: + case GGML_GTX_KERNEL_TANH_F16: + case GGML_GTX_KERNEL_SILU_F16: + case GGML_GTX_KERNEL_RELU_F16: + case GGML_GTX_KERNEL_LEAKY_RELU_F16: + case GGML_GTX_KERNEL_CLAMP_F16: + return true; + default: + return false; + } +} + +static bool gtx_scheduler_build_row_tile_desc(const ggml_tensor * node, + uint32_t row0, + uint32_t rows, + ggml_gtx_kernel_launch_desc_t & tile) { + if (!gtx_build_kernel_launch_desc(node, tile) || + node->ne[0] <= 0 || + node->ne[0] > UINT32_MAX || + rows == 0) { + return false; + } + + const uint32_t ne0 = (uint32_t)node->ne[0]; + tile.dst_addr += gtx_scheduler_row_offset(node, row0); + gtx_scheduler_make_dense_tile_shape(tile, ne0, rows); + + if (gtx_kernel_id_from_node(node) == GGML_GTX_KERNEL_SWIGLU_F16) { + const ggml_tensor * src = node->src[0]; + const uint32_t src_ne0 = ne0 * 2u; + tile.src_addr[0] += gtx_scheduler_row_offset(src, row0); + gtx_scheduler_make_dense_src_tile_shape(tile, 0, src_ne0, rows); + tile.flags |= GGML_GTX_DESC_FLAG_STASH_CHAIN; + return true; + } + + for (uint32_t i = 0; i < 4; ++i) { + const ggml_tensor * src = node->src[i]; + if (src == nullptr || tile.src_addr[i] == 0) { + continue; + } + if (gtx_scheduler_same_2d_shape(node, src)) { + tile.src_addr[i] += gtx_scheduler_row_offset(src, row0); + gtx_scheduler_make_dense_src_tile_shape(tile, i, ne0, rows); + } + } + + const uint32_t kernel_id = gtx_kernel_id_from_node(node); + if (gtx_scheduler_row_tile_kernel_is_flat(kernel_id)) { + const uint64_t tile_elems = (uint64_t)ne0 * rows; + if (tile_elems == 0u || tile_elems > UINT32_MAX) { + return false; + } + gtx_scheduler_make_dense_tile_shape(tile, (uint32_t)tile_elems, 1u); + for (uint32_t i = 0; i < 4; ++i) { + if (node->src[i] != nullptr && tile.src_addr[i] != 0u) { + gtx_scheduler_make_dense_src_tile_shape(tile, i, + (uint32_t)tile_elems, 1u); + } + } + } + return true; +} + +static bool gtx_compare_cpu_reference_after_launch(ggml_backend_gtx_context * ctx, + const ggml_tensor * node, + int node_index) { + if (gtx_cpu_ref_first_divergence_found && + gtx_env_truthy("GTX_CPU_REF_STOP_AFTER_DIVERGENCE")) { + return true; + } + char path[1024]; + if (!gtx_cpu_ref_path(path, sizeof(path), node_index)) { + return true; + } + FILE * file = fopen(path, "rb"); + if (file == nullptr) { + return true; + } + + const size_t elems = (size_t)ggml_nelements(node); + std::vector reference(elems); + const size_t got = fread(reference.data(), sizeof(float), elems, file); + const int trailing = fgetc(file); + fclose(file); + if (got != elems || trailing != EOF) { + fprintf(stderr, + "[GTX CPU REF] graph=%llu node=%d tensor=%s invalid_reference_size path=%s expected_elems=%zu got=%zu\n", + (unsigned long long)gtx_cpu_ref_current_graph, node_index, + node->name[0] != '\0' ? node->name : "(anon)", path, elems, got); + return false; + } + + std::vector actual; + if (!gtx_download_tensor_strided_f16_as_f32(ctx, node, actual) || actual.size() != elems) { + fprintf(stderr, + "[GTX CPU REF] graph=%llu node=%d tensor=%s download_failed elems=%zu\n", + (unsigned long long)gtx_cpu_ref_current_graph, node_index, + node->name[0] != '\0' ? node->name : "(anon)", elems); + return false; + } + + float max_diff = 0.0f; + size_t max_index = 0; + double diff_sumsq = 0.0; + double ref_sumsq = 0.0; + size_t nonfinite = 0; + for (size_t i = 0; i < elems; ++i) { + const float ref = gtx_fp16_to_fp32(gtx_fp32_to_fp16(reference[i])); + reference[i] = ref; + const float value = actual[i]; + if (!std::isfinite(ref) || !std::isfinite(value)) { + ++nonfinite; + max_diff = INFINITY; + max_index = i; + continue; + } + const float diff = fabsf(value - ref); + diff_sumsq += (double)diff * (double)diff; + ref_sumsq += (double)ref * (double)ref; + if (diff > max_diff) { + max_diff = diff; + max_index = i; + } + } + const double rmse = elems != 0 ? sqrt(diff_sumsq / (double)elems) : 0.0; + const double ref_rms = elems != 0 ? sqrt(ref_sumsq / (double)elems) : 0.0; + const double scale = std::max(ref_rms, 1.0e-6); + const double max_rel_rms = (double)max_diff / scale; + const double rmse_rel = rmse / scale; + const float abs_threshold = gtx_env_float_or_default("GTX_CPU_REF_THRESHOLD", 0.02f); + const float max_rel_threshold = gtx_env_float_or_default("GTX_CPU_REF_MAX_REL_RMS", 0.05f); + const float rmse_rel_threshold = gtx_env_float_or_default("GTX_CPU_REF_RMSE_REL", 0.01f); + const bool diverged = nonfinite != 0 || + (max_diff > abs_threshold && + max_rel_rms > (double)max_rel_threshold && + rmse_rel > (double)rmse_rel_threshold); + fprintf(stderr, + "[GTX CPU REF] graph=%llu node=%d op=%s tensor=%s shape=[%lld,%lld,%lld,%lld] " + "max_diff=%.9g index=%zu ref=%.9g got=%.9g rmse=%.9g ref_rms=%.9g " + "max_rel_rms=%.9g rmse_rel=%.9g nonfinite=%zu status=%s\n", + (unsigned long long)gtx_cpu_ref_current_graph, node_index, + gtx_tensor_op_display_name(node), node->name[0] != '\0' ? node->name : "(anon)", + (long long)node->ne[0], (long long)node->ne[1], + (long long)node->ne[2], (long long)node->ne[3], + max_diff, max_index, reference[max_index], actual[max_index], + rmse, ref_rms, max_rel_rms, rmse_rel, nonfinite, diverged ? "DIVERGED" : "ok"); + if (diverged) { + gtx_cpu_ref_first_divergence_found = true; + } + return !diverged; +} + +static bool gtx_scheduler_lower_row_island(const std::vector & nodes, + std::vector & out_descs) { + if (nodes.empty() || nodes[0] == nullptr || + !gtx_scheduler_dense_f16_2d(nodes[0])) { + return false; + } + + const int64_t ne0_64 = nodes[0]->ne[0]; + const int64_t rows64 = nodes[0]->ne[1]; + if (ne0_64 <= 0 || rows64 <= 0 || ne0_64 > UINT32_MAX || rows64 > UINT32_MAX) { + return false; + } + + for (const ggml_tensor * node : nodes) { + if (!gtx_scheduler_tileable_row_node(node) || + !gtx_scheduler_same_2d_shape(nodes[0], node)) { + return false; + } + } + + const uint32_t ne0 = (uint32_t)ne0_64; + const uint32_t total_rows = (uint32_t)rows64; + const uint64_t row_bytes = (uint64_t)ne0 * sizeof(uint16_t); + const uint64_t slot_budget = (gtx_stash_data_budget(nodes) / GTX_STASH_SLOT_COUNT) & ~31ull; + const uint64_t max_tile_bytes = std::min( + gtx_env_u64_or_default("GTX_TILE_MAX_BYTES", slot_budget), slot_budget); + if (row_bytes == 0 || max_tile_bytes < row_bytes) { + return false; + } + + uint32_t rows_per_tile = (uint32_t)std::min(total_rows, max_tile_bytes / row_bytes); + if (rows_per_tile == 0) { + return false; + } + + out_descs.clear(); + for (uint32_t row0 = 0; row0 < total_rows; row0 += rows_per_tile) { + const uint32_t rows = std::min(rows_per_tile, total_rows - row0); + for (const ggml_tensor * node : nodes) { + ggml_gtx_kernel_launch_desc_t desc = {}; + if (!gtx_scheduler_build_row_tile_desc(node, row0, rows, desc)) { + out_descs.clear(); + return false; + } + out_descs.push_back(desc); + } + } + return !out_descs.empty(); +} + +static bool gtx_scheduler_find_row_island(ggml_cgraph * cgraph, + int start, + int & end, + std::vector & nodes) { + nodes.clear(); + end = start; + if (cgraph == nullptr || start < 0 || start >= cgraph->n_nodes) { + return false; + } + + const ggml_tensor * first = cgraph->nodes[start]; + if (!gtx_scheduler_tileable_row_node(first)) { + return false; + } + + for (int i = start; i < cgraph->n_nodes; ++i) { + ggml_tensor * node = cgraph->nodes[i]; + if (!gtx_scheduler_tileable_row_node(node) || + !gtx_scheduler_same_2d_shape(first, node)) { + break; + } + nodes.push_back(node); + end = i + 1; + } + + return nodes.size() >= 2; +} + +static bool gtx_scheduler_tileable_col_node(const ggml_tensor * node) { + if (node == nullptr || + node->ne[0] <= 0 || + node->ne[1] <= 0 || + node->ne[2] <= 0 || + node->ne[3] <= 0 || + node->ne[0] > UINT32_MAX || + node->ne[1] > UINT32_MAX || + node->ne[2] > UINT32_MAX || + node->ne[3] > UINT32_MAX) { + return false; + } + + const uint32_t kernel_id = gtx_kernel_id_from_node(node); + if (kernel_id == GGML_GTX_KERNEL_MUL_MAT_F16) { + // `src0.ne[1]` is output-row count and may exceed the L1/kernel + // limit in the complete tensor. The host row tiler applies that + // limit to each emitted tile; do not reject it before tiling. + return node->src[0] != nullptr && + node->src[1] != nullptr && + gtx_tensor_remote_f16_element_contiguous(node) && + node->ne[1] <= UINT16_MAX && + node->ne[2] <= UINT16_MAX && + node->ne[3] <= UINT16_MAX && + node->src[0]->ne[0] == node->src[1]->ne[0] && + node->src[0]->ne[0] <= UINT16_MAX && + node->src[1]->ne[1] == node->ne[1] && + node->src[1]->ne[2] == node->ne[2] && + node->src[1]->ne[3] == node->ne[3] && + node->src[0]->ne[2] > 0 && node->src[0]->ne[3] > 0 && + node->ne[2] % node->src[0]->ne[2] == 0 && + node->ne[3] % node->src[0]->ne[3] == 0 && + gtx_tensor_remote_f16_vector_contiguous(node->src[0]) && + gtx_tensor_remote_f16_vector_contiguous(node->src[1]) && + node->src[1]->ne[2] == node->ne[2] && + node->src[1]->ne[3] == node->ne[3] && + node->src[1]->ne[1] == node->ne[1]; + } + + if (!gtx_tensor_remote_f16_dense(node)) { + return false; + } + + const gtx_l2_chainability chain = gtx_l2_chainability_from_node(node); + if (chain.kind != gtx_l2_chain_kind::native) { + return false; + } + + if (chain.kernel_id == GGML_GTX_KERNEL_SWIGLU_F16) { + return gtx_l2_swiglu_shape_ok(node); + } + + if (node->src[0] == nullptr || + !gtx_tensor_remote_f16_dense(node->src[0]) || + !gtx_tensor_same_shape(node, node->src[0])) { + return false; + } + if (chain.kernel_id == GGML_GTX_KERNEL_SOFT_MAX_F16) { + return gtx_l2_softmax_basic_shape_ok(node); + } + if (chain.kernel_id == GGML_GTX_KERNEL_ROPE_F16) { + return gtx_l2_rope_shape_ok(node); + } + if (chain.kernel_id == GGML_GTX_KERNEL_ADD_F16 || + chain.kernel_id == GGML_GTX_KERNEL_MUL_F16) { + return node->src[1] != nullptr && + gtx_tensor_remote_f16_dense(node->src[1]) && + gtx_tensor_same_shape(node, node->src[1]); + } + return chain.kernel_id == GGML_GTX_KERNEL_SCALE_F16 || + chain.kernel_id == GGML_GTX_KERNEL_GELU_F16 || + chain.kernel_id == GGML_GTX_KERNEL_RMS_NORM_F16 || + chain.kernel_id == GGML_GTX_KERNEL_ROPE_F16 || + chain.kernel_id == GGML_GTX_KERNEL_SWIGLU_F16 || + chain.kernel_id == GGML_GTX_KERNEL_ABS_F16 || + chain.kernel_id == GGML_GTX_KERNEL_NEG_F16 || + chain.kernel_id == GGML_GTX_KERNEL_SGN_F16 || + chain.kernel_id == GGML_GTX_KERNEL_CEIL_F16 || + chain.kernel_id == GGML_GTX_KERNEL_FLOOR_F16 || + chain.kernel_id == GGML_GTX_KERNEL_TANH_F16 || + chain.kernel_id == GGML_GTX_KERNEL_SILU_F16 || + chain.kernel_id == GGML_GTX_KERNEL_RELU_F16 || + chain.kernel_id == GGML_GTX_KERNEL_LEAKY_RELU_F16 || + chain.kernel_id == GGML_GTX_KERNEL_CLAMP_F16; +} + +static bool gtx_scheduler_node_consumes_prev(const ggml_tensor * node, + const ggml_tensor * prev) { + if (node == nullptr || prev == nullptr) { + return false; + } + for (int i = 0; i < 4; ++i) { + const ggml_tensor * src = node->src[i]; + while (src != nullptr && gtx_is_metadata_op(src->op) && src->src[0] != nullptr) { + src = src->src[0]; + } + if (src == prev) { + return true; + } + } + return false; +} + +static bool gtx_scheduler_build_col_tile_desc(const ggml_tensor * node, + uint32_t col0, + uint32_t cols, + ggml_gtx_kernel_launch_desc_t & tile) { + gtx_scheduler_col_tile_shape shape = {}; + if (node == nullptr || + !gtx_scheduler_make_col_tile_shape(node, col0, cols, shape)) { + return false; + } + + const uint32_t kernel_id = gtx_kernel_id_from_node(node); + const uint64_t dst_off = gtx_scheduler_col_offset(node, shape); + + if (kernel_id == GGML_GTX_KERNEL_SWIGLU_F16) { + if (!gtx_build_kernel_launch_desc(node, tile) || + node->src[0] == nullptr || + node->ne[0] > UINT32_MAX || + node->src[0]->ne[0] > UINT32_MAX) { + return false; + } + + const uint32_t ne0 = (uint32_t)node->ne[0]; + const uint32_t src_ne0 = (uint32_t)node->src[0]->ne[0]; + const ggml_tensor * src = node->src[0]; + const uint64_t src_off = gtx_scheduler_col_offset(src, shape); + tile.dst_addr += dst_off; + tile.src_addr[0] += src_off; + gtx_scheduler_make_dense_tile_shape_4d(tile, ne0, shape.ne1, shape.ne2, shape.ne3); + gtx_scheduler_make_dense_src_tile_shape_4d(tile, 0, src_ne0, shape.ne1, shape.ne2, shape.ne3); + tile.flags |= GGML_GTX_DESC_FLAG_STASH_CHAIN; + return true; + } + + if (kernel_id != GGML_GTX_KERNEL_MUL_MAT_F16) { + if (!gtx_build_kernel_launch_desc(node, tile) || + node->ne[0] > UINT32_MAX) { + return false; + } + const uint32_t ne0 = (uint32_t)node->ne[0]; + tile.dst_addr += dst_off; + gtx_scheduler_make_dense_tile_shape_4d(tile, ne0, shape.ne1, shape.ne2, shape.ne3); + for (uint32_t s = 0; s < 4; ++s) { + const ggml_tensor * src = node->src[s]; + if (src == nullptr || tile.src_addr[s] == 0) { + continue; + } + if (gtx_tensor_same_shape(node, src)) { + const uint64_t src_off = gtx_scheduler_col_offset(src, shape); + tile.src_addr[s] += src_off; + gtx_scheduler_make_dense_src_tile_shape_4d(tile, s, ne0, shape.ne1, shape.ne2, shape.ne3); + } + } + const bool flat_kernel = gtx_scheduler_row_tile_kernel_is_flat(kernel_id) && + kernel_id != GGML_GTX_KERNEL_ADD_F16 && + kernel_id != GGML_GTX_KERNEL_MUL_F16; + if (flat_kernel) { + const uint64_t tile_elems = (uint64_t)ne0 * shape.ne1 * shape.ne2 * shape.ne3; + if (tile_elems == 0u || tile_elems > UINT16_MAX) { + return false; + } + gtx_scheduler_make_dense_tile_shape(tile, (uint32_t)tile_elems, 1u); + for (uint32_t s = 0; s < 4; ++s) { + if (node->src[s] != nullptr && tile.src_addr[s] != 0u && + gtx_tensor_same_shape(node, node->src[s])) { + gtx_scheduler_make_dense_src_tile_shape( + tile, s, (uint32_t)tile_elems, 1u); + } + } + } + if (kernel_id == GGML_GTX_KERNEL_ROPE_F16) { + const ggml_tensor * pos = node->src[1]; + if (pos == nullptr || shape.i2 + shape.ne2 > (uint32_t)pos->ne[0]) { + return false; + } + tile.src_addr[1] += (uint64_t)shape.i2 * gtx_remote_byte_offset(pos, pos->nb[0]); + tile.src_ne[1][0] = shape.ne2; + tile.src_ne[1][1] = 1u; + tile.src_ne[1][2] = 1u; + tile.src_ne[1][3] = 1u; + tile.src_nb[1][0] = sizeof(int32_t); + tile.src_nb[1][1] = shape.ne2 * sizeof(int32_t); + tile.src_nb[1][2] = tile.src_nb[1][1]; + tile.src_nb[1][3] = tile.src_nb[1][1]; + tile.l2_scratch_size = ggml_gtx_rope_scratch_size(shape.ne2, + (uint32_t)tile.op_params[1]); + } + tile.flags |= GGML_GTX_DESC_FLAG_STASH_CHAIN; + return true; + } + + if (!gtx_build_kernel_launch_desc(node, tile) || + node->src[1] == nullptr || + node->ne[0] <= 0 || + node->src[1]->ne[0] <= 0 || + node->ne[0] > UINT32_MAX || + node->src[1]->ne[0] > UINT32_MAX) { + return false; + } + + const uint32_t dst_ne0 = (uint32_t)node->ne[0]; + const uint32_t src1_ne0 = (uint32_t)node->src[1]->ne[0]; + const uint32_t src0_ne2 = (uint32_t)node->src[0]->ne[2]; + const uint32_t src0_ne3 = (uint32_t)node->src[0]->ne[3]; + const uint32_t src0_r2 = (uint32_t)node->ne[2] / src0_ne2; + const uint32_t src0_r3 = (uint32_t)node->ne[3] / src0_ne3; + const uint32_t src0_i2 = shape.i2 / src0_r2; + const uint32_t src0_i3 = shape.i3 / src0_r3; + const uint64_t src0_off = + (uint64_t)src0_i2 * gtx_remote_byte_offset(node->src[0], node->src[0]->nb[2]) + + (uint64_t)src0_i3 * gtx_remote_byte_offset(node->src[0], node->src[0]->nb[3]); + const uint64_t src1_off = gtx_scheduler_col_offset(node->src[1], shape); + tile.dst_addr += dst_off; + tile.src_addr[0] += src0_off; + tile.src_addr[1] += src1_off; + // Preserve the raw DDR strides. Firmware stash packs this view into a + // dense L2 allocation before invoking the uploaded kernel. + gtx_scheduler_make_strided_dst_tile_shape_4d( + tile, node, dst_ne0, shape.ne1, shape.ne2, shape.ne3); + if (shape.ne2 > 1u) { + tile.src_ne[0][2] = (uint32_t)node->src[0]->ne[2]; + } else { + tile.src_ne[0][2] = 1; + } + tile.src_ne[0][3] = 1; + gtx_scheduler_make_strided_src_tile_shape_4d(tile, 1, node->src[1], src1_ne0, shape.ne1, shape.ne2, shape.ne3); + tile.flags |= GGML_GTX_DESC_FLAG_STASH_CHAIN; + return true; +} + +static bool gtx_scheduler_build_mul_mat_col_row_tile_desc(const ggml_tensor * node, + uint32_t col0, + uint32_t cols, + uint32_t row0, + uint32_t rows, + ggml_gtx_kernel_launch_desc_t & tile) { + gtx_scheduler_col_tile_shape shape = {}; + if (node == nullptr || + gtx_kernel_id_from_node(node) != GGML_GTX_KERNEL_MUL_MAT_F16 || + node->src[0] == nullptr || + node->src[1] == nullptr || + rows == 0 || + !gtx_scheduler_make_col_tile_shape(node, col0, cols, shape) || + !gtx_build_kernel_launch_desc(node, tile) || + node->ne[0] <= 0 || + node->src[1]->ne[0] <= 0 || + node->ne[0] > UINT32_MAX || + node->src[1]->ne[0] > UINT32_MAX || + row0 >= (uint32_t)node->ne[0] || + rows > (uint32_t)node->ne[0] - row0) { + return false; + } + + const uint32_t dst_ne0 = rows; + const uint32_t src1_ne0 = (uint32_t)node->src[1]->ne[0]; + const uint32_t src0_ne2 = (uint32_t)node->src[0]->ne[2]; + const uint32_t src0_ne3 = (uint32_t)node->src[0]->ne[3]; + const uint32_t src0_r2 = (uint32_t)node->ne[2] / src0_ne2; + const uint32_t src0_r3 = (uint32_t)node->ne[3] / src0_ne3; + const uint32_t src0_i2 = shape.i2 / src0_r2; + const uint32_t src0_i3 = shape.i3 / src0_r3; + const uint64_t src0_off = + (uint64_t)src0_i2 * gtx_remote_byte_offset(node->src[0], node->src[0]->nb[2]) + + (uint64_t)src0_i3 * gtx_remote_byte_offset(node->src[0], node->src[0]->nb[3]) + + (uint64_t)row0 * gtx_remote_byte_offset(node->src[0], node->src[0]->nb[1]); + const uint64_t src1_off = gtx_scheduler_col_offset(node->src[1], shape); + const uint64_t dst_off = gtx_scheduler_col_offset(node, shape) + + (uint64_t)row0 * sizeof(uint16_t); + + tile.dst_addr += dst_off; + tile.src_addr[0] += src0_off; + tile.src_addr[1] += src1_off; + // A row slice spanning multiple output columns retains the full output + // column stride in DDR. Stash scatters the compact L2 result through it. + gtx_scheduler_make_strided_dst_tile_shape_4d( + tile, node, dst_ne0, shape.ne1, shape.ne2, shape.ne3); + tile.src_ne[0][1] = rows; + if (shape.ne2 > 1u) { + tile.src_ne[0][2] = (uint32_t)node->src[0]->ne[2]; + } else { + tile.src_ne[0][2] = 1; + } + tile.src_ne[0][3] = 1; + gtx_scheduler_make_strided_src_tile_shape_4d(tile, 1, node->src[1], src1_ne0, shape.ne1, shape.ne2, shape.ne3); + tile.flags |= GGML_GTX_DESC_FLAG_STASH_CHAIN; + return true; +} + +static uint32_t gtx_scheduler_mul_mat_rows_per_output_tile(const std::vector & nodes, + size_t node_idx, + uint32_t cols) { + const ggml_tensor * node = node_idx < nodes.size() ? nodes[node_idx] : nullptr; + if (node == nullptr || + node->src[0] == nullptr || + node->src[1] == nullptr || + node->ne[0] <= 0 || + node->src[0]->ne[0] <= 0 || + node->src[1]->ne[0] <= 0 || + cols == 0 || + node->ne[0] > UINT32_MAX || + node->src[0]->ne[0] > UINT32_MAX || + node->src[1]->ne[0] > UINT32_MAX) { + return 0; + } + + const uint64_t data_budget = gtx_stash_data_budget(node); + const uint64_t l2_budget_bytes = std::min( + gtx_env_u64_or_default("GTX_TILE_MAX_BYTES", data_budget), data_budget); + const uint64_t reserve_bytes = gtx_env_u64_or_default("GTX_TILE_RESERVE_BYTES", 64ull * 1024ull); + const uint64_t dst_row_bytes = (uint64_t)cols * sizeof(uint16_t); + const uint64_t weight_row_bytes = (uint64_t)node->src[0]->ne[0] * sizeof(uint16_t); + const uint64_t src1_bytes = (uint64_t)node->src[1]->ne[0] * (uint64_t)cols * sizeof(uint16_t); + const uint64_t fixed_bytes = gtx_align_up((size_t)src1_bytes, 32) + reserve_bytes; + const uint64_t bytes_per_output_row = weight_row_bytes + dst_row_bytes; + if (bytes_per_output_row == 0 || + l2_budget_bytes <= fixed_bytes || + l2_budget_bytes - fixed_bytes < bytes_per_output_row) { + return 1; + } + + uint64_t rows = (l2_budget_bytes - fixed_bytes) / bytes_per_output_row; + rows = std::min(rows, (uint64_t)node->ne[0]); + // Keep an optional override for focused hardware-boundary experiments, + // but derive the normal row tile solely from the actual L2 budget. A + // single __mm_t(64, 128, 2) is verified on the active n1s1 FPGA, so a + // blanket 32-row cap would only multiply descriptor count. + rows = std::min( + rows, + gtx_env_u64_or_default("GTX_MUL_MAT_ROW_TILE_MAX_ROWS", (uint64_t)node->ne[0])); + rows = std::min(rows, UINT16_MAX); + while (rows > 1) { + const uint64_t src0_bytes = weight_row_bytes * rows; + const uint64_t dst_bytes = dst_row_bytes * rows; + const uint64_t total_bytes = + gtx_align_up((size_t)src0_bytes, 32) + + gtx_align_up((size_t)dst_bytes, 32) + + gtx_align_up((size_t)src1_bytes, 32) + + reserve_bytes; + if (total_bytes <= l2_budget_bytes) { + break; + } + --rows; + } + return rows == 0 ? 1u : (uint32_t)rows; +} + +static bool gtx_scheduler_col_tile_contiguous_for_node(const ggml_tensor * node, + uint32_t col0, + uint32_t cols) { + if (node == nullptr) { + return false; + } + + const uint32_t kernel_id = gtx_kernel_id_from_node(node); + if (kernel_id == GGML_GTX_KERNEL_MUL_MAT_F16) { + gtx_scheduler_col_tile_shape shape = {}; + if (!gtx_scheduler_make_col_tile_shape(node, col0, cols, shape) || + node->src[0] == nullptr || + node->src[1] == nullptr || + !gtx_tensor_remote_f16_element_contiguous(node) || + !gtx_scheduler_col_span_strided_rows_supported(node->src[1], col0, cols)) { + return false; + } + + // A row-sliced, multi-column destination is strided in DDR. Firmware + // stash already packs/unpacks raw strided views and exposes a dense L2 + // view to the kernel, so it is safe to retain the computed col tile. + if (!gtx_scheduler_col_span_strided_rows_supported(node, col0, cols)) { + return false; + } + if (shape.ne2 > 1u) { + const uint32_t src0_ne2 = (uint32_t)node->src[0]->ne[2]; + if (src0_ne2 == 0u || + shape.i2 % src0_ne2 != 0u || + shape.ne2 % src0_ne2 != 0u) { + return false; + } + } + return true; + } + + if (!gtx_scheduler_col_span_contiguous(node, col0, cols)) { + return false; + } + + if (kernel_id == GGML_GTX_KERNEL_SWIGLU_F16) { + return node->src[0] != nullptr && + gtx_scheduler_col_span_contiguous(node->src[0], col0, cols); + } + + for (uint32_t s = 0; s < 4; ++s) { + const ggml_tensor * src = node->src[s]; + if (src != nullptr && + gtx_tensor_same_shape(node, src) && + !gtx_scheduler_col_span_contiguous(src, col0, cols)) { + return false; + } + } + return true; +} + +static bool gtx_scheduler_col_tile_contiguous_for_island(const std::vector & nodes, + uint32_t col0, + uint32_t cols) { + for (const ggml_tensor * node : nodes) { + if (!gtx_scheduler_col_tile_contiguous_for_node(node, col0, cols)) { + return false; + } + } + return true; +} + + +// ⚠️ **flat 커널의 한계는 원소 수가 아니라 바이트다.** 업로드 커널(relu·clamp·scale 등)은 +// 전송 폭을 `(uint16_t)(elems * 2)` 로 만들어, 바이트가 65536 을 넘으면 감긴다. +// f16 이므로 원소로는 **32768** 이 한계다. 아래 타일러들이 `UINT16_MAX` 를 원소 수로 +// 쓰고 있어 정확히 두 배를 허용했고, 그만큼이 조용히 틀린 값으로 나갔다. +// 실측(`gtx_smoke --limits`): relu·clamp·leaky_relu·neg·scale 모두 32768 까지 맞고 +// 32769 부터 틀린다. ResNet-18 에서는 argmax 282 → 228 로 나타났다. +// ADD/MUL 은 4-D 모양을 유지하는 별도 커널이라 이 한계를 안 탄다. +static constexpr uint64_t GTX_FLAT_KERNEL_MAX_ELEMS = 32768u; + +static uint32_t gtx_scheduler_max_cols_per_tile( + const std::vector & nodes, + uint32_t total_cols) { + if (nodes.empty() || nodes[0] == nullptr || total_cols == 0u) { + return 0u; + } + + const uint64_t data_budget = gtx_stash_data_budget(nodes); + const uint64_t slot_budget = (data_budget / GTX_STASH_SLOT_COUNT) & ~31ull; + const uint64_t configured_budget = + gtx_env_u64_or_default("GTX_TILE_MAX_BYTES", slot_budget); + const uint64_t view_budget = std::min(slot_budget, configured_budget); + if (view_budget == 0u) { + return 0u; + } + + // Do not cross logical ne1 boundaries. Crossing into ne2 can select a + // different head/batch weight matrix and requires a separate raw view. + uint64_t layout_limit = total_cols; + uint64_t max_col_bytes = 0u; + for (const ggml_tensor * node : nodes) { + if (node == nullptr || node->ne[0] <= 0 || node->ne[1] <= 0) { + return 0u; + } + layout_limit = std::min(layout_limit, (uint64_t)node->ne[1]); + const uint32_t kernel_id = gtx_kernel_id_from_node(node); + if (kernel_id == GGML_GTX_KERNEL_MUL_MAT_F16) { + layout_limit = std::min( + layout_limit, + gtx_env_u64_or_default("GTX_MUL_MAT_COL_TILE_MAX_COLS", layout_limit)); + } + const bool flat_kernel = gtx_scheduler_row_tile_kernel_is_flat(kernel_id) && + kernel_id != GGML_GTX_KERNEL_ADD_F16 && + kernel_id != GGML_GTX_KERNEL_MUL_F16; + if (flat_kernel) { + const uint64_t flat_cols_limit = GTX_FLAT_KERNEL_MAX_ELEMS / (uint64_t)node->ne[0]; + if (flat_cols_limit == 0u) { + return 0u; + } + layout_limit = std::min(layout_limit, flat_cols_limit); + } + max_col_bytes = std::max( + max_col_bytes, (uint64_t)node->ne[0] * sizeof(uint16_t)); + + if (gtx_kernel_id_from_node(node) == GGML_GTX_KERNEL_MUL_MAT_F16) { + if (node->src[1] == nullptr || node->src[1]->ne[0] <= 0) { + return 0u; + } + max_col_bytes = std::max( + max_col_bytes, (uint64_t)node->src[1]->ne[0] * sizeof(uint16_t)); + } else { + for (uint32_t i = 0; i < 4u; ++i) { + const ggml_tensor * src = node->src[i]; + if (src != nullptr && src->ne[0] > 0 && + src->ne[1] * src->ne[2] * src->ne[3] == + node->ne[1] * node->ne[2] * node->ne[3]) { + max_col_bytes = std::max( + max_col_bytes, (uint64_t)src->ne[0] * sizeof(uint16_t)); + } + } + } + } + + if (max_col_bytes == 0u || view_budget < max_col_bytes) { + return 1u; + } + const uint64_t budget_limit = view_budget / max_col_bytes; + return (uint32_t)std::max( + 1u, std::min(layout_limit, budget_limit)); +} + +static bool gtx_scheduler_lower_col_island(const std::vector & nodes, + std::vector & out_descs) { + if (nodes.empty() || nodes[0] == nullptr) { + return false; + } + + const int64_t total_cols64 = nodes[0]->ne[1] * nodes[0]->ne[2] * nodes[0]->ne[3]; + if (total_cols64 <= 0 || total_cols64 > UINT32_MAX) { + return false; + } + + bool has_mul_mat = false; + for (const ggml_tensor * node : nodes) { + if (!gtx_scheduler_tileable_col_node(node) || + node->ne[1] * node->ne[2] * node->ne[3] != total_cols64) { + return false; + } + has_mul_mat = has_mul_mat || + gtx_kernel_id_from_node(node) == GGML_GTX_KERNEL_MUL_MAT_F16; + } + if (!has_mul_mat) { + return false; + } + + const uint32_t total_cols = (uint32_t)total_cols64; + // Coalesce consecutive N columns up to both the generic stash slot budget + // and the logical ne1 boundary. The uploaded MUL_MAT kernel consumes the + // resulting b_row directly; firmware does not split the tile. + const uint32_t max_cols_per_tile = + gtx_scheduler_max_cols_per_tile(nodes, total_cols); + if (max_cols_per_tile == 0u) { + return false; + } + + out_descs.clear(); + for (uint32_t col0 = 0; col0 < total_cols;) { + uint32_t cols = std::min(max_cols_per_tile, total_cols - col0); + while (cols > 1u && !gtx_scheduler_col_tile_contiguous_for_island(nodes, col0, cols)) { + --cols; + } + if (!gtx_scheduler_col_tile_contiguous_for_island(nodes, col0, cols)) { + out_descs.clear(); + return false; + } + for (size_t node_idx = 0; node_idx < nodes.size(); ++node_idx) { + const ggml_tensor * node = nodes[node_idx]; + ggml_gtx_kernel_launch_desc_t desc = {}; + const bool last_node = node_idx + 1 == nodes.size(); + if (last_node && + gtx_kernel_id_from_node(node) == GGML_GTX_KERNEL_MUL_MAT_F16) { + const uint32_t rows_per_tile = gtx_scheduler_mul_mat_rows_per_output_tile(nodes, node_idx, cols); + if (rows_per_tile == 0) { + out_descs.clear(); + return false; + } + for (uint32_t row0 = 0; row0 < (uint32_t)node->ne[0]; row0 += rows_per_tile) { + const uint32_t rows = std::min(rows_per_tile, (uint32_t)node->ne[0] - row0); + if (!gtx_scheduler_build_mul_mat_col_row_tile_desc(node, col0, cols, row0, rows, desc)) { + out_descs.clear(); + return false; + } + out_descs.push_back(desc); + } + } else { + if (!gtx_scheduler_build_col_tile_desc(node, col0, cols, desc)) { + out_descs.clear(); + return false; + } + out_descs.push_back(desc); + } + } + col0 += cols; + } + return !out_descs.empty(); +} + +static bool gtx_scheduler_find_col_island(ggml_cgraph * cgraph, + int start, + int & end, + std::vector & nodes) { + nodes.clear(); + end = start; + if (cgraph == nullptr || start < 0 || start >= cgraph->n_nodes) { + return false; + } + + ggml_tensor * first = cgraph->nodes[start]; + if (!gtx_scheduler_tileable_col_node(first)) { + return false; + } + + bool has_mul_mat = false; + ggml_tensor * prev = nullptr; + const int64_t total_cols = first->ne[1] * first->ne[2] * first->ne[3]; + for (int i = start; i < cgraph->n_nodes; ++i) { + ggml_tensor * node = cgraph->nodes[i]; + if (!gtx_scheduler_tileable_col_node(node) || + node->ne[1] * node->ne[2] * node->ne[3] != total_cols || + (prev != nullptr && !gtx_scheduler_node_consumes_prev(node, prev))) { + break; + } + has_mul_mat = has_mul_mat || + gtx_kernel_id_from_node(node) == GGML_GTX_KERNEL_MUL_MAT_F16; + nodes.push_back(node); + prev = node; + end = i + 1; + } + + return has_mul_mat && !nodes.empty(); +} + +static bool gtx_scheduler_lower_conv_l2_node( + const ggml_tensor * node, + std::vector & out_descs) { + if (node == nullptr || node->src[0] == nullptr || node->src[1] == nullptr) { + return false; + } + const uint32_t kernel_id = gtx_kernel_id_from_node(node); + const bool depthwise = kernel_id == GGML_GTX_KERNEL_CONV_2D_DW_F16; + if (!depthwise && kernel_id != GGML_GTX_KERNEL_CONV_2D_F16) { + return false; + } + const bool c_fast = depthwise && + gtx_tensor_remote_f16_c_fast_whc(node) && + gtx_tensor_remote_f16_c_fast_dw_weight(node->src[0]) && + gtx_tensor_remote_f16_c_fast_whc(node->src[1]); + + ggml_gtx_kernel_launch_desc_t base = {}; + if (!gtx_build_kernel_launch_desc(node, base)) { + return false; + } + const int32_t s1 = base.op_params[1]; + const int32_t p1 = base.op_params[3]; + const int32_t d1 = base.op_params[5]; + const uint32_t kh = base.src_ne[0][1]; + if (s1 <= 0 || p1 < 0 || d1 <= 0 || kh == 0 || base.ne[3] != 1u || + base.src_ne[1][3] != 1u) { + return false; + } + const uint64_t k_eff = (uint64_t)d1 * (kh - 1u) + 1u; + const uint64_t data_budget = gtx_stash_data_budget(node); + const uint64_t slot_budget = (data_budget / GTX_STASH_SLOT_COUNT) & ~31ull; + const uint32_t total_channels = base.ne[2]; + const uint32_t total_oh = base.ne[1]; + const uint32_t iw = base.src_ne[1][0]; + const uint32_t ih = base.src_ne[1][1]; + const uint32_t ic = base.src_ne[1][2]; + const uint32_t kw = base.src_ne[0][0]; + if (slot_budget == 0 || total_channels == 0 || total_oh == 0 || + iw == 0 || ih == 0 || ic == 0 || kw == 0) { + return false; + } + + const auto aligned = [](uint64_t bytes) { + return (bytes + 31u) & ~31ull; + }; + const auto make_tile_shape = [&](uint32_t oh0, uint32_t oh_count, + uint32_t channels, + uint32_t & clip_h0, uint32_t & clip_h, + uint32_t & pad_top, + uint64_t & input_bytes, + uint64_t & weight_bytes, + uint64_t & output_bytes) { + const int64_t raw_h0 = (int64_t)oh0 * s1 - p1; + const int64_t raw_h1 = (int64_t)(oh0 + oh_count - 1u) * s1 - p1 + (int64_t)k_eff; + const int64_t clipped_h0 = std::max(0, raw_h0); + const int64_t clipped_h1 = std::min((int64_t)ih, raw_h1); + if (clipped_h1 <= clipped_h0) { + return false; + } + clip_h0 = (uint32_t)clipped_h0; + clip_h = (uint32_t)(clipped_h1 - clipped_h0); + pad_top = raw_h0 < 0 ? (uint32_t)(-raw_h0) : 0u; + const uint64_t tile_ic = depthwise ? channels : ic; + input_bytes = aligned((uint64_t)iw * clip_h * tile_ic * sizeof(uint16_t)); + weight_bytes = aligned(depthwise ? + (uint64_t)kw * kh * channels * sizeof(uint16_t) : + (uint64_t)kw * kh * ic * channels * sizeof(uint16_t)); + output_bytes = aligned((uint64_t)base.ne[0] * oh_count * channels * sizeof(uint16_t)); + return input_bytes <= slot_budget && weight_bytes <= slot_budget && + output_bytes <= slot_budget && + input_bytes + weight_bytes + output_bytes <= data_budget; + }; + + out_descs.clear(); + for (uint32_t c0 = 0; c0 < total_channels;) { + uint32_t channels = total_channels - c0; + uint32_t dummy_h0 = 0, dummy_h = 0, dummy_pad = 0; + uint64_t input_bytes = 0, weight_bytes = 0, output_bytes = 0; + while (channels > 0 && + !make_tile_shape(0, 1, channels, dummy_h0, dummy_h, dummy_pad, + input_bytes, weight_bytes, output_bytes)) { + channels = channels == 1 ? 0 : (channels + 1u) / 2u; + } + if (channels == 0) { + out_descs.clear(); + return false; + } + + for (uint32_t oh0 = 0; oh0 < total_oh;) { + uint32_t oh_count = total_oh - oh0; + uint32_t clip_h0 = 0, clip_h = 0, pad_top = 0; + while (oh_count > 0 && + !make_tile_shape(oh0, oh_count, channels, + clip_h0, clip_h, pad_top, + input_bytes, weight_bytes, output_bytes)) { + oh_count = oh_count == 1 ? 0 : (oh_count + 1u) / 2u; + } + if (oh_count == 0) { + out_descs.clear(); + return false; + } + + ggml_gtx_kernel_launch_desc_t tile = base; + tile.dst_addr += (uint64_t)oh0 * base.nb[1] + (uint64_t)c0 * base.nb[2]; + tile.ne[1] = oh_count; + tile.ne[2] = channels; + tile.src_addr[1] += (uint64_t)clip_h0 * base.src_nb[1][1]; + tile.src_ne[1][1] = clip_h; + if (depthwise) { + tile.src_addr[1] += (uint64_t)c0 * base.src_nb[1][2]; + tile.src_ne[1][2] = channels; + tile.src_addr[0] += (uint64_t)c0 * base.src_nb[0][3]; + tile.src_ne[0][3] = channels; + } else { + tile.src_addr[0] += (uint64_t)c0 * base.src_nb[0][3]; + tile.src_ne[0][3] = channels; + } + tile.op_params[3] = (int32_t)pad_top; + if (c_fast) { + tile.op_params[GGML_GTX_CONV_LAYOUT_PARAM] = GGML_GTX_CONV_LAYOUT_C_FAST; + tile.op_params[GGML_GTX_CONV_LOGICAL_OW_PARAM] = (int32_t)base.ne[0]; + tile.op_params[GGML_GTX_CONV_LOGICAL_OH_PARAM] = (int32_t)oh_count; + tile.op_params[GGML_GTX_CONV_LOGICAL_C_PARAM] = (int32_t)channels; + tile.op_params[GGML_GTX_CONV_LOGICAL_IW_PARAM] = (int32_t)iw; + tile.op_params[GGML_GTX_CONV_LOGICAL_IH_PARAM] = (int32_t)clip_h; + tile.op_params[GGML_GTX_CONV_LOGICAL_KW_PARAM] = (int32_t)kw; + tile.op_params[GGML_GTX_CONV_LOGICAL_KH_PARAM] = (int32_t)kh; + + // Re-express each C-fast remote tile with C as raw ne0. The + // generic stash mapper can then pack it without op knowledge; + // the DW kernel reconstructs the logical WHC view in scratch. + tile.ne[0] = channels; + tile.ne[1] = base.ne[0]; + tile.ne[2] = oh_count; + tile.ne[3] = 1u; + tile.nb[0] = base.nb[2]; + tile.nb[1] = base.nb[0]; + tile.nb[2] = base.nb[1]; + tile.nb[3] = base.nb[3]; + + tile.src_ne[1][0] = channels; + tile.src_ne[1][1] = iw; + tile.src_ne[1][2] = clip_h; + tile.src_ne[1][3] = 1u; + tile.src_nb[1][0] = base.src_nb[1][2]; + tile.src_nb[1][1] = base.src_nb[1][0]; + tile.src_nb[1][2] = base.src_nb[1][1]; + tile.src_nb[1][3] = base.src_nb[1][3]; + + tile.src_ne[0][0] = channels; + tile.src_ne[0][1] = kw; + tile.src_ne[0][2] = kh; + tile.src_ne[0][3] = 1u; + tile.src_nb[0][0] = base.src_nb[0][3]; + tile.src_nb[0][1] = base.src_nb[0][0]; + tile.src_nb[0][2] = base.src_nb[0][1]; + tile.src_nb[0][3] = base.src_nb[0][2]; + } + tile.l2_scratch_size = 8u * 1024u * 1024u; + out_descs.push_back(tile); + oh0 += oh_count; + } + c0 += channels; + } + return !out_descs.empty(); +} + +static bool gtx_scheduler_lower_conv_transpose_l2_node( + const ggml_tensor * node, + std::vector & out_descs) { + if (node == nullptr || node->src[0] == nullptr || node->src[1] == nullptr || + gtx_kernel_id_from_node(node) != GGML_GTX_KERNEL_CONV_TRANSPOSE_2D_F16 || + node->ne[0] <= 0 || node->ne[1] <= 0 || node->ne[2] <= 0 || node->ne[3] <= 0) { + return false; + } + + ggml_gtx_kernel_launch_desc_t base = {}; + if (!gtx_build_kernel_launch_desc(node, base)) { + return false; + } + + // The L2-only kernel emits one packed dot-product sequence per output. + // Keep that command stream at or below the largest focused shape already + // verified on Spike. This is a host macro tile: firmware still only maps + // raw views, and the kernel retains its own SPU-level output partitioning. + constexpr uint64_t max_outputs_per_tile = 256u * 1024u; + const uint64_t outputs_per_channel = + (uint64_t)base.ne[0] * base.ne[1] * base.ne[3]; + if (outputs_per_channel == 0u || outputs_per_channel > max_outputs_per_tile) { + return false; + } + const uint32_t max_channels = (uint32_t)std::max( + 1u, max_outputs_per_tile / outputs_per_channel); + + out_descs.clear(); + for (uint32_t c0 = 0; c0 < base.ne[2]; c0 += max_channels) { + const uint32_t channels = std::min(max_channels, base.ne[2] - c0); + ggml_gtx_kernel_launch_desc_t tile = base; + tile.dst_addr += (uint64_t)c0 * base.nb[2]; + tile.ne[2] = channels; + tile.src_addr[0] += (uint64_t)c0 * base.src_nb[0][2]; + tile.src_ne[0][2] = channels; + out_descs.push_back(tile); + } + return !out_descs.empty(); +} + +static bool gtx_scheduler_lower_spatial_scalar_rhs_node( + const ggml_tensor * node, + std::vector & out_descs) { + if (node == nullptr || node->src[0] == nullptr || node->src[1] == nullptr || + (gtx_kernel_id_from_node(node) != GGML_GTX_KERNEL_ADD_F16 && + gtx_kernel_id_from_node(node) != GGML_GTX_KERNEL_MUL_F16) || + !gtx_l2_rhs_broadcast_shape_ok(node, true) || + node->src[1]->ne[0] != 1 || node->ne[3] != 1 || + node->src[1]->ne[1] != 1 || node->src[1]->ne[3] != 1) { + return false; + } + + ggml_gtx_kernel_launch_desc_t base = {}; + if (!gtx_build_kernel_launch_desc(node, base)) { + return false; + } + const uint64_t data_budget = gtx_stash_data_budget(node); + const uint64_t slot_budget = (data_budget / GTX_STASH_SLOT_COUNT) & ~31ull; + const uint32_t width = base.ne[0]; + const uint32_t height = base.ne[1]; + const uint32_t channels_total = base.ne[2]; + const auto aligned = [](uint64_t bytes) { return (bytes + 31u) & ~31ull; }; + const auto fits = [&](uint32_t rows, uint32_t channels) { + const uint64_t data_bytes = aligned((uint64_t)width * rows * channels * sizeof(uint16_t)); + const uint64_t rhs_channels = base.src_ne[1][2] == 1u ? 1u : channels; + const uint64_t rhs_bytes = aligned(rhs_channels * sizeof(uint16_t)); + return data_bytes > 0 && data_bytes <= slot_budget && rhs_bytes <= slot_budget && + data_bytes * 2u + rhs_bytes <= data_budget; + }; + + out_descs.clear(); + for (uint32_t c0 = 0; c0 < channels_total;) { + uint32_t channels = channels_total - c0; + while (channels > 0 && !fits(1u, channels)) { + channels = channels == 1u ? 0u : (channels + 1u) / 2u; + } + if (channels == 0u) { + out_descs.clear(); + return false; + } + for (uint32_t y0 = 0; y0 < height;) { + uint32_t rows = height - y0; + while (rows > 0 && !fits(rows, channels)) { + rows = rows == 1u ? 0u : (rows + 1u) / 2u; + } + if (rows == 0u) { + out_descs.clear(); + return false; + } + ggml_gtx_kernel_launch_desc_t tile = base; + const uint64_t data_off = (uint64_t)y0 * base.nb[1] + (uint64_t)c0 * base.nb[2]; + tile.dst_addr += data_off; + tile.src_addr[0] += data_off; + tile.ne[1] = rows; + tile.ne[2] = channels; + tile.src_ne[0][1] = rows; + tile.src_ne[0][2] = channels; + tile.src_ne[1][1] = 1u; + if (base.src_ne[1][2] != 1u) { + tile.src_addr[1] += (uint64_t)c0 * base.src_nb[1][2]; + tile.src_ne[1][2] = channels; + } + out_descs.push_back(tile); + y0 += rows; + } + c0 += channels; + } + return !out_descs.empty(); +} + +static bool gtx_scheduler_lower_spatial_dense_node( + const ggml_tensor * node, + std::vector & out_descs) { + if (node == nullptr || node->src[0] == nullptr || node->ne[3] != 1 || + !gtx_tensor_remote_f16_dense(node) || + !gtx_tensor_same_shape(node, node->src[0])) { + return false; + } + const uint32_t kernel_id = gtx_kernel_id_from_node(node); + const bool supported = + kernel_id == GGML_GTX_KERNEL_ADD_F16 || kernel_id == GGML_GTX_KERNEL_MUL_F16 || + kernel_id == GGML_GTX_KERNEL_SCALE_F16 || kernel_id == GGML_GTX_KERNEL_GELU_F16 || + kernel_id == GGML_GTX_KERNEL_ABS_F16 || kernel_id == GGML_GTX_KERNEL_NEG_F16 || + kernel_id == GGML_GTX_KERNEL_SGN_F16 || kernel_id == GGML_GTX_KERNEL_CEIL_F16 || + kernel_id == GGML_GTX_KERNEL_FLOOR_F16 || kernel_id == GGML_GTX_KERNEL_TANH_F16 || + kernel_id == GGML_GTX_KERNEL_SILU_F16 || kernel_id == GGML_GTX_KERNEL_RELU_F16 || + kernel_id == GGML_GTX_KERNEL_LEAKY_RELU_F16 || kernel_id == GGML_GTX_KERNEL_CLAMP_F16; + if (!supported) { + return false; + } + for (uint32_t i = 0; i < 4u; ++i) { + const ggml_tensor * src = node->src[i]; + if (src == nullptr) { + continue; + } + if (!gtx_tensor_remote_f16_dense(src) || src->ne[3] != 1) { + return false; + } + for (uint32_t d = 0; d < 3u; ++d) { + if (src->ne[d] != 1 && src->ne[d] != node->ne[d]) { + return false; + } + } + } + + ggml_gtx_kernel_launch_desc_t base = {}; + if (!gtx_build_kernel_launch_desc(node, base)) { + return false; + } + const uint64_t data_budget = gtx_stash_data_budget(node); + const uint64_t slot_budget = (data_budget / GTX_STASH_SLOT_COUNT) & ~31ull; + const uint32_t width = base.ne[0]; + const uint32_t height = base.ne[1]; + const uint32_t channels_total = base.ne[2]; + // Unary/scale uploaded kernels consume one flat vector through ne[0]. + // ADD/MUL are deliberately excluded: their kernels retain the 4-D shape + // to implement per-dimension broadcasting. + const bool flat_kernel = gtx_scheduler_row_tile_kernel_is_flat(kernel_id) && + kernel_id != GGML_GTX_KERNEL_ADD_F16 && + kernel_id != GGML_GTX_KERNEL_MUL_F16; + const auto aligned = [](uint64_t bytes) { return (bytes + 31u) & ~31ull; }; + const auto tile_bytes = [&](const ggml_tensor * tensor, uint32_t rows, uint32_t channels) { + const uint64_t w = tensor->ne[0] == 1 ? 1u : width; + const uint64_t h = tensor->ne[1] == 1 ? 1u : rows; + const uint64_t c = tensor->ne[2] == 1 ? 1u : channels; + return aligned(w * h * c * sizeof(uint16_t)); + }; + const auto fits = [&](uint32_t rows, uint32_t channels) { + const uint64_t logical_elems = (uint64_t)width * rows * channels; + if (flat_kernel && logical_elems > GTX_FLAT_KERNEL_MAX_ELEMS) { + return false; + } + uint64_t total = aligned((uint64_t)width * rows * channels * sizeof(uint16_t)); + if (total == 0 || total > slot_budget) { + return false; + } + for (uint32_t i = 0; i < 4u; ++i) { + if (node->src[i] == nullptr) { + continue; + } + const uint64_t bytes = tile_bytes(node->src[i], rows, channels); + if (bytes == 0 || bytes > slot_budget || total > UINT64_MAX - bytes) { + return false; + } + total += bytes; + } + return total <= data_budget; + }; + + // 평평한 1-D 입력은 아래 공간 타일러가 쪼갤 행도 채널도 없다. `ne[0]` 을 직접 + // 잘라 두 뷰의 주소를 함께 전진시킨다. + // ⚠️ **이 경로가 SCALE 에만 열려 있었다.** 같은 ABI 를 쓰는 relu·clamp·leaky_relu· + // neg 등은 1-D 가 커지면 그대로 한 번에 나가 조용히 틀렸다 + // (실측: 32769 원소부터. `gtx_smoke --limits`). flat 커널 전체로 넓힌다. + if (flat_kernel && + height == 1u && channels_total == 1u && width > GTX_FLAT_KERNEL_MAX_ELEMS) { + constexpr uint32_t max_aligned_f16_elems = (uint32_t) GTX_FLAT_KERNEL_MAX_ELEMS & ~15u; + out_descs.clear(); + for (uint32_t x0 = 0; x0 < width;) { + uint32_t elems = std::min(width - x0, max_aligned_f16_elems); + while (elems > 0u) { + const uint64_t bytes = aligned((uint64_t)elems * sizeof(uint16_t)); + if (bytes <= slot_budget && bytes <= data_budget / 2u) { + break; + } + elems /= 2u; + } + if (elems == 0u) { + out_descs.clear(); + return false; + } + + ggml_gtx_kernel_launch_desc_t tile = base; + tile.dst_addr += (uint64_t)x0 * base.nb[0]; + tile.src_addr[0] += (uint64_t)x0 * base.src_nb[0][0]; + gtx_scheduler_make_dense_tile_shape(tile, elems, 1u); + gtx_scheduler_make_dense_src_tile_shape(tile, 0u, elems, 1u); + out_descs.push_back(tile); + x0 += elems; + } + return !out_descs.empty(); + } + + out_descs.clear(); + for (uint32_t c0 = 0; c0 < channels_total;) { + uint32_t channels = channels_total - c0; + if (flat_kernel) { + const uint64_t plane_elems = (uint64_t)width * height; + // A flattened descriptor must also describe a physically + // contiguous raw DDR view. Use complete channel planes when a + // plane fits the flat-kernel ABI; otherwise restrict the tile to + // one channel and split that plane by rows below. + if (plane_elems <= GTX_FLAT_KERNEL_MAX_ELEMS) { + channels = std::min( + channels, (uint32_t)(UINT16_MAX / plane_elems)); + } else { + channels = 1u; + } + } + while (channels > 0 && !fits(1u, channels)) { + channels = channels == 1u ? 0u : (channels + 1u) / 2u; + } + if (channels == 0u) { + out_descs.clear(); + return false; + } + for (uint32_t y0 = 0; y0 < height;) { + uint32_t rows = height - y0; + while (rows > 0 && !fits(rows, channels)) { + rows = rows == 1u ? 0u : (rows + 1u) / 2u; + } + if (rows == 0u) { + out_descs.clear(); + return false; + } + ggml_gtx_kernel_launch_desc_t tile = base; + tile.dst_addr += (uint64_t)y0 * base.nb[1] + (uint64_t)c0 * base.nb[2]; + tile.ne[1] = rows; + tile.ne[2] = channels; + for (uint32_t i = 0; i < 4u; ++i) { + const ggml_tensor * src = node->src[i]; + if (src == nullptr) { + continue; + } + if (src->ne[1] != 1) { + tile.src_addr[i] += (uint64_t)y0 * base.src_nb[i][1]; + tile.src_ne[i][1] = rows; + } + if (src->ne[2] != 1) { + tile.src_addr[i] += (uint64_t)c0 * base.src_nb[i][2]; + tile.src_ne[i][2] = channels; + } + } + if (flat_kernel) { + const uint64_t logical_elems = (uint64_t)width * rows * channels; + if (logical_elems == 0u || logical_elems > UINT16_MAX) { + out_descs.clear(); + return false; + } + gtx_scheduler_make_dense_tile_shape(tile, (uint32_t)logical_elems, 1u); + for (uint32_t i = 0; i < 4u; ++i) { + if (node->src[i] != nullptr && tile.src_addr[i] != 0u) { + gtx_scheduler_make_dense_src_tile_shape( + tile, i, (uint32_t)logical_elems, 1u); + } + } + } + out_descs.push_back(tile); + y0 += rows; + } + c0 += channels; + } + return !out_descs.empty(); +} + +/* Lower a single L2-only kernel without relying on firmware-side identity. + * This is also used when the multi-node scheduler is disabled: the host still + * has to provide a raw L2 plan for an uploaded L2-only executable. */ +static bool gtx_scheduler_lower_single_l2_node(const ggml_tensor * node, + std::vector & out_descs) { + if (node == nullptr) { + return false; + } + + const uint32_t kernel_id = gtx_kernel_id_from_node(node); + if ((kernel_id == GGML_GTX_KERNEL_CONV_2D_F16 || + kernel_id == GGML_GTX_KERNEL_CONV_2D_DW_F16) && + gtx_scheduler_lower_conv_l2_node(node, out_descs)) { + return true; + } + if (kernel_id == GGML_GTX_KERNEL_CONV_TRANSPOSE_2D_F16 && + gtx_scheduler_lower_conv_transpose_l2_node(node, out_descs)) { + return true; + } + if (gtx_scheduler_lower_spatial_dense_node(node, out_descs)) { + return true; + } + if (gtx_scheduler_lower_spatial_scalar_rhs_node(node, out_descs)) { + return true; + } + + std::vector one = { const_cast(node) }; + if (gtx_scheduler_tileable_row_node(node) && + gtx_scheduler_lower_row_island(one, out_descs)) { + return true; + } + + // PAD has only two simultaneously-live tensor views (src0 and dst). + // Do not infer a fixed per-slot capacity from the generic three-view + // budget: the host does not own the firmware stash allocator or know its + // current residency/fragmentation. Admit the whole raw view when its + // aggregate live bytes fit; firmware remains authoritative for mapping, + // eviction, and reclaim/reset retry. + if (kernel_id == GGML_GTX_KERNEL_PAD_F16 && node->src[0] != nullptr) { + const uint64_t data_budget = gtx_stash_data_budget(node); + const uint64_t dst_bytes = gtx_remote_size_bytes(node); + const uint64_t src_bytes = gtx_remote_size_bytes(node->src[0]); + const uint64_t scratch_bytes = gtx_kernel_l2_scratch_size(node); + ggml_gtx_kernel_launch_desc_t desc = {}; + if (dst_bytes > 0u && src_bytes > 0u && + dst_bytes <= data_budget && src_bytes <= data_budget && + scratch_bytes <= data_budget && + dst_bytes <= UINT64_MAX - src_bytes && + dst_bytes + src_bytes <= UINT64_MAX - scratch_bytes && + dst_bytes + src_bytes + scratch_bytes <= data_budget && + gtx_build_kernel_launch_desc(node, desc)) { + out_descs = { desc }; + return true; + } + return false; + } + + // Binary broadcast is a kernel-side ABI feature. Keep host lowering + // generic: submit one raw macro view and preserve src1's smaller shape; + // stash packs it and the uploaded ADD/MUL kernel selects the broadcast + // index from src_ne[1], without host-side row launches. + const uint64_t slot_budget = (gtx_stash_data_budget(node) / GTX_STASH_SLOT_COUNT) & ~31ull; + if ((kernel_id == GGML_GTX_KERNEL_REPEAT_F16 || + kernel_id == GGML_GTX_KERNEL_CONCAT_F16 || + kernel_id == GGML_GTX_KERNEL_CONV_TRANSPOSE_2D_F16 || + kernel_id == GGML_GTX_KERNEL_CONV_2D_F16 || + kernel_id == GGML_GTX_KERNEL_CONV_2D_DW_F16 || + kernel_id == GGML_GTX_KERNEL_SIN_F16 || + kernel_id == GGML_GTX_KERNEL_COS_F16)) { + const uint64_t dst_bytes = gtx_remote_size_bytes(node); + uint64_t total_bytes = dst_bytes; + bool fits = dst_bytes > 0u && dst_bytes <= slot_budget; + for (uint32_t i = 0; i < 4u && fits; ++i) { + if (node->src[i] == nullptr) { + continue; + } + const uint64_t src_bytes = gtx_remote_size_bytes(node->src[i]); + if (src_bytes == 0u || src_bytes > slot_budget || + total_bytes > UINT64_MAX - src_bytes) { + fits = false; + break; + } + total_bytes += src_bytes; + } + ggml_gtx_kernel_launch_desc_t desc = {}; + if (fits && total_bytes <= gtx_stash_data_budget(node) && + gtx_build_kernel_launch_desc(node, desc)) { + out_descs = { desc }; + return true; + } + return false; + } + if ((kernel_id == GGML_GTX_KERNEL_ADD_F16 || + kernel_id == GGML_GTX_KERNEL_MUL_F16) && + node->src[0] != nullptr && node->src[1] != nullptr && + gtx_tensor_remote_f16_vector_contiguous(node) && + gtx_tensor_remote_f16_vector_contiguous(node->src[0]) && + gtx_tensor_remote_f16_vector_contiguous(node->src[1]) && + gtx_tensor_same_shape(node, node->src[0]) && + (node->src[1]->ne[0] == node->ne[0] || node->src[1]->ne[0] == 1) && + (uint64_t) gtx_remote_size_bytes(node) <= slot_budget && + (uint64_t) gtx_remote_size_bytes(node->src[0]) <= slot_budget && + (uint64_t) gtx_remote_size_bytes(node->src[1]) <= slot_budget) { + bool broadcastable = true; + for (int d = 1; d < GGML_MAX_DIMS; ++d) { + broadcastable = broadcastable && + (node->src[1]->ne[d] == 1 || node->src[1]->ne[d] == node->ne[d]); + } + ggml_gtx_kernel_launch_desc_t desc = {}; + if (broadcastable && gtx_build_kernel_launch_desc(node, desc)) { + out_descs = { desc }; + return true; + } + } + + if (!gtx_scheduler_tileable_col_node(node)) { + return false; + } + if (kernel_id == GGML_GTX_KERNEL_MUL_MAT_F16) { + return gtx_scheduler_lower_col_island(one, out_descs); + } + const int64_t cols64 = node->ne[1] * node->ne[2] * node->ne[3]; + if (cols64 <= 0 || cols64 > UINT32_MAX) { + return false; + } + + const uint32_t total_cols = (uint32_t)cols64; + // Standalone ROPE used to force one descriptor per flattened column + // (effectively one launch per head and token). Coalesce columns within + // the existing ne1/layout and stash-budget limits. Keep the legacy + // single-column behavior for the other standalone column kernels. + const uint32_t max_cols_per_tile = + kernel_id == GGML_GTX_KERNEL_ROPE_F16 ? + gtx_scheduler_max_cols_per_tile(one, total_cols) : 1u; + if (max_cols_per_tile == 0u) { + return false; + } + + out_descs.clear(); + for (uint32_t col = 0; col < total_cols;) { + uint32_t cols = std::min(max_cols_per_tile, total_cols - col); + while (cols > 1u && + !gtx_scheduler_col_tile_contiguous_for_node(node, col, cols)) { + --cols; + } + ggml_gtx_kernel_launch_desc_t desc = {}; + if (!gtx_scheduler_col_tile_contiguous_for_node(node, col, cols) || + !gtx_scheduler_build_col_tile_desc(node, col, cols, desc)) { + out_descs.clear(); + return false; + } + out_descs.push_back(desc); + col += cols; + } + return !out_descs.empty(); +} + +static enum ggml_status gtx_compute_node_host_driven(ggml_backend_gtx_context * ctx, + ggml_tensor * node, + int node_index, + bool request_stash_chain = false) { + if (node == nullptr || gtx_is_metadata_op(node->op)) { + return GGML_STATUS_SUCCESS; + } + + if (ggml_nelements(node) == 0) { + return GGML_STATUS_SUCCESS; + } + + gtx_log_op_shapes(node); + if (node->op == GGML_OP_SOFT_MAX) { + gtx_log_softmax_tensor_stats(ctx, node->src[0], "pre-src0"); + gtx_log_softmax_tensor_stats(ctx, node->src[1], "pre-mask"); + } + + if (node->op == GGML_OP_CPY && !ctx->use_uart) { + if (node->src[0] == nullptr || !gtx_copy_tensor_rpc(ctx->conn.get(), node->src[0], node)) { + ctx->last_error = "COPY_TENSOR failed"; + return GGML_STATUS_FAILED; + } + return GGML_STATUS_SUCCESS; + } + + ggml_gtx_kernel_launch_desc_t desc = {}; + if (gtx_build_kernel_launch_desc(node, desc)) { + const gtx_l2_chainability chain = gtx_l2_chainability_from_node(node); + if (chain.kind == gtx_l2_chain_kind::native || + desc.kernel_id == GGML_GTX_KERNEL_MUL_MAT_F16 || + desc.kernel_id == GGML_GTX_KERNEL_CONV_TRANSPOSE_2D_F16) { + std::vector l2_descs; + if (gtx_scheduler_lower_single_l2_node(node, l2_descs)) { + for (ggml_gtx_kernel_launch_desc_t & l2_desc : l2_descs) { + l2_desc.flags &= ~GGML_GTX_DESC_FLAG_DDR_ALLOWED; + l2_desc.flags |= GGML_GTX_DESC_FLAG_STASH_CHAIN | + GGML_GTX_DESC_FLAG_STASH_HOST_TILED | + GGML_GTX_DESC_FLAG_L2_REQUIRED; + } + gtx_dump_op_sample_before_launch(ctx, node); + const gtx_op_check_pre_state check_pre = gtx_check_ops_before_launch(ctx, node); + if (gtx_submit_launch_descs(ctx, l2_descs)) { + if (!gtx_diagnostic_materialize_stash(ctx, node)) { + ctx->last_error = "diagnostic stash flush/reset failed"; + return GGML_STATUS_FAILED; + } + gtx_diagnostic_report_first_nonfinite(ctx, node); + gtx_check_mul_mat_after_launch(ctx, node); + gtx_check_ops_after_launch(ctx, node, check_pre); + gtx_dump_op_sample_after_launch(ctx, node); + if (gtx_cpu_ref_exists(node_index) && ctx->use_uart && ctx->uart != nullptr) { + if (!ctx->uart->stash_flush_reset()) { + ctx->last_error = "CPU reference stash materialization failed"; + return GGML_STATUS_FAILED; + } + (void)gtx_compare_cpu_reference_after_launch(ctx, node, node_index); + } + return GGML_STATUS_SUCCESS; + } + ctx->last_error = "L2-only descriptor stream launch failed"; + return GGML_STATUS_FAILED; + } + GTX_LOG_ERROR("host cannot lower L2-only kernel into raw L2 views: %s " + "dst=[%lld,%lld,%lld,%lld] src0=[%lld,%lld,%lld,%lld] " + "src1=[%lld,%lld,%lld,%lld]\n", + gtx_tensor_op_display_name(node), + (long long) node->ne[0], (long long) node->ne[1], + (long long) node->ne[2], (long long) node->ne[3], + node->src[0] ? (long long) node->src[0]->ne[0] : 0, + node->src[0] ? (long long) node->src[0]->ne[1] : 0, + node->src[0] ? (long long) node->src[0]->ne[2] : 0, + node->src[0] ? (long long) node->src[0]->ne[3] : 0, + node->src[1] ? (long long) node->src[1]->ne[0] : 0, + node->src[1] ? (long long) node->src[1]->ne[1] : 0, + node->src[1] ? (long long) node->src[1]->ne[2] : 0, + node->src[1] ? (long long) node->src[1]->ne[3] : 0); + ctx->last_error = "L2-only kernel lacks host raw-view lowering"; + return GGML_STATUS_FAILED; + } + if (request_stash_chain) { + desc.flags |= GGML_GTX_DESC_FLAG_STASH_CHAIN; + } + gtx_dump_op_sample_before_launch(ctx, node); + const gtx_op_check_pre_state check_pre = gtx_check_ops_before_launch(ctx, node); + if (ctx->use_uart ? gtx_uart_launch_desc(*ctx->uart, desc) : gtx_submit_launch_desc(ctx, desc)) { + if (!gtx_diagnostic_materialize_stash(ctx, node)) { + ctx->last_error = "diagnostic stash flush/reset failed"; + return GGML_STATUS_FAILED; + } + gtx_diagnostic_report_first_nonfinite(ctx, node); + gtx_check_mul_mat_after_launch(ctx, node); + gtx_check_ops_after_launch(ctx, node, check_pre); + if (node->op == GGML_OP_SOFT_MAX) { + gtx_log_softmax_tensor_stats(ctx, node, "post-dst"); + } + gtx_dump_op_sample_after_launch(ctx, node); + if (gtx_cpu_ref_exists(node_index) && ctx->use_uart && ctx->uart != nullptr) { + if (!ctx->uart->stash_flush_reset()) { + ctx->last_error = "CPU reference stash materialization failed"; + return GGML_STATUS_FAILED; + } + (void)gtx_compare_cpu_reference_after_launch(ctx, node, node_index); + } + return GGML_STATUS_SUCCESS; + } + } + + GTX_LOG_ERROR("host-driven graph failed op: %s (%d)\n", + gtx_tensor_op_display_name(node), (int) node->op); + ctx->last_error = "Graph op launch failed"; + return GGML_STATUS_FAILED; +} + +static enum ggml_status gtx_compute_graph_tiling_scheduler(ggml_backend_gtx_context * ctx, + ggml_cgraph * cgraph) { + if (ctx == nullptr || cgraph == nullptr) { + return GGML_STATUS_FAILED; + } + + for (int i = 0; i < cgraph->n_nodes;) { + ggml_tensor * node = cgraph->nodes[i]; + if (gtx_scheduler_node_skippable(node)) { + ++i; + continue; + } + + int island_end = i; + std::vector island_nodes; + std::vector descs; + if (gtx_scheduler_find_col_island(cgraph, i, island_end, island_nodes) && + gtx_scheduler_lower_col_island(island_nodes, descs)) { + for (ggml_gtx_kernel_launch_desc_t & desc : descs) { + // The host has already lowered the matrix view into one + // contiguous L2 tile with its raw shape/stride metadata. + // The kernel, not firmware, owns the ensuing L1/SPU split. + desc.flags &= ~GGML_GTX_DESC_FLAG_DDR_ALLOWED; + desc.flags |= GGML_GTX_DESC_FLAG_STASH_CHAIN | + GGML_GTX_DESC_FLAG_STASH_HOST_TILED | + GGML_GTX_DESC_FLAG_L2_REQUIRED; + } + if (gtx_log_chain_ops_enabled()) { + const long long total_cols = + (long long)island_nodes[0]->ne[1] * + (long long)island_nodes[0]->ne[2] * + (long long)island_nodes[0]->ne[3]; + const size_t groups = island_nodes.empty() ? 0 : descs.size() / island_nodes.size(); + const size_t rem = island_nodes.empty() ? 0 : descs.size() % island_nodes.size(); + fprintf(stderr, + "[GTX TILING] col-island nodes=%zu descs=%zu groups=%zu rem=%zu cols=%lld start=%d end=%d\n", + island_nodes.size(), + descs.size(), + groups, + rem, + total_cols, + i, + island_end); + } + if (!gtx_submit_launch_descs(ctx, descs)) { + ctx->last_error = "Tiled descriptor stream launch failed"; + return GGML_STATUS_FAILED; + } + bool has_reference = false; + for (int node_index = i; node_index < island_end; ++node_index) { + has_reference = has_reference || gtx_cpu_ref_exists(node_index); + } + if (has_reference && ctx->use_uart && ctx->uart != nullptr) { + if (!ctx->uart->stash_flush_reset()) { + ctx->last_error = "CPU reference island materialization failed"; + return GGML_STATUS_FAILED; + } + for (int node_index = i; node_index < island_end; ++node_index) { + if (gtx_cpu_ref_exists(node_index)) { + (void)gtx_compare_cpu_reference_after_launch( + ctx, cgraph->nodes[node_index], node_index); + } + } + } + i = island_end; + continue; + } + + if (gtx_scheduler_find_row_island(cgraph, i, island_end, island_nodes) && + gtx_scheduler_lower_row_island(island_nodes, descs)) { + for (ggml_gtx_kernel_launch_desc_t & desc : descs) { + desc.flags &= ~GGML_GTX_DESC_FLAG_DDR_ALLOWED; + desc.flags |= GGML_GTX_DESC_FLAG_STASH_CHAIN | + GGML_GTX_DESC_FLAG_STASH_HOST_TILED | + GGML_GTX_DESC_FLAG_L2_REQUIRED; + } + if (gtx_log_chain_ops_enabled()) { + fprintf(stderr, + "[GTX TILING] row-island nodes=%zu descs=%zu rows=%lld ne0=%lld start=%d end=%d\n", + island_nodes.size(), + descs.size(), + (long long)island_nodes[0]->ne[1], + (long long)island_nodes[0]->ne[0], + i, + island_end); + } + if (!gtx_submit_launch_descs(ctx, descs)) { + ctx->last_error = "Tiled descriptor stream launch failed"; + return GGML_STATUS_FAILED; + } + bool has_reference = false; + for (int node_index = i; node_index < island_end; ++node_index) { + has_reference = has_reference || gtx_cpu_ref_exists(node_index); + } + if (has_reference && ctx->use_uart && ctx->uart != nullptr) { + if (!ctx->uart->stash_flush_reset()) { + ctx->last_error = "CPU reference island materialization failed"; + return GGML_STATUS_FAILED; + } + for (int node_index = i; node_index < island_end; ++node_index) { + if (gtx_cpu_ref_exists(node_index)) { + (void)gtx_compare_cpu_reference_after_launch( + ctx, cgraph->nodes[node_index], node_index); + } + } + } + i = island_end; + continue; + } + + enum ggml_status status = gtx_compute_node_host_driven(ctx, node, i); + if (status != GGML_STATUS_SUCCESS) { + return status; + } + + ++i; + } + + return GGML_STATUS_SUCCESS; +} + +static ggml_backend_buffer_type_i ggml_backend_gtx_buffer_type_interface = { + /* .get_name = */ ggml_backend_gtx_buffer_type_get_name, + /* .alloc_buffer = */ ggml_backend_gtx_buffer_type_alloc_buffer, + /* .get_alignment = */ ggml_backend_gtx_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_gtx_buffer_type_get_max_size, + /* .get_alloc_size= */ ggml_backend_gtx_buffer_type_get_alloc_size, + /* .is_host = */ ggml_backend_gtx_buffer_type_is_host, +}; + +/*============================================================================= + * Buffer Implementation + *============================================================================*/ + +static void ggml_backend_gtx_buffer_free_buffer(ggml_backend_buffer_t buffer) { + ggml_backend_gtx_buffer_context* ctx = (ggml_backend_gtx_buffer_context*)buffer->context; + + if (ctx->conn && ctx->remote_ptr != 0) { + gtx_msg_free_buffer_req req = { ctx->remote_ptr }; + ctx->conn->send_cmd(GTX_CMD_FREE_BUFFER, &req, sizeof(req), nullptr, 0); + } + if (ctx->backend_ctx != nullptr && ctx->remote_ptr != 0) { + ctx->backend_ctx->allocated_buffers.erase(ctx->remote_ptr); + } + + delete ctx; +} + +static void* ggml_backend_gtx_buffer_get_base(ggml_backend_buffer_t buffer) { + ggml_backend_gtx_buffer_context* ctx = (ggml_backend_gtx_buffer_context*)buffer->context; + + /* Return remote pointer as base (will be used as offset) */ + return (void*)(uintptr_t)ctx->remote_ptr; +} + +static void ggml_backend_gtx_buffer_memset_tensor( + ggml_backend_buffer_t buffer, struct ggml_tensor* tensor, + uint8_t value, size_t offset, size_t size) { + + ggml_backend_gtx_buffer_context* ctx = (ggml_backend_gtx_buffer_context*)buffer->context; + GGML_UNUSED(size); + + if (ctx->backend_ctx != nullptr && ctx->backend_ctx->use_uart) { + std::vector zeros(gtx_remote_byte_offset(tensor, size), value); + ctx->backend_ctx->uart->upload_bytes(gtx_remote_data_ptr(tensor) + gtx_remote_byte_offset(tensor, offset), + zeros.data(), zeros.size()); + } else if (ctx->conn) { + gtx_msg_buffer_clear_req req; + req.remote_ptr = (uint64_t)(uintptr_t)tensor->data + offset; + req.value = value; + ctx->conn->send_cmd(GTX_CMD_BUFFER_CLEAR, &req, sizeof(req), nullptr, 0); + } +} + +static void ggml_backend_gtx_buffer_set_tensor( + ggml_backend_buffer_t buffer, struct ggml_tensor* tensor, + const void* data, size_t offset, size_t size) { + + ggml_backend_gtx_buffer_context* ctx = (ggml_backend_gtx_buffer_context*)buffer->context; + + GTX_LOG("set_tensor: %s, offset=%zu, size=%zu\n", tensor->name, offset, size); + + if (ctx->backend_ctx != nullptr && ctx->backend_ctx->use_uart) { + const size_t chunk_size = gtx_upload_chunk_size(); + size_t sent = 0; + + while (sent < size) { + size_t chunk = gtx_upload_local_chunk_size(tensor, size - sent, chunk_size); + std::vector packed = gtx_pack_tensor_chunk(tensor, (const uint8_t *) data + sent, chunk); + uint64_t remote = gtx_remote_data_ptr(tensor) + gtx_remote_byte_offset(tensor, offset + sent); + if (!ctx->backend_ctx->uart->upload_bytes(remote, packed.data(), packed.size())) { + GTX_LOG_ERROR("Failed to upload tensor over UART\n"); + return; + } + sent += chunk; + } + + if (offset + size <= ctx->shadow_buffer.size()) { + memcpy(ctx->shadow_buffer.data() + offset, data, size); + } + } else if (ctx->conn) { + const size_t chunk_size = gtx_upload_chunk_size() - sizeof(gtx_rpc_tensor); + size_t sent = 0; + + while (sent < size) { + size_t chunk = gtx_upload_local_chunk_size(tensor, size - sent, chunk_size); + gtx_rpc_tensor rpc_tensor = gtx_serialize_tensor(tensor); + rpc_tensor.data = gtx_remote_data_ptr(tensor) + gtx_remote_byte_offset(tensor, offset + sent); + + std::vector packed = gtx_pack_tensor_chunk(tensor, (const uint8_t *) data + sent, chunk); + + std::vector msg_data(sizeof(gtx_rpc_tensor) + packed.size()); + memcpy(msg_data.data(), &rpc_tensor, sizeof(gtx_rpc_tensor)); + memcpy(msg_data.data() + sizeof(gtx_rpc_tensor), packed.data(), packed.size()); + + if (!ctx->conn->send_data(GTX_BAR2_STAGING_OFFSET, msg_data.data(), msg_data.size()) || + !ctx->conn->send_cmd_staged(GTX_CMD_SET_TENSOR, + GTX_BAR2_DEVICE_BASE + GTX_BAR2_STAGING_OFFSET, + (uint32_t) msg_data.size(), nullptr, 0)) { + GTX_LOG_ERROR("Failed to stage tensor upload\n"); + return; + } + + sent += chunk; + } + + if (offset + size <= ctx->shadow_buffer.size()) { + memcpy(ctx->shadow_buffer.data() + offset, data, size); + } + } +} + +static void ggml_backend_gtx_buffer_get_tensor( + ggml_backend_buffer_t buffer, const struct ggml_tensor* tensor, + void* data, size_t offset, size_t size) { + + ggml_backend_gtx_buffer_context* ctx = (ggml_backend_gtx_buffer_context*)buffer->context; + + GTX_LOG("get_tensor: %s, offset=%zu, size=%zu\n", tensor->name, offset, size); + + if (ctx->backend_ctx != nullptr && ctx->backend_ctx->use_uart) { + const size_t chunk_size = 16ULL * 1024ULL; + size_t received = 0; + + while (received < size) { + size_t chunk = std::min(chunk_size, size - received); + const size_t remote_size = (size_t) gtx_remote_byte_offset(tensor, chunk); + std::vector packed(remote_size); + uint64_t remote = gtx_remote_data_ptr(tensor) + gtx_remote_byte_offset(tensor, offset + received); + if (!ctx->backend_ctx->uart->download_bytes(remote, packed.data(), packed.size())) { + GTX_LOG_ERROR("Failed to download tensor over UART\n"); + return; + } + gtx_unpack_tensor_chunk(tensor, packed.data(), (uint8_t *) data + received, chunk); + received += chunk; + } + + if (offset + size <= ctx->shadow_buffer.size()) { + memcpy(ctx->shadow_buffer.data() + offset, data, size); + } + } else if (ctx->conn) { + const size_t chunk_size = GTX_BAR2_DOWNLOAD_CHUNK; + size_t received = 0; + + while (received < size) { + size_t chunk = std::min(chunk_size, size - received); + gtx_msg_get_tensor_req req = {}; + req.tensor = gtx_serialize_tensor(tensor); + req.offset = gtx_remote_byte_offset(tensor, offset + received); + req.size = gtx_remote_byte_offset(tensor, chunk); + + std::vector packed(req.size); + + if (!ctx->conn->send_data(GTX_BAR2_STAGING_OFFSET, &req, sizeof(req)) || + !ctx->conn->send_cmd_staged(GTX_CMD_GET_TENSOR, + GTX_BAR2_DEVICE_BASE + GTX_BAR2_STAGING_OFFSET, + (uint32_t) sizeof(req), nullptr, 0) || + !ctx->conn->recv_data(GTX_BAR2_STAGING_OFFSET, + packed.data(), packed.size())) { + GTX_LOG_ERROR("Failed to stage tensor download\n"); + return; + } + + gtx_unpack_tensor_chunk(tensor, packed.data(), (uint8_t *) data + received, chunk); + + received += chunk; + } + + if (offset + size <= ctx->shadow_buffer.size()) { + memcpy(ctx->shadow_buffer.data() + offset, data, size); + } + } +} + +static void ggml_backend_gtx_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { + ggml_backend_gtx_buffer_context* ctx = (ggml_backend_gtx_buffer_context*)buffer->context; + + if (ctx->backend_ctx != nullptr && ctx->backend_ctx->use_uart) { + std::vector zeros(ctx->size, value); + ctx->backend_ctx->uart->upload_bytes(ctx->remote_ptr, zeros.data(), zeros.size()); + } else if (ctx->conn) { + gtx_msg_buffer_clear_req req; + req.remote_ptr = ctx->remote_ptr; + req.value = value; + ctx->conn->send_cmd(GTX_CMD_BUFFER_CLEAR, &req, sizeof(req), nullptr, 0); + } + + /* Clear shadow buffer too */ + memset(ctx->shadow_buffer.data(), value, ctx->shadow_buffer.size()); +} + +static bool ggml_backend_gtx_buffer_cpy_tensor( + ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) { + ggml_backend_gtx_buffer_context * ctx = + (ggml_backend_gtx_buffer_context *) buffer->context; + + if (ctx == nullptr) { + return false; + } + + if (ctx->backend_ctx != nullptr && ctx->backend_ctx->use_uart) { + return ctx->backend_ctx->uart != nullptr && + ctx->backend_ctx->uart->copy_tensor(src, dst); + } + + if (ctx->conn == nullptr) { + return false; + } + + return gtx_copy_tensor_rpc(ctx->conn, src, dst); +} + +static ggml_backend_buffer_i ggml_backend_gtx_buffer_interface = { + /* .free_buffer = */ ggml_backend_gtx_buffer_free_buffer, + /* .get_base = */ ggml_backend_gtx_buffer_get_base, + /* .init_tensor = */ nullptr, + /* .memset_tensor = */ ggml_backend_gtx_buffer_memset_tensor, + /* .set_tensor = */ ggml_backend_gtx_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_gtx_buffer_get_tensor, + /* .cpy_tensor = */ ggml_backend_gtx_buffer_cpy_tensor, + /* .clear = */ ggml_backend_gtx_buffer_clear, + /* .reset = */ nullptr, +}; + +static ggml_backend_buffer_t ggml_backend_gtx_buffer_type_alloc_buffer( + ggml_backend_buffer_type_t buft, size_t size) { + + ggml_backend_gtx_context* backend_ctx = (ggml_backend_gtx_context*)buft->context; + + if (backend_ctx->use_uart) { + if (!backend_ctx->uart->open()) { + GTX_LOG_ERROR("Failed to open %s stream connection for buffer allocation\n", + gtx_selected_transport()); + return nullptr; + } + const size_t aligned_offset = gtx_align_up(backend_ctx->uart_arena_next, 64); + const size_t aligned_size = gtx_align_up(size, 64); + const uint64_t arena_size = gtx_fixed_tensor_arena_size(); + if (aligned_offset > arena_size || + aligned_size > arena_size - aligned_offset) { + GTX_LOG_ERROR("UART VP fixed tensor arena too small: requested=%zu used=%zu arena=%llu\n", + size, + aligned_offset, + (unsigned long long) arena_size); + return nullptr; + } + const uint64_t remote_ptr = gtx_fixed_tensor_arena_addr() + aligned_offset; + backend_ctx->uart_arena_next = aligned_offset + aligned_size; + backend_ctx->allocated_buffers[remote_ptr] = size; + + if (gtx_log_ops_enabled()) { + fprintf(stderr, "[GTX ALLOC] uart buffer addr=0x%llx size=%zu next=%zu arena=%llu\n", + (unsigned long long) remote_ptr, + size, + backend_ctx->uart_arena_next, + (unsigned long long) gtx_fixed_tensor_arena_size()); + } + + ggml_backend_gtx_buffer_context* ctx = new ggml_backend_gtx_buffer_context(); + ctx->device_id = backend_ctx->device_id; + ctx->remote_ptr = remote_ptr; + ctx->size = size; + ctx->conn = nullptr; + ctx->backend_ctx = backend_ctx; + ctx->shadow_buffer.resize(size); + return ggml_backend_buffer_init(buft, ggml_backend_gtx_buffer_interface, ctx, size); + } + + if (!ggml_backend_gtx_open_device_connection(backend_ctx->conn.get(), backend_ctx->device_id)) { + GTX_LOG_ERROR("Failed to open GTX connection for buffer allocation\n"); + return nullptr; + } + + /* Allocate buffer on device */ + gtx_msg_alloc_buffer_req req; + req.device = backend_ctx->device_id; + req.size = size; + + gtx_msg_alloc_buffer_rsp rsp = {}; + + GTX_LOG("ALLOC_BUFFER request: device=%d size=%zu\n", backend_ctx->device_id, size); + + if (!backend_ctx->conn->send_cmd(GTX_CMD_ALLOC_BUFFER, &req, sizeof(req), &rsp, sizeof(rsp))) { + GTX_LOG_ERROR("Failed to allocate buffer: size=%zu\n", size); + return nullptr; + } + + GTX_LOG("ALLOC_BUFFER response: remote_ptr=0x%llx remote_size=%llu\n", + (unsigned long long) rsp.remote_ptr, + (unsigned long long) rsp.remote_size); + + if (rsp.remote_ptr == 0) { + GTX_LOG_ERROR("Device returned null pointer for buffer allocation: size=%zu remote_size=%llu\n", + size, + (unsigned long long) rsp.remote_size); + return nullptr; + } + + GTX_LOG("Allocated buffer: ptr=0x%llx, size=%llu\n", + (unsigned long long)rsp.remote_ptr, + (unsigned long long)rsp.remote_size); + + /* Create buffer context */ + ggml_backend_gtx_buffer_context* ctx = new ggml_backend_gtx_buffer_context(); + ctx->device_id = backend_ctx->device_id; + ctx->remote_ptr = rsp.remote_ptr; + ctx->size = rsp.remote_size; + ctx->conn = backend_ctx->conn.get(); + ctx->backend_ctx = backend_ctx; + ctx->shadow_buffer.resize(size); + + return ggml_backend_buffer_init(buft, ggml_backend_gtx_buffer_interface, ctx, size); +} + +/*============================================================================= + * Backend Implementation + *============================================================================*/ + +static const char* ggml_backend_gtx_get_name(ggml_backend_t backend) { + ggml_backend_gtx_context* ctx = (ggml_backend_gtx_context*)backend->context; + static char name[64]; + snprintf(name, sizeof(name), "GTX%d", ctx->device_id); + return name; +} + +static void ggml_backend_gtx_free(ggml_backend_t backend) { + ggml_backend_gtx_context* ctx = (ggml_backend_gtx_context*)backend->context; + delete ctx; + delete backend; +} + +void ggml_backend_gtx_synchronize(ggml_backend_t backend) { + ggml_backend_gtx_context* ctx = (ggml_backend_gtx_context*)backend->context; + + /* Wait for all pending operations using IRQ if available */ + gtx_pcie_irq_ctx* irq_ctx = ctx->get_irq_ctx(); + if (irq_ctx) { + /* Poll and process any pending MSIs */ + gtx_pcie_irq_process(irq_ctx); + } +} + +static enum ggml_status ggml_backend_gtx_graph_compute(ggml_backend_t backend, struct ggml_cgraph* cgraph) { + ggml_backend_gtx_context* ctx = (ggml_backend_gtx_context*)backend->context; + gtx_cpu_ref_current_graph = gtx_cpu_ref_graph_sequence++; + + GTX_LOG("graph_compute: %d nodes\n", cgraph->n_nodes); + gtx_dump_graph_node_types(cgraph); + gtx_log_graph_chainability(cgraph); + + auto start = std::chrono::steady_clock::now(); + + /* Host interprets ggml graph nodes and submits descriptor/copy RPCs. */ + if (ctx->use_uart ? ctx->uart->is_open() : ctx->conn->is_open()) { + if (gtx_tiling_scheduler_enabled() && !gtx_tiling_scheduler_diagnostics_active()) { + enum ggml_status status = gtx_compute_graph_tiling_scheduler(ctx, cgraph); + if (status != GGML_STATUS_SUCCESS) { + return status; + } + } else { + for (int i = 0; i < cgraph->n_nodes; i++) { + enum ggml_status status = gtx_compute_node_host_driven(ctx, cgraph->nodes[i], i); + if (status != GGML_STATUS_SUCCESS) { + return status; + } + } + } + } else { + GTX_LOG_ERROR("GTX connection not open\n"); + ctx->last_error = "GTX connection not open"; + return GGML_STATUS_FAILED; + } + + auto end = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast(end - start).count(); + + ctx->stats.graph_computes++; + ctx->stats.total_compute_us += elapsed; + + return GGML_STATUS_SUCCESS; +} + +static ggml_backend_i ggml_backend_gtx_interface = { + /* .get_name = */ ggml_backend_gtx_get_name, + /* .free = */ ggml_backend_gtx_free, + /* .set_tensor_async = */ nullptr, + /* .get_tensor_async = */ nullptr, + /* .cpy_tensor_async = */ nullptr, + /* .synchronize = */ ggml_backend_gtx_synchronize, + /* .graph_plan_create = */ nullptr, + /* .graph_plan_free = */ nullptr, + /* .graph_plan_update = */ nullptr, + /* .graph_plan_compute = */ nullptr, + /* .graph_compute = */ ggml_backend_gtx_graph_compute, + /* .event_record = */ nullptr, + /* .event_wait = */ nullptr, + /* .graph_optimize = */ nullptr, +}; + +/*============================================================================= + * GUID + *============================================================================*/ + +static ggml_guid_t ggml_backend_gtx_guid() { + static ggml_guid guid = { + 0x47, 0x54, 0x58, 0x00, /* 'GTX\0' */ + 0x4e, 0x50, 0x55, 0x00, /* 'NPU\0' */ + 0x42, 0x41, 0x43, 0x4b, /* 'BACK' */ + 0x45, 0x4e, 0x44, 0x00 /* 'END\0' */ + }; + return &guid; +} + +/*============================================================================= + * Device Implementation + *============================================================================*/ + +struct ggml_backend_gtx_device_context { + int device_id; + std::string description; + std::shared_ptr conn; + uint8_t firmware_version[3]; + + ggml_backend_gtx_device_context(int id) : device_id(id) { + char desc[128]; + snprintf(desc, sizeof(desc), "GTX NPU Device %d", id); + description = desc; + conn = ggml_backend_gtx_shared_connection(id); + memset(firmware_version, 0, sizeof(firmware_version)); + } +}; + +static const char * ggml_backend_gtx_device_get_name(ggml_backend_dev_t dev) { + ggml_backend_gtx_device_context * ctx = (ggml_backend_gtx_device_context *)dev->context; + static char name[64]; + snprintf(name, sizeof(name), "GTX%d", ctx->device_id); + return name; +} + +static const char * ggml_backend_gtx_device_get_description(ggml_backend_dev_t dev) { + ggml_backend_gtx_device_context * ctx = (ggml_backend_gtx_device_context *)dev->context; + return ctx->description.c_str(); +} + +static void ggml_backend_gtx_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { + (void)dev; + if (gtx_use_xdma_stream_transport()) { + *total = GTX_FPGA_TENSOR_ARENA_SIZE; + *free = GTX_FPGA_TENSOR_ARENA_SIZE; + } else { + *total = 16ULL * 1024 * 1024 * 1024; + *free = 15ULL * 1024 * 1024 * 1024; + } +} + +static enum ggml_backend_dev_type ggml_backend_gtx_device_get_type(ggml_backend_dev_t dev) { + (void)dev; + return GGML_BACKEND_DEVICE_TYPE_GPU; +} + +static void ggml_backend_gtx_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) { + props->name = ggml_backend_gtx_device_get_name(dev); + props->description = ggml_backend_gtx_device_get_description(dev); + props->type = ggml_backend_gtx_device_get_type(dev); + ggml_backend_gtx_device_get_memory(dev, &props->memory_free, &props->memory_total); + props->caps = { + /* .async = */ false, + /* .host_buffer = */ false, + /* .buffer_from_host_ptr = */ false, + /* .events = */ false, + }; +} + +static ggml_backend_t ggml_backend_gtx_device_init_backend(ggml_backend_dev_t dev, const char * params) { + (void)params; + ggml_backend_gtx_device_context * dev_ctx = (ggml_backend_gtx_device_context *)dev->context; + int device_id = dev_ctx->device_id; + + /* Create backend context */ + ggml_backend_gtx_context* ctx = new ggml_backend_gtx_context(device_id); + fprintf(stderr, "[GTX] selected transport=%s stream_path=%s\n", + gtx_selected_transport(), ctx->use_uart ? "yes" : "no"); + /* Share PCIe connection from device context */ + ctx->conn = dev_ctx->conn; + + /* Open PCIe connection */ + char device_path[64]; +#ifdef _WIN32 + snprintf(device_path, sizeof(device_path), "\\\\.\\gtx%d", device_id); +#else + snprintf(device_path, sizeof(device_path), "/dev/gtx%d", device_id); +#endif + + if (ctx->use_uart) { + if (!ctx->uart->open()) { + GTX_LOG_ERROR("Failed to open %s stream transport\n", + gtx_selected_transport()); + } else { + GTX_LOG("%s stream transport opened successfully\n", + gtx_selected_transport()); + } + } else if (!ctx->conn->open(device_path)) { + GTX_LOG_ERROR("Failed to open PCIe connection\n"); + /* Continue anyway - may work in simulation mode */ + } else { + GTX_LOG("PCIe connection opened successfully\n"); + } + + /* Send HELLO to verify connection */ + gtx_msg_hello_rsp hello_rsp = {}; + if (!ctx->use_uart && ctx->conn->is_open()) { + if (ctx->conn->send_cmd(GTX_CMD_HELLO, nullptr, 0, &hello_rsp, sizeof(hello_rsp))) { + ctx->firmware_version[0] = hello_rsp.major; + ctx->firmware_version[1] = hello_rsp.minor; + ctx->firmware_version[2] = hello_rsp.patch; + GTX_LOG("Connected to firmware v%d.%d.%d\n", + hello_rsp.major, hello_rsp.minor, hello_rsp.patch); + } + } + + /* Create backend */ + ggml_backend_t backend = new ggml_backend(); + backend->guid = ggml_backend_gtx_guid(); + backend->iface = ggml_backend_gtx_interface; + backend->device = dev; + backend->context = ctx; + + return backend; +} + +static ggml_backend_buffer_type_t ggml_backend_gtx_device_get_buffer_type(ggml_backend_dev_t dev) { + ggml_backend_gtx_device_context * ctx = (ggml_backend_gtx_device_context *)dev->context; + return ggml_backend_gtx_buffer_type(ctx->device_id); +} + +static bool ggml_backend_gtx_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { + (void)dev; + + if (op == nullptr) { + return false; + } + + auto finish = [&](bool supported, const char * reason) { + gtx_log_supports_op_decision(op, supported, reason); + return supported; + }; + + const uint32_t normalized_op = gtx_normalize_host_op(op); + const char * op_name = gtx_normalized_op_name(normalized_op); + + if (gtx_env_list_contains("GTX_DISABLE_OPS", op_name) || + gtx_env_substr_matches("GTX_DISABLE_NAME_SUBSTR", op->name)) { + return finish(false, "disabled_by_env"); + } + if (normalized_op == (uint32_t) GGML_OP_ADD && + gtx_env_substr_matches("GTX_DISABLE_ADD_NAME_SUBSTR", op->name)) { + return finish(false, "disabled_add_by_name_env"); + } + if (gtx_is_add_f16_f32_vector_to_f16(op)) { + return finish(true, "add_f16_f32_vector_to_f16"); + } + if (gtx_is_add_f16_bias_vector_basic(op)) { + return finish(true, "add_f16_bias_vector_basic"); + } + if (normalized_op == (uint32_t) GGML_OP_CONT && + !gtx_is_cont_same_shape(op)) { + return finish(false, "cont_shape_change_cpu_fallback"); + } + if (normalized_op == (uint32_t) GGML_OP_GLU) { + return finish(gtx_is_swiglu_node(op), "glu_swiglu_only"); + } + + if (op->op == GGML_OP_SCALE) { + const bool supported = op->src[0] != nullptr && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && + // ⚠️ **타일러가 4차원 flat 커널을 아직 못 쪼갠다.** + // 1차원 경로는 2026-09-09 에 고쳤고 `gtx_smoke --limits` + // 가 131072 까지 통과한다. 그러나 모델의 4차원 relu + // (`[112,112,64,1]` 등)는 다른 lowering 경로를 타고 + // 여전히 한 번에 나간다 — ResNet-18 argmax 282 → 912. + // 그 경로를 고칠 때까지 큰 것은 CPU 로 보낸다. + ggml_nelements(op) <= 32768; + return finish(supported, supported ? "scale_f32_f16" : "scale_type_guard"); + } + + if (op->op == GGML_OP_NORM) { + const uint64_t rows64 = (uint64_t) op->ne[1] * (uint64_t) op->ne[2] * (uint64_t) op->ne[3]; + const bool supported = op->src[0] != nullptr && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && + gtx_remote_tensor_type(op) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[0]) == GGML_TYPE_F16 && + op->ne[0] > 0 && + op->ne[0] <= UINT16_MAX && + rows64 > 0 && + rows64 <= UINT16_MAX && + ggml_is_contiguous(op) && + ggml_is_contiguous(op->src[0]); + return finish(supported, supported ? "norm_f32_f16_dense" : "norm_type_or_layout_guard"); + } + + if (gtx_is_gelu_node(op)) { + const bool supported = op->src[0] != nullptr && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && + ggml_is_contiguous(op) && + ggml_is_contiguous(op->src[0]); + return finish(supported, supported ? "gelu_f32_f16_dense" : "gelu_type_or_layout_guard"); + } + + if (op->op == GGML_OP_SIN) { + const uint64_t elems64 = (uint64_t) op->ne[0] * (uint64_t) op->ne[1] * + (uint64_t) op->ne[2] * (uint64_t) op->ne[3]; + const bool supported = op->src[0] != nullptr && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && + gtx_remote_tensor_type(op) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[0]) == GGML_TYPE_F16 && + elems64 > 0 && + (elems64 % 16u) == 0 && + ggml_is_contiguous(op) && + ggml_is_contiguous(op->src[0]); + return finish(supported, supported ? "sin_f16_reference" : "sin_reference_guard"); + } + + if (op->op == GGML_OP_COS) { + const uint64_t elems64 = (uint64_t) op->ne[0] * (uint64_t) op->ne[1] * + (uint64_t) op->ne[2] * (uint64_t) op->ne[3]; + const bool supported = op->src[0] != nullptr && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && + gtx_remote_tensor_type(op) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[0]) == GGML_TYPE_F16 && + elems64 > 0 && + (elems64 % 16u) == 0 && + ggml_is_contiguous(op) && + ggml_is_contiguous(op->src[0]); + return finish(supported, supported ? "cos_f16_l2" : "cos_l2_guard"); + } + + if (op->op == GGML_OP_UNARY) { + const ggml_unary_op uop = (ggml_unary_op) op->op_params[0]; + const bool p2_unary = + uop == GGML_UNARY_OP_ABS || + uop == GGML_UNARY_OP_NEG || + uop == GGML_UNARY_OP_SGN || + uop == GGML_UNARY_OP_CEIL || + uop == GGML_UNARY_OP_FLOOR || + uop == GGML_UNARY_OP_TANH || + uop == GGML_UNARY_OP_SILU || + uop == GGML_UNARY_OP_RELU; + const bool supported = p2_unary && + op->src[0] != nullptr && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && + gtx_remote_tensor_type(op) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[0]) == GGML_TYPE_F16 && + ggml_is_contiguous(op) && + ggml_is_contiguous(op->src[0]) && + // ⚠️ **타일러가 4차원 flat 커널을 아직 못 쪼갠다.** + // 1차원 경로는 2026-09-09 에 고쳤고 `gtx_smoke --limits` + // 가 131072 까지 통과한다. 그러나 모델의 4차원 relu + // (`[112,112,64,1]` 등)는 다른 lowering 경로를 타고 + // 여전히 한 번에 나간다 — ResNet-18 argmax 282 → 912. + // 그 경로를 고칠 때까지 큰 것은 CPU 로 보낸다. + ggml_nelements(op) <= 32768; + return finish(supported, supported ? "unary_p2_f16_reference" : "unary_p2_reference_guard"); + } + + if (op->op == GGML_OP_LEAKY_RELU) { + const bool supported = op->src[0] != nullptr && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && + gtx_remote_tensor_type(op) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[0]) == GGML_TYPE_F16 && + ggml_is_contiguous(op) && + ggml_is_contiguous(op->src[0]) && + // ⚠️ **타일러가 4차원 flat 커널을 아직 못 쪼갠다.** + // 1차원 경로는 2026-09-09 에 고쳤고 `gtx_smoke --limits` + // 가 131072 까지 통과한다. 그러나 모델의 4차원 relu + // (`[112,112,64,1]` 등)는 다른 lowering 경로를 타고 + // 여전히 한 번에 나간다 — ResNet-18 argmax 282 → 912. + // 그 경로를 고칠 때까지 큰 것은 CPU 로 보낸다. + ggml_nelements(op) <= 32768; + return finish(supported, supported ? "leaky_relu_f16_reference" : "leaky_relu_reference_guard"); + } + + if (normalized_op == (uint32_t) GGML_OP_CLAMP) { + const bool supported = op->src[0] != nullptr && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && + gtx_remote_tensor_type(op) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[0]) == GGML_TYPE_F16 && + ggml_is_contiguous(op) && + ggml_is_contiguous(op->src[0]) && + // ⚠️ **타일러가 4차원 flat 커널을 아직 못 쪼갠다.** + // 1차원 경로는 2026-09-09 에 고쳤고 `gtx_smoke --limits` + // 가 131072 까지 통과한다. 그러나 모델의 4차원 relu + // (`[112,112,64,1]` 등)는 다른 lowering 경로를 타고 + // 여전히 한 번에 나간다 — ResNet-18 argmax 282 → 912. + // 그 경로를 고칠 때까지 큰 것은 CPU 로 보낸다. + ggml_nelements(op) <= 32768; + return finish(supported, supported ? "clamp_f16_reference" : "clamp_reference_guard"); + } + + if (op->op == GGML_OP_REPEAT) { + constexpr uint64_t gtx_repeat_max_bytes = 4ull * 1024ull * 1024ull; + bool multiples = op->src[0] != nullptr; + for (int i = 0; multiples && i < GGML_MAX_DIMS; ++i) { + multiples = op->src[0]->ne[i] > 0 && + op->ne[i] >= op->src[0]->ne[i] && + (op->ne[i] % op->src[0]->ne[i]) == 0 && + op->ne[i] <= UINT16_MAX && + op->src[0]->ne[i] <= UINT16_MAX; + } + const uint64_t bytes = (uint64_t) ggml_nbytes(op); + const bool supported = op->src[0] != nullptr && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && + gtx_remote_tensor_type(op) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[0]) == GGML_TYPE_F16 && + multiples && + ggml_is_contiguous(op) && + ggml_is_contiguous(op->src[0]) && + bytes > 0 && + bytes <= gtx_repeat_max_bytes; + return finish(supported, supported ? "repeat_f32_f16_dense" : "repeat_type_or_shape_guard"); + } + + if (op->op == GGML_OP_CONCAT) { + constexpr uint64_t gtx_concat_max_bytes = 4ull * 1024ull * 1024ull; + const int32_t dim = ((const int32_t *) op->op_params)[0]; + bool shape_ok = op->src[0] != nullptr && op->src[1] != nullptr && + dim >= 0 && dim < GGML_MAX_DIMS; + for (int i = 0; shape_ok && i < GGML_MAX_DIMS; ++i) { + const int64_t expected = i == dim ? + op->src[0]->ne[i] + op->src[1]->ne[i] : op->src[0]->ne[i]; + shape_ok = op->src[0]->ne[i] <= UINT16_MAX && + op->src[1]->ne[i] <= UINT16_MAX && + op->ne[i] <= UINT16_MAX && + (i == dim || op->src[0]->ne[i] == op->src[1]->ne[i]) && + op->ne[i] == expected; + } + const uint64_t bytes = (uint64_t) ggml_nbytes(op); + const bool supported = op->src[0] != nullptr && + op->src[1] != nullptr && + op->src[0]->type == op->src[1]->type && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && + gtx_remote_tensor_type(op) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[0]) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[1]) == GGML_TYPE_F16 && + shape_ok && + ggml_is_contiguous(op) && + ggml_is_contiguous(op->src[0]) && + ggml_is_contiguous(op->src[1]) && + bytes > 0 && + bytes <= gtx_concat_max_bytes; + return finish(supported, supported ? "concat_f32_f16_dense" : "concat_type_or_shape_guard"); + } + + if (op->op == GGML_OP_IM2COL) { + const int32_t * params = (const int32_t *) op->op_params; + const bool is_2D = params[6] == 1; + constexpr int64_t gtx_im2col_1d_max_output_bytes = 512 * 1024; + constexpr int64_t gtx_im2col_2d_max_output_bytes = 4 * 1024 * 1024; + const int64_t iw = op->src[1] != nullptr ? op->src[1]->ne[0] : 0; + const int64_t ih = op->src[1] != nullptr ? op->src[1]->ne[1] : 0; + const int64_t ic = op->src[1] != nullptr ? op->src[1]->ne[2] : 0; + const int64_t n = op->src[1] != nullptr ? op->src[1]->ne[3] : 0; + const int64_t kw = op->src[0] != nullptr ? op->src[0]->ne[0] : 0; + const int64_t kh = op->src[0] != nullptr ? op->src[0]->ne[1] : 0; + const int64_t w_in_pad = iw + (int64_t) params[2] * 2; + const int64_t h_in_pad = ih + (int64_t) params[3] * 2; + const int64_t k_eff_w = (kw - 1) * (int64_t) params[4] + 1; + const int64_t k_eff_h = (kh - 1) * (int64_t) params[5] + 1; + const int64_t w_out = (params[0] > 0 && w_in_pad >= k_eff_w) ? + (w_in_pad - k_eff_w) / params[0] + 1 : 0; + const int64_t h_out = (is_2D && params[1] > 0 && h_in_pad >= k_eff_h) ? + (h_in_pad - k_eff_h) / params[1] + 1 : 1; + const int64_t output_bytes = w_out * h_out * ic * kw * kh * n * (int64_t) ggml_type_size(op->type); + + bool supported = false; + const char * reason = "im2col_guard"; + if (op->src[0] != nullptr && op->src[1] != nullptr && + op->src[1]->type == GGML_TYPE_F32 && + op->type == GGML_TYPE_F16 && + params[0] > 0 && + params[2] >= 0 && params[2] <= 255 && + params[4] > 0 && + output_bytes > 0) { + if (!is_2D) { + supported = op->src[0]->ne[1] == op->src[1]->ne[1] && + op->src[1]->ne[3] == 1 && + kw <= 255 && + output_bytes <= gtx_im2col_1d_max_output_bytes; + reason = supported ? "im2col_1d" : "im2col_1d_guard"; + } else { + supported = op->src[0]->ne[2] == op->src[1]->ne[2] && + kh == kw && + params[1] == params[0] && + params[3] == params[2] && + params[5] == params[4] && + kw <= 255 && + output_bytes <= gtx_im2col_2d_max_output_bytes; + reason = supported ? "im2col_2d_square" : "im2col_2d_guard"; + } + } + return finish(supported, reason); + } + + if (op->op == GGML_OP_CONV_2D) { + const bool supported = op->src[0] != nullptr && + op->src[1] != nullptr && + op->src[0]->ne[2] == op->src[1]->ne[2] && + op->src[0]->ne[0] > 0 && op->src[0]->ne[1] > 0 && + op->src[0]->ne[2] > 0 && op->src[0]->ne[3] > 0 && + op->src[1]->ne[0] > 0 && op->src[1]->ne[1] > 0 && + op->src[1]->ne[3] > 0 && + op->src[1]->ne[3] == 1 && + op->ne[3] == 1 && + op->src[0]->ne[0] == op->src[0]->ne[1] && + op->op_params[0] > 0 && op->op_params[1] > 0 && + op->op_params[4] > 0 && op->op_params[5] > 0 && + op->op_params[0] == op->op_params[1] && + op->op_params[4] == op->op_params[5] && + op->op_params[2] >= 0 && op->op_params[3] >= 0 && + op->src[0]->ne[0] <= UINT16_MAX && + op->src[0]->ne[1] <= UINT16_MAX && + op->src[0]->ne[2] <= UINT16_MAX && + op->src[0]->ne[3] <= UINT16_MAX && + op->src[1]->ne[0] <= UINT16_MAX && + op->src[1]->ne[1] <= UINT16_MAX && + op->src[1]->ne[2] <= UINT16_MAX && + op->src[1]->ne[3] <= UINT16_MAX && + op->ne[0] <= UINT16_MAX && + op->ne[1] <= UINT16_MAX && + op->ne[2] <= UINT16_MAX && + op->ne[3] <= UINT16_MAX && + gtx_remote_tensor_type(op) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[0]) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[1]) == GGML_TYPE_F16 && + gtx_l2_conv_layout_ok(op) && + ggml_nbytes(op) > 0; + return finish(supported, supported ? "conv_2d_f16_reference" : "conv_2d_reference_guard"); + } + + if (op->op == GGML_OP_CONV_2D_DW) { + const bool supported = op->src[0] != nullptr && + op->src[1] != nullptr && + op->src[0]->ne[2] == 1 && + op->src[0]->ne[3] == op->src[1]->ne[2] && + op->ne[2] == op->src[1]->ne[2] && + op->ne[3] == op->src[1]->ne[3] && + op->src[1]->ne[3] == 1 && + op->ne[3] == 1 && + op->src[0]->ne[0] == op->src[0]->ne[1] && + op->op_params[0] > 0 && op->op_params[1] > 0 && + op->op_params[4] > 0 && op->op_params[5] > 0 && + op->op_params[0] == op->op_params[1] && + op->op_params[4] == op->op_params[5] && + op->op_params[2] >= 0 && op->op_params[3] >= 0 && + op->src[0]->ne[0] <= UINT16_MAX && + op->src[0]->ne[1] <= UINT16_MAX && + op->src[0]->ne[3] <= UINT16_MAX && + op->src[1]->ne[0] <= UINT16_MAX && + op->src[1]->ne[1] <= UINT16_MAX && + op->src[1]->ne[2] <= UINT16_MAX && + op->src[1]->ne[3] <= UINT16_MAX && + op->ne[0] <= UINT16_MAX && + op->ne[1] <= UINT16_MAX && + op->ne[2] <= UINT16_MAX && + op->ne[3] <= UINT16_MAX && + gtx_remote_tensor_type(op) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[0]) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[1]) == GGML_TYPE_F16 && + gtx_l2_conv_layout_ok(op) && + ggml_nbytes(op) > 0; + return finish(supported, supported ? "conv_2d_dw_f16_reference" : "conv_2d_dw_reference_guard"); + } + + if (normalized_op == (uint32_t) GGML_OP_CONV_TRANSPOSE_2D) { + const int64_t dst_bytes = (int64_t) gtx_remote_size_bytes(op); + const int64_t input_bytes = op->src[1] != nullptr ? (int64_t) gtx_remote_size_bytes(op->src[1]) : 0; + const int64_t kernel_bytes = op->src[0] != nullptr ? (int64_t) gtx_remote_size_bytes(op->src[0]) : 0; + const int32_t stride = op->op_params[0]; + const int64_t kw = op->src[0] != nullptr ? op->src[0]->ne[0] : 0; + const int64_t kh = op->src[0] != nullptr ? op->src[0]->ne[1] : 0; + const int64_t oc = op->src[0] != nullptr ? op->src[0]->ne[2] : 0; + const int64_t ic = op->src[0] != nullptr ? op->src[0]->ne[3] : 0; + const int64_t iw = op->src[1] != nullptr ? op->src[1]->ne[0] : 0; + const int64_t ih = op->src[1] != nullptr ? op->src[1]->ne[1] : 0; + const int64_t batch = op->src[1] != nullptr ? op->src[1]->ne[3] : 0; + const bool shape_ok = op->src[0] != nullptr && + op->src[1] != nullptr && + stride > 0 && + kw > 0 && kh > 0 && oc > 0 && ic > 0 && + iw > 0 && ih > 0 && batch > 0 && + op->src[1]->ne[2] == ic && + op->ne[0] == (iw - 1) * stride + kw && + op->ne[1] == (ih - 1) * stride + kh && + op->ne[2] == oc && + op->ne[3] == batch && + kw * kh * ic <= 32768 && + op->ne[0] <= UINT16_MAX && + op->src[1]->ne[0] <= UINT16_MAX && + op->src[0]->ne[0] <= UINT16_MAX && + op->ne[1] * op->ne[2] * op->ne[3] <= UINT16_MAX && + op->src[1]->ne[1] * op->src[1]->ne[2] * op->src[1]->ne[3] <= UINT16_MAX && + op->src[0]->ne[1] * op->src[0]->ne[2] * op->src[0]->ne[3] <= UINT16_MAX; + const bool l2_ok = input_bytes > 0 && + kernel_bytes > 0 && + dst_bytes > 0 && + kernel_bytes <= 2 * 1024 * 1024 && + input_bytes <= 6 * 1024 * 1024 && + dst_bytes <= 8 * 1024 * 1024; + const bool supported = op->src[0] != nullptr && + op->src[1] != nullptr && + gtx_remote_tensor_type(op) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[0]) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[1]) == GGML_TYPE_F16 && + ggml_is_contiguous(op) && + ggml_is_contiguous(op->src[0]) && + ggml_is_contiguous(op->src[1]) && + shape_ok && + l2_ok; + return finish(supported, supported ? "conv_transpose_2d_f16_reference" : "conv_transpose_2d_reference_guard"); + } + + if (normalized_op == (uint32_t) GGML_OP_UPSCALE) { + const uint32_t mode_flags = (uint32_t) op->op_params[0]; + const uint32_t mode = mode_flags & 0xffu; + const bool scale_mode_ok = mode == GGML_SCALE_MODE_NEAREST || + mode == GGML_SCALE_MODE_BILINEAR || + mode == GGML_SCALE_MODE_BICUBIC; + const bool antialias = (mode_flags & GGML_SCALE_FLAG_ANTIALIAS) != 0; + const int64_t dst_bytes = (int64_t) gtx_remote_size_bytes(op); + const bool supported = op->src[0] != nullptr && + scale_mode_ok && + !antialias && + gtx_remote_tensor_type(op) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[0]) == GGML_TYPE_F16 && + op->src[0]->ne[0] > 0 && + op->src[0]->ne[1] > 0 && + op->src[0]->ne[2] > 0 && + op->src[0]->ne[3] > 0 && + op->ne[0] > 0 && + op->ne[1] > 0 && + op->ne[2] > 0 && + op->ne[3] > 0 && + op->ne[2] == op->src[0]->ne[2] && + op->ne[3] == op->src[0]->ne[3] && + op->src[0]->ne[0] <= UINT16_MAX && + op->src[0]->ne[1] <= UINT16_MAX && + op->src[0]->ne[2] <= UINT16_MAX && + op->src[0]->ne[3] <= UINT16_MAX && + op->ne[0] <= UINT16_MAX && + op->ne[1] <= UINT16_MAX && + op->ne[2] <= UINT16_MAX && + op->ne[3] <= UINT16_MAX && + dst_bytes > 0 && + (uint64_t) dst_bytes <= GTX_STASH_L2_BYTES; + return finish(supported, supported ? "upscale_f16_resize" : "upscale_resize_guard"); + } + + if (normalized_op == (uint32_t) GGML_OP_PAD) { + const int32_t * p = (const int32_t *) op->op_params; + const int64_t remote_bytes = (int64_t) gtx_remote_size_bytes(op); + const bool shape_ok = op->src[0] != nullptr && + p[0] == 0 && p[2] == 0 && p[4] == 0 && p[6] == 0 && + p[1] >= 0 && p[3] >= 0 && p[5] >= 0 && p[7] == 0 && + p[8] == 0 && + op->ne[0] == op->src[0]->ne[0] + p[1] && + op->ne[1] == op->src[0]->ne[1] + p[3] && + op->ne[2] == op->src[0]->ne[2] + p[5] && + op->ne[3] == op->src[0]->ne[3] && + op->ne[0] <= UINT16_MAX && + op->ne[1] <= UINT16_MAX && + op->src[0]->ne[0] <= UINT16_MAX && + op->src[0]->ne[1] <= UINT16_MAX; + const bool supported = op->src[0] != nullptr && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && + gtx_remote_tensor_type(op) == GGML_TYPE_F16 && + gtx_remote_tensor_type(op->src[0]) == GGML_TYPE_F16 && + shape_ok && + ggml_is_contiguous(op) && + ggml_is_contiguous(op->src[0]) && + remote_bytes > 0 && + remote_bytes <= 14 * 1024 * 1024; + return finish(supported, supported ? "pad_f16_reference" : "pad_reference_guard"); + } + + switch (normalized_op) { + case GGML_OP_NONE: + case GGML_OP_CPY: + case GGML_OP_CONT: + case GGML_OP_RESHAPE: + case GGML_OP_VIEW: + case GGML_OP_PERMUTE: + case GGML_OP_TRANSPOSE: + case GGML_OP_ADD: + case GGML_OP_REPEAT: + case GGML_OP_CONCAT: + case GGML_OP_MUL: + case GGML_OP_CONV_2D: + case GGML_OP_CONV_2D_DW: + case GGML_OP_CONV_TRANSPOSE_2D: + case GGML_OP_MUL_MAT: + case GGML_OP_UPSCALE: + case GGML_OP_PAD: + case GGML_OP_LEAKY_RELU: + case GGML_OP_CLAMP: + case GGML_OP_SOFT_MAX: + case GGML_OP_NORM: + case GGML_OP_RMS_NORM: + case GGML_OP_GET_ROWS: + case GGML_OP_SET_ROWS: + case GGML_OP_ROPE: + return finish(true, "supported_descriptor_or_metadata"); + default: + return finish(false, "unsupported_op"); + } +} + +static bool ggml_backend_gtx_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { + ggml_backend_gtx_device_context * ctx = (ggml_backend_gtx_device_context *)dev->context; + return buft == ggml_backend_gtx_buffer_type(ctx->device_id); +} + +static bool ggml_backend_gtx_device_offload_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { + if (!gtx_env_truthy("GTX_ENABLE_OFFLOAD_OP")) { + return false; + } + return ggml_backend_gtx_device_supports_op(dev, op); +} + +static const struct ggml_backend_device_i ggml_backend_gtx_device_i = { + /* .get_name = */ ggml_backend_gtx_device_get_name, + /* .get_description = */ ggml_backend_gtx_device_get_description, + /* .get_memory = */ ggml_backend_gtx_device_get_memory, + /* .get_type = */ ggml_backend_gtx_device_get_type, + /* .get_props = */ ggml_backend_gtx_device_get_props, + /* .init_backend = */ ggml_backend_gtx_device_init_backend, + /* .get_buffer_type = */ ggml_backend_gtx_device_get_buffer_type, + /* .get_host_buffer_type= */ nullptr, + /* .buffer_from_host_ptr= */ nullptr, + /* .supports_op = */ ggml_backend_gtx_device_supports_op, + /* .supports_buft = */ ggml_backend_gtx_device_supports_buft, + /* .offload_op = */ ggml_backend_gtx_device_offload_op, + /* .event_new = */ nullptr, + /* .event_free = */ nullptr, + /* .event_synchronize = */ nullptr, +}; + +/*============================================================================= + * Registry Implementation + *============================================================================*/ + +static const char * ggml_backend_gtx_reg_get_name(ggml_backend_reg_t reg) { + (void)reg; + return "GTX"; +} + +static size_t ggml_backend_gtx_reg_get_device_count(ggml_backend_reg_t reg) { + (void)reg; + return 1; /* TODO: Enumerate actual devices */ +} + +static ggml_backend_dev_t ggml_backend_gtx_reg_get_device(ggml_backend_reg_t reg, size_t index) { + static std::vector> device_contexts; + static std::vector devices; + static bool initialized = false; + + if (!initialized) { + size_t count = ggml_backend_gtx_reg_get_device_count(reg); + + /* First, create all device contexts */ + for (size_t i = 0; i < count; i++) { + device_contexts.push_back(std::make_unique((int)i)); + } + + /* Then, create devices with stable pointers */ + devices.reserve(count); + for (size_t i = 0; i < count; i++) { + devices.push_back({ + /* .iface = */ ggml_backend_gtx_device_i, + /* .reg = */ reg, + /* .context = */ device_contexts[i].get(), + }); + } + + initialized = true; + } + + if (index >= devices.size()) { + return nullptr; + } + return &devices[index]; +} + +static ggml_backend_feature * ggml_backend_gtx_get_features(ggml_backend_reg_t reg) { + (void)reg; + static std::vector features = { + { "GTX_DESCRIPTOR_GRAPH", "1" }, + { "GTX_STASH_L2_CHAINING", "1" }, + { "GTX_STASH_L2_CHAIN_NATIVE_OPS", + "ADD,MUL,SCALE,MUL_MAT,ROPE,SOFT_MAX,RMS_NORM,SWIGLU,GELU,RELU,LEAKY_RELU,CLAMP,ABS,NEG,SGN,CEIL,FLOOR,TANH,SILU" }, + { "GTX_STASH_L2_CHAIN_PRODUCER_OPS", "host-tiled contiguous descriptors" }, + { "GTX_STASH_L2_CHAIN_BOUNDARY_OPS", + "CONT,CPY,GET_ROWS,SET_ROWS,NORM,ADD_F16_F32_TO_F16,SIN,COS" }, + { nullptr, nullptr }, + }; + return features.data(); +} + +static void * ggml_backend_gtx_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) { + (void)reg; + if (name != nullptr && strcmp(name, "ggml_backend_get_features") == 0) { + return (void *)ggml_backend_gtx_get_features; + } + return nullptr; +} + +static const struct ggml_backend_reg_i ggml_backend_gtx_reg_i = { + /* .get_name = */ ggml_backend_gtx_reg_get_name, + /* .get_device_count = */ ggml_backend_gtx_reg_get_device_count, + /* .get_device = */ ggml_backend_gtx_reg_get_device, + /* .get_proc_address = */ ggml_backend_gtx_reg_get_proc_address, +}; + +ggml_backend_reg_t ggml_backend_gtx_reg(void) { + static struct ggml_backend_reg reg = { + /* .api_version = */ GGML_BACKEND_API_VERSION, + /* .iface = */ ggml_backend_gtx_reg_i, + /* .context = */ nullptr, + }; + return ® +} + +/*============================================================================= + * Public API Implementation + *============================================================================*/ + +extern "C" { + +int ggml_gtx_get_device_count(void) { + /* TODO: Enumerate PCIe devices */ + /* For now, return 1 simulated device */ + return 1; +} + +bool ggml_gtx_get_device_info(int device, ggml_gtx_device_info_t* info) { + if (info == nullptr || device < 0 || device >= ggml_gtx_get_device_count()) { + return false; + } + + memset(info, 0, sizeof(*info)); + info->device_id = device; + snprintf(info->name, sizeof(info->name), "GTX%d", device); + snprintf(info->description, sizeof(info->description), "GTX NPU Device %d", device); + if (gtx_use_xdma_stream_transport()) { + info->total_memory = GTX_FPGA_TENSOR_ARENA_SIZE; + info->free_memory = GTX_FPGA_TENSOR_ARENA_SIZE; + } else { + info->total_memory = 16ULL * 1024 * 1024 * 1024; + info->free_memory = 15ULL * 1024 * 1024 * 1024; + } + info->num_nests = 1; + info->num_spus = 1; + info->pcie_gen = 4; + info->pcie_width = 4; + info->firmware_version[0] = 0; + info->firmware_version[1] = 1; + info->firmware_version[2] = 0; + + return true; +} + +ggml_backend_t ggml_backend_gtx_init(int device) { + if (device < 0 || device >= ggml_gtx_get_device_count()) { + GTX_LOG_ERROR("Invalid device: %d\n", device); + return nullptr; + } + + /* Get device from registry and use device init */ + ggml_backend_dev_t dev = ggml_backend_reg_dev_get(ggml_backend_gtx_reg(), device); + if (!dev) { + GTX_LOG_ERROR("Failed to get device %d\n", device); + return nullptr; + } + + return ggml_backend_dev_init(dev, nullptr); +} + +bool ggml_backend_is_gtx(ggml_backend_t backend) { + return backend != nullptr && ggml_guid_matches(backend->guid, ggml_backend_gtx_guid()); +} + +int ggml_backend_gtx_get_device(ggml_backend_t backend) { + if (!ggml_backend_is_gtx(backend)) { + return -1; + } + ggml_backend_gtx_context* ctx = (ggml_backend_gtx_context*)backend->context; + return ctx->device_id; +} + +ggml_backend_buffer_type_t ggml_backend_gtx_buffer_type(int device) { + if (device < 0 || device >= ggml_gtx_get_device_count()) { + return nullptr; + } + + /* Lazy initialization of buffer types */ + static std::vector> buffer_types; + static std::vector> buffer_type_contexts; + static bool initializing = false; + static bool initialized = false; + + if (!initialized && !initializing) { + initializing = true; + + int count = ggml_gtx_get_device_count(); + + /* First pass: create contexts and buffer types without device reference */ + for (int i = 0; i < count; i++) { + /* Create a context for buffer type operations */ + auto ctx = std::make_unique(i); + + auto buft = std::make_unique(); + buft->iface = ggml_backend_gtx_buffer_type_interface; + buft->device = nullptr; /* Will be set later to avoid circular reference */ + buft->context = ctx.get(); + + buffer_type_contexts.push_back(std::move(ctx)); + buffer_types.push_back(std::move(buft)); + } + + /* Second pass: set device references after all buffer types are created */ + for (int i = 0; i < count; i++) { + buffer_types[i]->device = ggml_backend_reg_dev_get(ggml_backend_gtx_reg(), i); + } + + initialized = true; + initializing = false; + } + + return buffer_types[device].get(); +} + +ggml_backend_buffer_type_t ggml_backend_gtx_host_buffer_type(int device) { + (void)device; + return nullptr; /* Not supported yet */ +} + +void ggml_backend_gtx_get_stats(ggml_backend_t backend, ggml_gtx_stats_t* stats) { + if (!ggml_backend_is_gtx(backend) || stats == nullptr) { + return; + } + ggml_backend_gtx_context* ctx = (ggml_backend_gtx_context*)backend->context; + *stats = ctx->stats; +} + +void ggml_backend_gtx_reset_stats(ggml_backend_t backend) { + if (!ggml_backend_is_gtx(backend)) { + return; + } + ggml_backend_gtx_context* ctx = (ggml_backend_gtx_context*)backend->context; + memset(&ctx->stats, 0, sizeof(ctx->stats)); +} + +void ggml_gtx_set_debug(bool enable) { + g_gtx_debug = enable; +} + +bool ggml_backend_gtx_is_ready(ggml_backend_t backend) { + if (!ggml_backend_is_gtx(backend)) { + return false; + } + ggml_backend_gtx_context* ctx = (ggml_backend_gtx_context*)backend->context; + return ctx->use_uart ? ctx->uart->is_open() : ctx->conn->is_open(); +} + +const char* ggml_backend_gtx_get_error(ggml_backend_t backend) { + if (!ggml_backend_is_gtx(backend)) { + return nullptr; + } + ggml_backend_gtx_context* ctx = (ggml_backend_gtx_context*)backend->context; + return ctx->last_error.empty() ? nullptr : ctx->last_error.c_str(); +} + +} /* extern "C" */ + +GGML_BACKEND_DL_IMPL(ggml_backend_gtx_reg) diff --git a/depend/ggml-gtx/src/intrinsics/gtx_utils.c b/depend/ggml-gtx/src/intrinsics/gtx_utils.c new file mode 100644 index 0000000..1e7cd16 --- /dev/null +++ b/depend/ggml-gtx/src/intrinsics/gtx_utils.c @@ -0,0 +1,57 @@ +#include "ggml-gtx/intrinsics/intrin.h" + + +#define NUM_NESTS 4 +#define NUM_SPUS 16 + +#define SHARED_DBG 0xf00000 // 0x00f0_0000 ~ 0x00ff_ffff + +#define L1_SPRAM_BYTES (384u * 1024u) + + +void __capture_l1_nest(uint16_t nest, uintptr_t dst) +{ + const uint16_t maskall = UINT16_C(0xFFFF); + + __start_plan(nest); + + __start_shared(); + __credit_chk(maskall); + __store(SHARED_DBG, (uint64_t)dst, 0x6000UL, 0x6000UL, 16 * 16, 0x6000UL); + __end_shared(); + + for (uint32_t spu = 0; spu < NUM_SPUS; ++spu) { + const uint16_t mask = UINT16_C(1) << spu; + __start_thread(spu); + __store(0x00000, SHARED_DBG + 0x60000UL * spu, 0x6000UL, 0x6000UL, 16, 0x6000UL); + __credit_st(mask); + __end_thread(spu); + } + + __end_plan(nest); +} + +void __capture_l1(uintptr_t dst) +{ + const uint16_t maskall = UINT16_C(0xFFFF); + + for (uint32_t nest = 0; nest < NUM_NESTS; ++nest) { + + __start_plan(nest); + + __start_shared(); + __credit_chk(maskall); + __store(SHARED_DBG, (uint64_t)dst + nest * NUM_SPUS * L1_SPRAM_BYTES, 0x6000UL, 0x6000UL, 16 * 16, 0x6000UL); + __end_shared(); + + for (uint32_t spu = 0; spu < NUM_SPUS; ++spu) { + const uint16_t mask = UINT16_C(1) << spu; + __start_thread(spu); + __store(0x00000, SHARED_DBG + 0x60000UL * spu, 0x6000UL, 0x6000UL, 16, 0x6000UL); + __credit_st(mask); + __end_thread(spu); + } + + __end_plan(nest); + } +} diff --git a/depend/ggml-gtx/src/intrinsics/intrin_level1.c b/depend/ggml-gtx/src/intrinsics/intrin_level1.c new file mode 100644 index 0000000..ab7a5d7 --- /dev/null +++ b/depend/ggml-gtx/src/intrinsics/intrin_level1.c @@ -0,0 +1,1243 @@ +//================================================================== +// Copyright : (C) Supergate - All Rights Reserved +// Project : GSF / VTS +// Description : GTX intrinsic function description (level 1) +// base instruction intrinsic function +// Author : mh.kim ( NPU Div - NPU Core Team ) +// Last Update : 2026/01/23 +//================================================================== + +#ifndef INTRIN_LEVEL1_C +#define INTRIN_LEVEL1_C + +#include "ggml-gtx/intrinsics/intrin_level1.h" +#include "ggml-gtx/intrinsics/gtx_csr.h" + + +//================================= +// Matrix multiplication +//================================= +// [mm.s] rs1 +// rs1 = col_B_size[63:48], col_A_size[31:16], row_A_size[15:0] +__attribute__((noinline)) void __mm_s(uint16_t row_A_size, uint16_t col_A_size, uint16_t col_B_size) +{ + uint64_t rs1 = ((uint64_t) col_B_size << 48) | ((uint64_t) col_A_size << 16) | ((uint64_t) row_A_size); + + __asm__ volatile ("mm.s %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [mm.o] rs1 rs3 +// rs1 = col_A_size[31:16] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __mm_o(uint16_t col_A_size, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) col_A_size << 16); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile ("mm.o %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [mm.v] rs1 rs3 +// rs1 = col_A_size[31:16] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __mm_v(uint16_t col_A_size, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) col_A_size << 16); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile ("mm.v %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [mm] rs1 +// rs1 = col_B_size[63:48], col_A_size[31:16], row_A_size[15:0] +__attribute__((noinline)) void __mm(uint16_t row_A_size, uint16_t col_A_size, uint16_t col_B_size) +{ + uint64_t rs1 = ((uint64_t) col_B_size << 48) | ((uint64_t) col_A_size << 16) | ((uint64_t) row_A_size); + + __asm__ volatile ("mm %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [mm.t] rs1 +// rs1 = col_B_size[63:48], col_A_size[31:16], row_A_size[15:0] +__attribute__((noinline)) void __mm_t(uint16_t row_A_size, uint16_t col_A_size, uint16_t col_B_size) +{ + uint64_t rs1 = ((uint64_t) col_B_size << 48) | ((uint64_t) col_A_size << 16) | ((uint64_t) row_A_size); + + __asm__ volatile ("mm.t %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [mmc.s] rs1 +// rs1 = col_B_size[63:48], col_A_size[31:16], row_A_size[15:0] +__attribute__((noinline)) void __mmc_s(uint16_t row_A_size, uint16_t col_A_size, uint16_t col_B_size) +{ + uint64_t rs1 = ((uint64_t) col_B_size << 48) | ((uint64_t) col_A_size << 16) | ((uint64_t) row_A_size); + + __asm__ volatile ("mmc.s %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [mmc.o] rs1 rs3 +// rs1 = col_A_size[31:16] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __mmc_o(uint16_t col_A_size, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) col_A_size << 16); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile ("mmc.o %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [mmc.v] rs1 rs3 +// rs1 = col_A_size[31:16] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __mmc_v(uint16_t col_A_size, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) col_A_size << 16); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile ("mmc.v %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [mmc] rs1 +// rs1 = col_B_size[63:48], col_A_size[31:16], row_A_size[15:0] +__attribute__((noinline)) void __mmc(uint16_t row_A_size, uint16_t col_A_size, uint16_t col_B_size) +{ + uint64_t rs1 = ((uint64_t) col_B_size << 48) | ((uint64_t) col_A_size << 16) | ((uint64_t) row_A_size); + + __asm__ volatile ("mmc %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [mmc.t] rs1 +// rs1 = col_B_size[63:48], col_A_size[31:16], row_A_size[15:0] +__attribute__((noinline)) void __mmc_t(uint16_t row_A_size, uint16_t col_A_size, uint16_t col_B_size) +{ + uint64_t rs1 = ((uint64_t) col_B_size << 48) | ((uint64_t) col_A_size << 16) | ((uint64_t) row_A_size); + + __asm__ volatile ("mmc.t %[r1]\n" : : [r1] "r"(rs1)); +} + + + +//================================= +// Convolution +//================================= +// [im2col.n] rs1 rs2 +// rs1 = col_A_size[31:16], row_A_size[15:0] +// rs2 = num_of_channel[47:32], stride[31:16], dilation[9:8], kernel_size[4:0] +__attribute__((noinline)) void __im2col_n(uint16_t row_A_size , uint16_t col_A_size, + uint8_t kernel_size, uint8_t dilation , uint16_t stride, uint16_t num_of_channel) +{ + uint64_t rs1 = ((uint64_t) col_A_size << 16) | ((uint64_t) row_A_size); + uint64_t rs2 = ((uint64_t) num_of_channel << 32) | ((uint64_t) stride << 16) | ((uint64_t) dilation << 8) | ((uint64_t) kernel_size); + + __asm__ volatile ("im2col.n %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [im2col.d] rs1 rs2 +// rs1 = col_A_size[31:16], row_A_size[15:0] +// rs2 = num_of_channel[47:32], stride[31:16], dilation[9:8], kernel_size[4:0] +__attribute__((noinline)) void __im2col_d(uint16_t row_A_size , uint16_t col_A_size, + uint8_t kernel_size, uint8_t dilation , uint16_t stride, uint16_t num_of_channel) +{ + uint64_t rs1 = ((uint64_t) col_A_size << 16) | ((uint64_t) row_A_size); + uint64_t rs2 = ((uint64_t) num_of_channel << 32) | ((uint64_t) stride << 16) | ((uint64_t) dilation << 8) | ((uint64_t) kernel_size); + + __asm__ volatile ("im2col.d %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + + +//================================= +// Scalar calculation +//================================= +// [add.vs] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = scalar_value[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __add_vs(uint32_t vector_size, uint16_t scalar_value, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) scalar_value); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(1, rs4); + __asm__ volatile ("add.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [sub.vs] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = scalar_value[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __sub_vs(uint32_t vector_size, uint16_t scalar_value, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) scalar_value); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(1, rs4); + __asm__ volatile ("sub.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [mul.vs] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = scalar_value[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __mul_vs(uint32_t vector_size, uint16_t scalar_value, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) scalar_value); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(1, rs4); + __asm__ volatile ("mul.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [div.vs] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = scalar_value[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __div_vs(uint32_t vector_size, uint16_t scalar_value, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) scalar_value); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(1, rs4); + __asm__ volatile ("div.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [fmadd.vss] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = scalar_value1[31:16], scalar_value0[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __fmadd_vss(uint32_t vector_size, uint16_t scalar_value0, uint16_t scalar_value1, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) scalar_value1 << 16) | ((uint64_t) scalar_value0); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(1, rs4); + __asm__ volatile("fmadd.vss %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [max.vs] rs1 rs2 rs3 rs4 +// rs1 = vector_size[23:0] +// rs2 = previous_max_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __max_vs(uint32_t vector_size, uint16_t previous_max_value, uint8_t result_svr_addr, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) previous_max_value); + uint64_t rs3 = ((uint64_t) result_svr_addr); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(0, rs3); + __opset(1, rs4); + __asm__ volatile("max.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [min.vs] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = previous_min_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __min_vs(uint32_t vector_size, uint16_t previous_min_value, uint8_t result_svr_addr, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) previous_min_value); + uint64_t rs3 = ((uint64_t) result_svr_addr); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(0, rs3); + __opset(1, rs4); + __asm__ volatile("min.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [add.is] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr[4:0] +// rs2 = scalar_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __add_is(uint8_t src_svr_addr, uint16_t scalar_value, uint8_t result_svr_addr, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr); + uint64_t rs2 = ((uint64_t) scalar_value); + uint64_t rs3 = ((uint64_t) result_svr_addr); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(0, rs3); + __opset(1, rs4); + __asm__ volatile("add.is %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [sub.is] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr[4:0] +// rs2 = scalar_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __sub_is(uint8_t src_svr_addr, uint16_t scalar_value, uint8_t result_svr_addr, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr); + uint64_t rs2 = ((uint64_t) scalar_value); + uint64_t rs3 = ((uint64_t) result_svr_addr); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(0, rs3); + __opset(1, rs4); + __asm__ volatile("sub.is %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [mul.is] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr[4:0] +// rs2 = scalar_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __mul_is(uint8_t src_svr_addr, uint16_t scalar_value, uint8_t result_svr_addr, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr); + uint64_t rs2 = ((uint64_t) scalar_value); + uint64_t rs3 = ((uint64_t) result_svr_addr); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(0, rs3); + __opset(1, rs4); + __asm__ volatile("mul.is %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [mul.is.f32] rs1 rs2 rs3 +// rs1 = src_svr_addr[4:0] +// rs2 = FP32 scalar raw bits[31:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __mul_is_f32(uint8_t src_svr_addr, float scalar_value, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr); + union { + float f; + uint32_t u; + } scalar_bits = { scalar_value }; + uint64_t rs2 = ((uint64_t) scalar_bits.u); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile(".insn r 0x0B, 6, 0x12, x0, %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [div.is] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr[4:0] +// rs2 = scalar_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __div_is(uint8_t src_svr_addr, uint16_t scalar_value, uint8_t result_svr_addr, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr); + uint64_t rs2 = ((uint64_t) scalar_value); + uint64_t rs3 = ((uint64_t) result_svr_addr); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(0, rs3); + __opset(1, rs4); + __asm__ volatile("div.is %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [fmadd.iss] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr[4:0] +// rs2 = scalar_value1[31:16], scalar_value0[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __fmadd_iss(uint8_t src_svr_addr, uint16_t scalar_value0, uint16_t scalar_value1, uint8_t result_svr_addr, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr); + uint64_t rs2 = ((uint64_t) scalar_value1 << 16) | ((uint64_t) scalar_value0); + uint64_t rs3 = ((uint64_t) result_svr_addr); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(0, rs3); + __opset(1, rs4); + __asm__ volatile("fmadd.iss %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [max.is] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr[4:0] +// rs2 = previous_max_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __max_is(uint8_t src_svr_addr, uint16_t previous_max_value, uint8_t result_svr_addr, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr); + uint64_t rs2 = ((uint64_t) previous_max_value); + uint64_t rs3 = ((uint64_t) result_svr_addr); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(0, rs3); + __opset(1, rs4); + __asm__ volatile("max.is %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [min.is] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr[4:0] +// rs2 = previous_min_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __min_is(uint8_t src_svr_addr, uint16_t previous_min_value, uint8_t result_svr_addr, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr); + uint64_t rs2 = ((uint64_t) previous_min_value); + uint64_t rs3 = ((uint64_t) result_svr_addr); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(0, rs3); + __opset(1, rs4); + __asm__ volatile("min.is %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + + +//================================= +// Vector calculation +//================================= +// [clamp.min] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = scalar_value[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __clamp_min(uint32_t vector_size, uint16_t scalar_value, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) scalar_value); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(1, rs4); + __asm__ volatile("clamp.min %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [clamp.max] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = scalar_value[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __clamp_max(uint32_t vector_size, uint16_t scalar_value, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) scalar_value); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(1, rs4); + __asm__ volatile("clamp.max %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [arange] rs1 rs2 +// rs1 = vector_size[23:0] +// rs2 = step[31:16], start_value[15:0] +__attribute__((noinline)) void __arange(uint32_t vector_size, uint16_t start_value, uint16_t step) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) step << 16) | ((uint64_t) start_value); + + __asm__ volatile("arange %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [add.ii] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = src_svr_addr_B[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __add_ii(uint8_t src_svr_addr_A, uint8_t src_svr_addr_B, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs2 = ((uint64_t) src_svr_addr_B); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("add.ii %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [sub.ii] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = src_svr_addr_B[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __sub_ii(uint8_t src_svr_addr_A, uint8_t src_svr_addr_B, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs2 = ((uint64_t) src_svr_addr_B); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("sub.ii %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [mul.ii] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = src_svr_addr_B[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __mul_ii(uint8_t src_svr_addr_A, uint8_t src_svr_addr_B, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs2 = ((uint64_t) src_svr_addr_B); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("mul.ii %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [div.ii] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = src_svr_addr_B[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __div_ii(uint8_t src_svr_addr_A, uint8_t src_svr_addr_B, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs2 = ((uint64_t) src_svr_addr_B); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("div.ii %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [fmadd.iii] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = src_svr_addr_C[9:5], src_svr_addr_B[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __fmadd_iii(uint8_t src_svr_addr_A, uint8_t src_svr_addr_B, uint8_t src_svr_addr_C, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs2 = ((uint64_t) src_svr_addr_C << 5) | ((uint64_t) src_svr_addr_B); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("fmadd.iii %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [sqrt.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __sqrt_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("sqrt.i %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [exp.i] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = mode[1:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __exp_i(uint8_t src_svr_addr_A, uint8_t mode, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs2 = ((uint64_t) mode); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + // __asm__ volatile("exp.i %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + + +// [ln.i] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = mode[1:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __ln_i(uint8_t src_svr_addr_A, uint8_t mode, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs2 = ((uint64_t) mode); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + // __asm__ volatile("ln.i %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [abs.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __abs_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("abs.i %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [neg.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __neg_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("neg.i %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [sign.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __sign_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("sign.i %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [step.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __step_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("step.i %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [ceil.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __ceil_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("ceil.i %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [trunc.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __trunc_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("trunc.i %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [floor.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __floor_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("floor.i %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [rne.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __rne_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("rne.i %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [and.ii] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = src_svr_addr_B[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __and_ii(uint8_t src_svr_addr_A, uint8_t src_svr_addr_B, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs2 = ((uint64_t) src_svr_addr_B); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("and.ii %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [or.ii] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = src_svr_addr_B[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __or_ii(uint8_t src_svr_addr_A, uint8_t src_svr_addr_B, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs2 = ((uint64_t) src_svr_addr_B); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("or.ii %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [not.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __not_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("not.i %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [shift.i] rs1 rs2 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs2 = shift_mode[4], shift_num[3:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __shift_i(uint8_t src_svr_addr_A, uint8_t shift_num, uint8_t shift_mode, uint8_t result_svr_addr) + +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs2 = ((uint64_t) shift_mode << 4) | ((uint64_t) shift_num); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("shift.i %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + + +//================================= +// Format conversion +//================================= +// [scvt.qh] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = offset[31:16], scale[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __scvt_qh(uint32_t vector_size, uint16_t scale, uint16_t offset, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) offset << 16) | ((uint64_t) scale); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(1, rs4); + __asm__ volatile ("scvt.qh %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [scvt.hq] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = offset[31:16], scale[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __scvt_hq(uint32_t vector_size, uint16_t scale, uint16_t offset, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) offset << 16) | ((uint64_t) scale); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(1, rs4); + __asm__ volatile ("scvt.hq %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [scvt.ih] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = offset[31:16], scale[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __scvt_ih(uint32_t vector_size, uint16_t scale, uint16_t offset, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) offset << 16) | ((uint64_t) scale); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(1, rs4); + __asm__ volatile ("scvt.ih %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [scvt.hi] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = offset[31:16], scale[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __scvt_hi(uint32_t vector_size, uint16_t scale, uint16_t offset, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) offset << 16) | ((uint64_t) scale); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(1, rs4); + __asm__ volatile ("scvt.hi %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [scvt.hn] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = offset[31:16], scale[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __scvt_hn(uint32_t vector_size, uint16_t scale, uint16_t offset, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) offset << 16) | ((uint64_t) scale); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(1, rs4); + __asm__ volatile ("scvt.hn %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + + +//================================= +// Activation +//================================= +// [prelu] rs1 rs2 rs4 +// rs1 = vector_size[23:0] +// rs2 = slop_value[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __prelu(uint32_t vector_size, uint16_t slop_value, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) slop_value); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(1, rs4); + __asm__ volatile ("prelu %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [prelu.i] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr_A[4:0] +// rs2 = slop_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __prelu_i(uint8_t src_svr_addr_A, uint16_t slop_value, uint8_t result_svr_addr, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs2 = ((uint64_t) slop_value); + uint64_t rs3 = ((uint64_t) result_svr_addr); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(0, rs3); + __opset(1, rs4); + __asm__ volatile("prelu.i %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [gelu.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __gelu_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("gelu.i %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [tanh.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __tanh_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("tanh.i %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [sigm.i] rs1 rs3 +// rs1 = src_svr_addr_A[4:0] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __sigm_i(uint8_t src_svr_addr_A, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile("sigm.i %[r1]\n" : : [r1] "r"(rs1)); +} + + + +//================================= +// Softmax +//================================= +// [esum] rs1 rs2 rs3 rs4 +// rs1 = vertor_size[23:0] +// rs2 = accumulated_data[31:16], max_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __esum(uint32_t vector_size, uint16_t max_value, uint16_t accumulated_data, uint8_t result_svr_addr, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) accumulated_data << 16) | ((uint64_t) max_value); + uint64_t rs3 = ((uint64_t) result_svr_addr); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(0, rs3); + __opset(1, rs4); + __asm__ volatile ("esum %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [softmax] rs1 rs2 rs4 +// rs1 = vertor_size[23:0] +// rs2 = esum_value[31:16], max_value[15:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __softmax(uint32_t vector_size, uint16_t max_value, uint16_t esum_value, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) esum_value << 16) | ((uint64_t) max_value); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(1, rs4); + __asm__ volatile ("softmax %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [esum.i] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr_A[4:0] +// rs2 = accumulated_data[31:16], max_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __esum_i(uint8_t src_svr_addr_A, uint16_t max_value, uint16_t accumulated_data, uint8_t result_svr_addr, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs2 = ((uint64_t) accumulated_data << 16) | ((uint64_t) max_value); + uint64_t rs3 = ((uint64_t) result_svr_addr); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(0, rs3); + __opset(1, rs4); + __asm__ volatile ("esum.i %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [softmax.i] rs1 rs2 rs3 rs4 +// rs1 = src_svr_addr_A[4:0] +// rs2 = esum_value[31:16], max_value[15:0] +// rs3 = result_svr_addr[4:0] +// rs4 = r2_sel[8:0] +__attribute__((noinline)) void __softmax_i(uint8_t src_svr_addr_A, uint16_t max_value, uint16_t esum_value, uint8_t result_svr_addr, uint16_t r2_sel) +{ + uint64_t rs1 = ((uint64_t) src_svr_addr_A); + uint64_t rs2 = ((uint64_t) esum_value << 16) | ((uint64_t) max_value); + uint64_t rs3 = ((uint64_t) result_svr_addr); + uint64_t rs4 = ((uint64_t) r2_sel); + + __opset(0, rs3); + __opset(1, rs4); + __asm__ volatile("softmax.i %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + + +//================================= +// Pooling +//================================= +// [pool.m] rs1 rs2 +// rs1 = col_OUT[63:48], row_OUT[47:32], col_IN[31:16], row_IN[15:0] +// rs2 = col_stride[31:24], row_stride[23:16], col_K[15:8], row_K[7:0] +__attribute__((noinline)) void __pool_m(uint16_t row_IN, uint16_t col_IN, uint16_t row_OUT , uint16_t col_OUT , + uint8_t row_K , uint8_t col_K , uint8_t row_stride, uint8_t col_stride) +{ + uint64_t rs1 = ((uint64_t) col_OUT << 48) | ((uint64_t) row_OUT << 32) | ((uint64_t) col_IN << 16) | ((uint64_t) row_IN); + uint64_t rs2 = ((uint64_t) col_stride << 24) | ((uint64_t) row_stride << 16) | ((uint64_t) col_K << 8) | ((uint64_t) row_K); + + __asm__ volatile ("pool.m %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [pool.a] rs1, rs2 +// rs1 = col_OUT[63:48], row_OUT[47:32], col_IN[31:16], row_IN[15:0] +// rs2 = k_value[47:32]. col_stride[31:24], row_stride[23:16], col_K[15:8], row_K[7:0] +__attribute__((noinline)) void __pool_a(uint16_t row_IN, uint16_t col_IN, uint16_t row_OUT , uint16_t col_OUT , + uint8_t row_K , uint8_t col_K , uint8_t row_stride, uint8_t col_stride, uint16_t k_value) +{ + uint64_t rs1 = ((uint64_t) col_OUT << 48) | ((uint64_t) row_OUT << 32) | ((uint64_t) col_IN << 16) | ((uint64_t) row_IN); + uint64_t rs2 = ((uint64_t) k_value << 32) | ((uint64_t) col_stride << 24) | ((uint64_t) row_stride << 16) | ((uint64_t) col_K << 8) | ((uint64_t) row_K); + + __asm__ volatile ("pool.a %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + + +//================================= +// Memory operation +//================================= +// [tpose] rs1 rs2 rs3 +// rs1 = dtype[56], rw_dir[49:48], src_addr[36:0] +// rs2 = dim2_size[63:48], dim1_size[47:32], dim0_size[31:16], dim2[9:8], dim1[5:4], dim0[1:0] +// rs3 = dst_addr[36:0] +__attribute__((noinline)) void __tpose(uint64_t src_addr , uint64_t dst_addr , + uint16_t dim2_size, uint16_t dim1_size, uint16_t dim0_size, uint8_t dim2, uint8_t dim1, uint8_t dim0, + uint8_t rw_dir , uint8_t dtype ) +{ + src_addr = src_addr & 0x0000001FFFFFFFFF; + + uint64_t rs1 = ((uint64_t) dtype << 56) | ((uint64_t) rw_dir << 48) | ((uint64_t) src_addr); + uint64_t rs2 = ((uint64_t) dim2_size << 48) | ((uint64_t) dim1_size << 32) | ((uint64_t) dim0_size << 16) | ((uint64_t) dim2 << 8) | ((uint64_t) dim1 << 4) | ((uint64_t) dim0); + uint64_t rs3 = dst_addr; + + __opset(0, rs3); + __asm__ volatile("tpose %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [fill] rs1 rs2 rs3 +// rs1 = dir[48], dst_addr[36:0] +// rs2 = height[63:48], length[47:32], write_stride[31:0] +// rs3 = fill_pattern[63:0] +__attribute__((noinline)) void __fill(uint64_t dst_addr, uint32_t write_stride, uint16_t length, uint16_t height, uint64_t fill_pattern, uint8_t dir) +{ + dst_addr = dst_addr & 0x0000001FFFFFFFFF; + + uint64_t rs1 = ((uint64_t) dir << 48) | ((uint64_t) dst_addr); + uint64_t rs2 = ((uint64_t) height << 48) | ((uint64_t) length << 32) | ((uint64_t) write_stride); + uint64_t rs3 = fill_pattern; + + __opset(0, rs3); + __asm__ volatile("fill %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + + +//================================= +// DMA +//================================= +// [load] rs1 rs2 rs3 +// rs1 = src_addr[63:27], dst_addr[26:0] +// rs2 = heigth[63:48], length[47:32], read_stride[31:0] +// rs3 = write_stride[31:0] +__attribute__((noinline)) void __load(uint64_t src_addr, uint32_t dst_addr, uint32_t read_stride, uint16_t length, uint16_t height, uint32_t write_stride) +{ + src_addr = src_addr & 0x0000001FFFFFFFFF; + dst_addr = dst_addr & 0x07FFFFFF; + + uint64_t rs1 = ((uint64_t) src_addr << 27) | ((uint64_t) dst_addr); + uint64_t rs2 = ((uint64_t) height << 48) | ((uint64_t) length << 32) | ((uint64_t) read_stride); + uint64_t rs3 = ((uint64_t) write_stride); + + __opset(0, rs3); + __asm__ volatile ("load %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [store] rs1 rs2 rs3 +// rs1 = dst_addr[63:27], src_addr[26:0] +// rs2 = heigth[63:48], length[47:32], write_stride[31:0] +// rs3 = read_stride[31:0] +__attribute__((noinline)) void __store(uint32_t src_addr, uint64_t dst_addr, uint32_t write_stride, uint16_t length, uint16_t height, uint32_t read_stride) +{ + src_addr = src_addr & 0x07FFFFFF; + dst_addr = dst_addr & 0x0000001FFFFFFFFF; + + uint64_t rs1 = ((uint64_t) dst_addr << 27) | ((uint64_t) src_addr); + uint64_t rs2 = ((uint64_t) height << 48) | ((uint64_t) length << 32) | ((uint64_t) write_stride); + uint64_t rs3 = ((uint64_t) read_stride); + + __opset(0, rs3); + __asm__ volatile ("store %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [copy] rs1 rs2 rs3 +// rs1 = dst_addr[58:32], src_addr[26:0] +// rs2 = heigth[63:48], length[47:32], read_stride[31:0] +// rs3 = write_stride[31:0] +__attribute__((noinline)) void __copy(uint32_t src_addr, uint32_t dst_addr, uint32_t read_stride, uint16_t length, uint16_t height, uint32_t write_stride) +{ + uint64_t rs1 = ((uint64_t) dst_addr << 32) | ((uint64_t) src_addr); + uint64_t rs2 = ((uint64_t) height << 48) | ((uint64_t) length << 32) | ((uint64_t) read_stride); + uint64_t rs3 = ((uint64_t) write_stride); + + __opset(0, rs3); + __asm__ volatile ("copy %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + + +// [mcast.s2l] rs1 rs2 rs3 +// rs1 = src_addr[58:32], dst_addr[23:0] +// rs2 = heigth[63:48], length[47:32], read_stride[31:0] +// rs3 = target_spu[63:0] +__attribute__((noinline)) void __mcast_s2l(uint32_t src_addr, uint32_t dst_addr, uint32_t read_stride, uint16_t length, uint16_t height, uint64_t target_spu) +{ + uint64_t rs1 = ((uint64_t) src_addr << 32) | ((uint64_t) dst_addr); + uint64_t rs2 = ((uint64_t) height << 48) | ((uint64_t) length << 32) | ((uint64_t) read_stride); + uint64_t rs3 = target_spu; + + __opset(0, rs3); + __asm__ volatile("mcast.s2l %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [mcast.g2s] rs1 rs2 rs3 +// rs1 = src_addr[63:27], dst_addr[26:0] +// rs2 = heigth[63:48], length[47:32], read_stride[31:0] +// rs3 = target_nest[63:0] +__attribute__((noinline)) void __mcast_g2s(uint64_t src_addr, uint32_t dst_addr, uint32_t read_stride, uint16_t length, uint16_t height, uint64_t target_nest) +{ + src_addr = src_addr & 0x0000001FFFFFFFFF; + + uint64_t rs1 = ((uint64_t) src_addr << 27) | ((uint64_t) dst_addr); + uint64_t rs2 = ((uint64_t) height << 48) | ((uint64_t) length << 32) | ((uint64_t) read_stride); + uint64_t rs3 = target_nest; + + __opset(0, rs3); + __asm__ volatile("mcast.g2s %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [mcast.s2s] rs1 rs2 rs3 +// rs1 = target_nest_sel[63], src_nest_id[61:56], dst_addr[55:27], src_addr[26:0] +// rs2 = heigth[63:48], length[47:32], read_stride[31:0] +// rs3 = target_nest[63:32], write_stride[31:0] +__attribute__((noinline)) void __mcast_s2s(uint32_t src_addr, uint32_t dst_addr, uint32_t read_stride, uint16_t length, uint16_t height, uint32_t write_stride, + uint8_t src_nest_id, uint8_t target_nest_sel, uint32_t target_nest) +{ + src_addr = src_addr & 0x07FFFFFF; + dst_addr = dst_addr & 0x07FFFFFF; + + uint64_t rs1 = ((uint64_t) target_nest_sel << 63) | ((uint64_t) src_nest_id << 56) | ((uint64_t) dst_addr << 27) | ((uint64_t) src_addr); + uint64_t rs2 = ((uint64_t) height << 48) | ((uint64_t) length << 32) | ((uint64_t) read_stride); + uint64_t rs3 = ((uint64_t) target_nest << 32) | ((uint64_t) write_stride); + + __opset(0, rs3); + __asm__ volatile("mcast.s2s %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [copy.mem] rs1 rs2 rs3 +// rs1 = write_stride_L[63:48], src_addr_DDR[36:0] +// rs3 = heigth[63:48], length[47:32], read_stride[31:0] +// rs2 = write_stride_H[63:48], dst_addr_DDR[36:0] +__attribute__((noinline)) void __copy_mem(uint64_t src_addr_DDR, uint64_t dst_addr_DDR, uint32_t read_stride, uint16_t length, uint16_t height, uint16_t write_stride_L, uint16_t write_stride_H) +{ + src_addr_DDR = src_addr_DDR & 0x0000001FFFFFFFFF; + dst_addr_DDR = dst_addr_DDR & 0x0000001FFFFFFFFF; + + uint64_t rs1 = ((uint64_t) write_stride_L << 48) | ((uint64_t) src_addr_DDR); + uint64_t rs2 = ((uint64_t) height << 48) | ((uint64_t) length << 32) | ((uint64_t) read_stride); + uint64_t rs3 = ((uint64_t) write_stride_H << 48) | ((uint64_t) dst_addr_DDR); + + __opset(0, rs3); + __asm__ volatile("copy.mem %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + + +//================================= +// SPR +//================================= +// [rdspr] rd rs1 +// rd = spr_data[63:0] +// rs1 = nest_id[29:24], spu_id[21:16], spr_addr[11:0] +__attribute__((noinline)) uint64_t __rdspr(uint16_t spr_addr, uint8_t nest_id, uint8_t spu_id) +{ + uint64_t res; + uint64_t rs1 = ((uint64_t) nest_id << 24) | ((uint64_t) spu_id << 16) | ((uint64_t) spr_addr); + + __asm__ volatile("rdspr %[result], %[r1]\n" : [result] "=r"(res) : [r1] "r"(rs1)); + + return res; +} + + +// [wrspr] rs1 rs2 rs3 rs4 +// rs1 = wrstb_n[23:16], spr_addr[11:0] +// rs2 = spr_data[63:0] +// rs3 = target_mask[63:0] +__attribute__((noinline)) void __wrspr(uint16_t spr_addr, uint8_t wrstb_n, uint64_t spr_data, uint64_t target_mask) +{ + uint64_t rs1 = ((uint64_t) wrstb_n << 16) | ((uint64_t) spr_addr); + uint64_t rs2 = spr_data; + uint64_t rs3 = target_mask; + + __opset(0, rs3); + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [mvsvr] rs1 rs2 rs3 +// rs1 = src_svr_addr[4:0] +// rs2 = dst_svr_addr[4:0] +// rs3 = wrstrb_n[31:0] +__attribute__((noinline)) void __mvsvr(uint8_t src_svr_addr, uint8_t dst_svr_addr, uint32_t wrstrb_n) +{ + uint64_t rs3 = ((uint64_t) wrstrb_n); + + __opset(0, rs3); + __asm__ volatile("mvsvr %[r1], %[r2]\n" : : [r1] "r"(src_svr_addr), [r2] "r"(dst_svr_addr)); +} + + + +//================================= +// Credit +//================================= + + + +//================================= +// Micro Code +//================================= + + + +//================================= +// Sync +//================================= + + +//================================= +// Warp operation +//================================= + + + +#endif \ No newline at end of file diff --git a/depend/ggml-gtx/src/intrinsics/intrin_level2.c b/depend/ggml-gtx/src/intrinsics/intrin_level2.c new file mode 100644 index 0000000..bf33ed1 --- /dev/null +++ b/depend/ggml-gtx/src/intrinsics/intrin_level2.c @@ -0,0 +1,309 @@ +//================================================================== +// Copyright : (C) Supergate - All Rights Reserved +// Project : GSF / VTS +// Description : GTX intrinsic function description (level 2) +// convenience intrinsic function for programming +// Author : mh.kim ( NPU Div - NPU Core Team ) +// Last Update : 2026/01/23 +//================================================================== + +#ifndef INTRIN_LEVEL2_C +#define INTRIN_LEVEL2_C + +#include "ggml-gtx/intrinsics/intrin_level1.h" +#include "ggml-gtx/intrinsics/intrin_level2.h" +#include "ggml-gtx/intrinsics/gtx_csr.h" + + +//================================= +// Auto setting +//================================= +// [init L1SPM bank addr] +// 4bit input: en_restore_R[3], en_restore_C[2], en_restore_B[1], en_restore_A[0] +// initial spm_addr_A - 0x00000 +// initial spm_addr_B - 0x20000 +// initial spm_addr_C - 0x30000 +// initial spm_addr_R - 0x50000 +__attribute__((noinline)) void __init_spm_addr(uint8_t target_spm_bank) +{ + uint64_t rs1, rs2; + + if (target_spm_bank & 0x1) { + rs1 = ((uint64_t) SPM_ADDR_A); + rs2 = ((uint64_t) 0x00000); + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + } + + if (target_spm_bank & 0x2) { + rs1 = ((uint64_t) SPM_ADDR_B); + rs2 = ((uint64_t) 0x20000); + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + } + + if (target_spm_bank & 0x4) { + rs1 = ((uint64_t) SPM_ADDR_C); + rs2 = ((uint64_t) 0x30000); + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + } + + if (target_spm_bank & 0x8) { + rs1 = ((uint64_t) SPM_ADDR_R); + rs2 = ((uint64_t) 0x50000); + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + } +} + + +// [set L1SPM bank addr] +// spm_addr_A - 24bit +// spm_addr_B - 24bit +// spm_addr_C - 24bit +// spm_addr_R - 24bit +__attribute__((noinline)) void __set_spm_addr(uint32_t spm_addr_R, uint32_t spm_addr_C, uint32_t spm_addr_B, uint32_t spm_addr_A) +{ + uint64_t rs1, rs2; + + rs1 = ((uint64_t) SPM_ADDR_A); + rs2 = ((uint64_t) spm_addr_A); + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + + rs1 = ((uint64_t) SPM_ADDR_B); + rs2 = ((uint64_t) spm_addr_B); + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + + rs1 = ((uint64_t) SPM_ADDR_C); + rs2 = ((uint64_t) spm_addr_C); + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + + rs1 = ((uint64_t) SPM_ADDR_R); + rs2 = ((uint64_t) spm_addr_R); + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [set L1SPM bank addr A] +// spm_addr_A - 24bit +__attribute__((noinline)) void __set_spm_addr_A(uint32_t spm_addr_A) +{ + uint64_t rs1 = ((uint64_t) SPM_ADDR_A); + uint64_t rs2 = ((uint64_t) spm_addr_A); + + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [set L1SPM bank addr B] +// spm_addr_B - 24bit +__attribute__((noinline)) void __set_spm_addr_B(uint32_t spm_addr_B) +{ + uint64_t rs1 = ((uint64_t) SPM_ADDR_B); + uint64_t rs2 = ((uint64_t) spm_addr_B); + + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [set L1SPM bank addr C] +// spm_addr_C - 24bit +__attribute__((noinline)) void __set_spm_addr_C(uint32_t spm_addr_C) +{ + uint64_t rs1 = ((uint64_t) SPM_ADDR_C); + uint64_t rs2 = ((uint64_t) spm_addr_C); + + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [set L1SPM bank addr R] +// spm_addr_R - 24bit +__attribute__((noinline)) void __set_spm_addr_R(uint32_t spm_addr_R) +{ + uint64_t rs1 = ((uint64_t) SPM_ADDR_R); + uint64_t rs2 = ((uint64_t) spm_addr_R); + + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + + +//================================= +// DMA with credit +//================================= +// [load with credit] +// load rs1 = src_addr[63:27], dst_addr[26:0] +// load rs2 = heigth[63:48], length[47:32], read_stride[31:0] +// load rs3 = write_stride[31:0] +// additional = auto_credit(1bit), target_spu(64bit), target_nest(64bit) +// auto_credit 1: 'load' with 'credit_ld' +// 0: 'load' +__attribute__((noinline)) void __load_cr(uint64_t src_addr , uint32_t dst_addr , uint32_t read_stride, uint16_t length, uint16_t height, uint32_t write_stride, + uint8_t auto_credit, uint64_t target_spu, uint64_t target_nest) +{ + src_addr = src_addr & 0x0000001FFFFFFFFF; + dst_addr = dst_addr & 0x07FFFFFF; + + uint64_t rs1 = ((uint64_t) src_addr << 27) | ((uint64_t) dst_addr); + uint64_t rs2 = ((uint64_t) height << 48) | ((uint64_t) length << 32) | ((uint64_t) read_stride); + uint64_t rs3 = ((uint64_t) write_stride); + + __opset(0, rs3); + __asm__ volatile ("load %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + + if (auto_credit & 0x1) { + __credit_ld(target_spu, target_nest); + } +} + + +// [store with credit] +// store rs1 = dst_addr[63:27], src_addr[26:0] +// store rs2 = heigth[63:48], length[47:32], write_stride[31:0] +// store rs3 = read_stride[31:0] +// additional = auto_credit(1bit), target_spu(64bit) +// auto_credit 1: 'store' with 'credit_st' +// 0: 'store' +__attribute__((noinline)) void __store_cr(uint32_t src_addr , uint64_t dst_addr , uint32_t write_stride, uint16_t length, uint16_t height, uint32_t read_stride, + uint8_t auto_credit, uint64_t target_spu) +{ + src_addr = src_addr & 0x07FFFFFF; + dst_addr = dst_addr & 0x0000001FFFFFFFFF; + + uint64_t rs1 = ((uint64_t) dst_addr << 27) | ((uint64_t) src_addr); + uint64_t rs2 = ((uint64_t) height << 48) | ((uint64_t) length << 32) | ((uint64_t) write_stride); + uint64_t rs3 = ((uint64_t) read_stride); + + __opset(0, rs3); + __asm__ volatile ("store %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + + if (auto_credit & 0x1) { + __credit_st(target_spu); + } +} + + + +//================================= +// Renaming intrinsic for programming (same operation) +//================================= +// [sum] +// same as __mm_o +// rs1 = col_A_size[31:16] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __sum(uint16_t col_A_size, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) col_A_size << 16); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile ("mm.o %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [sum_acc] +// same as __mmc_o +// rs1 = col_A_size[31:16] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __sum_acc(uint16_t col_A_size, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) col_A_size << 16); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile ("mmc.o %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [dot_product] +// same as __mm_v +// rs1 = col_A_size[31:16] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __dot_product(uint16_t col_A_size, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) col_A_size << 16); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile ("mm.v %[r1]\n" : : [r1] "r"(rs1)); +} + + +// [dot_product_acc] +// same as __mmc_v +// rs1 = col_A_size[31:16] +// rs3 = result_svr_addr[4:0] +__attribute__((noinline)) void __dot_product_acc(uint16_t col_A_size, uint8_t result_svr_addr) +{ + uint64_t rs1 = ((uint64_t) col_A_size << 16); + uint64_t rs3 = ((uint64_t) result_svr_addr); + + __opset(0, rs3); + __asm__ volatile ("mmc.v %[r1]\n" : : [r1] "r"(rs1)); +} + + + +//================================= +// Usable function (not consisting of GTX-ISA) +//================================= +// [int16_to_fp16] +// integer 16 format convert to floating point 16 format (RNE role) +__attribute__((noinline)) uint16_t __int16_to_fp16(int16_t integer) +{ + uint16_t exp; + uint16_t mantissa; + uint16_t sign = 0; + uint16_t abs_val; + uint16_t integer_msb = 15; + uint8_t guard_bit; + uint8_t round_bit; + uint8_t sticky_bit; + + if (integer == 0) { + return 0; + } + + if (integer < 0) { + sign = 1; + abs_val = (uint16_t)(-integer); + } else { + abs_val = (uint16_t)integer; + } + + while ((abs_val & (1 << integer_msb)) == 0) { + integer_msb--; + } + + exp = integer_msb + 15; + + if (integer_msb <= 10) { + mantissa = (abs_val << (10 - integer_msb)) & 0x3FF; + } else { + uint8_t shift = integer_msb - 10; + + mantissa = (abs_val >> shift) & 0x3FF; + + guard_bit = (abs_val >> (shift - 1)) & 1; + round_bit = (abs_val >> (shift - 2)) & 1; + sticky_bit = (shift > 2) ? ((abs_val & ((1U << (shift - 2)) - 1)) != 0) : 0; + + if ((guard_bit && (round_bit || sticky_bit)) || + (guard_bit && !round_bit && !sticky_bit && (mantissa & 1))) { + mantissa++; + } + + if (mantissa == 0x400) { + mantissa = 0; + exp++; + } + } + + if (exp >= 31) { + return (sign << 15) | 0x7C00; + } + + return (sign << 15) | ((exp & 0x1F) << 10) | (mantissa & 0x3FF); +} + + + +#endif \ No newline at end of file diff --git a/depend/ggml-gtx/src/intrinsics/intrin_level3.c b/depend/ggml-gtx/src/intrinsics/intrin_level3.c new file mode 100644 index 0000000..0c7b889 --- /dev/null +++ b/depend/ggml-gtx/src/intrinsics/intrin_level3.c @@ -0,0 +1,613 @@ +//================================================================== +// Copyright : (C) Supergate - All Rights Reserved +// Project : GSF / VTS +// Description : GTX intrinsic function description (level 3) +// complex operation intrinsic function consisting of compound instructions +// Author : mh.kim ( NPU Div - NPU Core Team ) +// Last Update : 2025/12/15 +//================================================================== + +#ifndef INTRIN_LEVEL3_C +#define INTRIN_LEVEL3_C + +#include "ggml-gtx/intrinsics/intrin_level1.h" +#include "ggml-gtx/intrinsics/intrin_level2.h" +#include "ggml-gtx/intrinsics/intrin_level3.h" +#include "ggml-gtx/intrinsics/gtx_csr.h" + + +//================================= +// Special function - memory +//================================= +// [pad] +// padding: fill 0 to dst_addr & DMA to dst_addr per channel +__attribute__((noinline)) void __pad(uint64_t src_addr , uint64_t dst_addr , + uint32_t fill_stride , uint16_t length , uint16_t height, + uint8_t ystr , uint8_t yend , uint8_t xstr , uint8_t xend , uint64_t fill_pattern, + uint32_t channel_stride, uint16_t channel_num, uint8_t rw , uint8_t d_type) +{ + volatile int i; + + uint64_t rs1, rs2, rs3; + + xstr = xstr << d_type; + xend = xend << d_type; + + uint16_t out_l = (length + xstr + xend); + uint16_t out_h = (height + ystr + yend); + uint16_t fill_h = out_h * channel_num; + uint32_t img_size = out_l * out_h; + + // [01:00] : 11 = x, 10 = read DDR write L2SPM , 01 = read L2SPM write DDR , 00 = read L2SPM write L2SPM + uint8_t fill_dir =0; + switch (rw) { + case 0: + src_addr = src_addr & 0x0000000007FFFFFF; + dst_addr = dst_addr & 0x0000000007FFFFFF; + fill_dir = 0; + break; + case 1: + src_addr = src_addr & 0x0000000007FFFFFF; + dst_addr = dst_addr & 0x0000001FFFFFFFFF; + fill_dir = 1; + break; + case 2: + src_addr = src_addr & 0x0000001FFFFFFFFF; + dst_addr = dst_addr & 0x0000000007FFFFFF; + fill_dir = 0; + break; + case 3: + break; + default: + break; + } + + // fill destination + rs1 = ((uint64_t) fill_dir << 48) | dst_addr; + rs2 = ((uint64_t) fill_h << 48) | ((uint64_t) out_l << 32) | ((uint64_t) out_l); + rs3 = fill_pattern; + __opset(0, rs3); + __asm__ volatile("fill %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + + // move source data + uint32_t write_stride = out_l; + dst_addr = dst_addr + (out_l * ystr) + xstr; + + for(i = 0 ; i < channel_num ; i++) { + + // [01:00] : 11 = x, 10 = read DDR write L2SPM , 01 = read L2SPM write DDR , 00 = read L2SPM write L2SPM + switch (rw) { + case 0: + // L2 -> L2: copy + rs1 = ((uint64_t) dst_addr << 32) | ((uint64_t) src_addr); + rs2 = ((uint64_t) height << 48) | ((uint64_t) length << 32) | ((uint64_t) fill_stride); + rs3 = ((uint64_t) write_stride); + __opset(0, rs3); + __asm__ volatile("copy %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + break; + case 1: + // L2 -> DDR: store + rs1 = ((uint64_t) dst_addr << 27) | ((uint64_t) src_addr); + rs2 = ((uint64_t) height << 48) | ((uint64_t) length << 32) | ((uint64_t) write_stride); + rs3 = ((uint64_t) fill_stride); + __opset(0, rs3); + __asm__ volatile("store %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + break; + case 2: + // DDR -> l2: load + rs1 = ((uint64_t) src_addr << 27) | ((uint64_t) dst_addr); + rs2 = ((uint64_t) height << 48) | ((uint64_t) length << 32) | ((uint64_t) fill_stride); + rs3 = ((uint64_t) write_stride); + __opset(0, rs3); + __asm__ volatile("load %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + break; + case 3: + break; + default: + break; + } + + src_addr += channel_stride; + dst_addr += img_size; + } +} + + +// [load_3d] +// 3D load from L2SPM to L1SPM +// src_addr_L2SPM(28bit) / dst_addr_L1SPM(24bit) +// heigth(16bit) / length(16bit) / read_stride(32bit) / depth(16bit) / depth_stride(32bit) +__attribute__((noinline)) void __load_3d(uint32_t src_addr_L2SPM, uint32_t dst_addr_L1SPM, uint32_t read_stride, uint16_t length, uint16_t height, uint32_t depth_stride, uint16_t depth) +{ + volatile int i; + + uint64_t rs1, rs2; + uint32_t l1spm_offset = length * height; + + for(i = 0; i < depth; i++) { + rs1 = ((uint64_t) src_addr_L2SPM << 27) | ((uint64_t) dst_addr_L1SPM); + rs2 = ((uint64_t) height << 48) | ((uint64_t) length << 32) | ((uint64_t) read_stride); + __asm__ volatile ("load %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + + src_addr_L2SPM += depth_stride; + dst_addr_L1SPM += l1spm_offset; + } +} + + +// [store_3d] +// 3D store from L1SPM to L2SPM +// src_addr_L1SPM(24bit) / dst_addr_L2SPM(28bit) +// heigth(16bit) / length(16bit) / write_stride(32bit) / depth(16bit) / depth_stride(32bit) +__attribute__((noinline)) void __store_3d(uint32_t src_addr_L1SPM, uint32_t dst_addr_L2SPM, uint32_t write_stride, uint16_t length, uint16_t height, uint32_t depth_stride, uint16_t depth) +{ + volatile int i; + + uint64_t rs1, rs2; + uint32_t l1spm_offset = length * height; + + for(i = 0; i < depth; i++) { + rs1 = ((uint64_t) dst_addr_L2SPM << 27) | ((uint64_t) src_addr_L1SPM); + rs2 = ((uint64_t) height << 48) | ((uint64_t) length << 32) | ((uint64_t) write_stride); + __asm__ volatile ("store %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + + dst_addr_L2SPM += depth_stride; + src_addr_L1SPM += l1spm_offset; + } +} + + + +//================================= +// Special function - activation +//================================= +// [relu] +// relu(x) = (x < 0) ? 0 : x +// input : L1SPM_R_BANK (l1_R_addr) +// result: L1SPM_A_BANK (l1_A_addr) +__attribute__((noinline)) void __relu(uint32_t vector_size, uint32_t l1_A_addr, uint32_t l1_R_addr) +{ + // relu + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) 0x0); + __set_spm_addr_A(l1_R_addr); + __set_spm_addr_R(l1_A_addr); + __asm__ volatile("clamp.min %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + + // Restore L1SPM addr + __set_spm_addr_A(l1_A_addr); + __set_spm_addr_R(l1_R_addr); +} + + +// [relu6] +// relu6(x) = (x < 0) ? 0 : +// (x > 6) ? 6 : x +// input : L1SPM_R_BANK (l1_R_addr) +// result: L1SPM_A_BANK (l1_A_addr) +__attribute__((noinline)) void __relu6(uint32_t vector_size, uint32_t l1_A_addr, uint32_t l1_R_addr) +{ + uint64_t rs1, rs2; + + // relu6 (min clamp) + rs1 = ((uint64_t) vector_size); + rs2 = ((uint64_t) 0x0); + __set_spm_addr_A(l1_R_addr); + __set_spm_addr_R(l1_A_addr); + __asm__ volatile("clamp.min %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + + // relu6 (max clamp) + rs2 = ((uint64_t) 0x4600); + __set_spm_addr_A(l1_A_addr); + __asm__ volatile("clamp.max %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); + + // Restore L1SPM addr + __set_spm_addr_R(l1_R_addr); +} + + +// [lrelu] +// lrelu(x) = (x < 0) ? 0 : 0.01 * x +// input : L1SPM_R_BANK (l1_R_addr) +// result: L1SPM_A_BANK (l1_A_addr) +__attribute__((noinline)) void __lrelu(uint32_t vector_size) +{ + uint64_t rs1 = ((uint64_t) vector_size); + uint64_t rs2 = ((uint64_t) 0x211E); // fp16_hex(0.01) + __asm__ volatile ("prelu %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); +} + + +// [silu] +// silu(x) = x * sigmoid(x) +// input : L1SPM_R_BANK (l1_R_addr) +// result: L1SPM_A_BANK (l1_A_addr) +__attribute__((noinline)) void __silu(uint32_t vector_size, uint32_t l1_A_addr, uint32_t l1_B_addr, uint32_t l1_R_addr) +{ + // sigmoid(x) + uint64_t rs1 = ((uint64_t) vector_size); + __asm__ volatile ("sigm %[r1]\n" : : [r1] "r"(rs1)); // sigmoid(x) result: A bank + + // silu(x) = x * sigmoid(x) + __set_spm_addr_B(l1_R_addr); + __set_spm_addr_R(l1_A_addr); + __asm__ volatile ("mul.vv %[r1]\n" : : [r1] "r"(rs1)); // silu(x) result: A bank + + // Restore L1SPM addr + __set_spm_addr_A(l1_A_addr); + __set_spm_addr_B(l1_B_addr); + __set_spm_addr_R(l1_R_addr); +} + + + +//================================= +// Special function - math +//================================= +// [var] +// variance(x) = sum(((x-mean(x)/sqrt(x_num))^2) +// input : L1SPM_A_BANK (l1_A_addr) +// mean : SVR_1 +// result: SVR_2 +__attribute__((noinline)) void __var(uint32_t vector_size, uint32_t l1_A_addr, uint32_t l1_B_addr, uint32_t l1_R_addr) +{ + uint64_t rs1, rs2, rs3, rs4; + + // sum(x) + rs1 = ((uint64_t) vector_size << 16); + __asm__ volatile("mm.o %[r1]\n" : : [r1] "r"(rs1)); // sum(x): SVR_0 + + // mean(x) = sum(x) / x_num + rs1 = ((uint64_t) 0x0); + rs2 = ((uint64_t) __int16_to_fp16(vector_size)); + rs3 = ((uint64_t) 0x1); + __opset(0, rs3); + __asm__ volatile("div.is %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // mean(x): SVR_1 + + // x - mean(x) + rs1 = ((uint64_t) vector_size); + rs4 = ((uint64_t) 0x184); + __opset(1, rs4); + __asm__ volatile("sub.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // x - mean(x): R bank + + // store x_num to SVR + rs1 = ((uint64_t) SGPR_0); + rs2 = ((uint64_t) __int16_to_fp16(vector_size)); + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // x_num store: SVR_0 + + // sqrt(x_num) + rs1 = ((uint64_t) 0x0); + __asm__ volatile("sqrt.i %[r1]\n" : : [r1] "r"(rs1)); // sqrt(x_num): SVR_0 + + // (x - mean(x)) / sqrt(x_num) + rs1 = ((uint64_t) vector_size); + rs4 = ((uint64_t) 0x180); + __set_spm_addr_A(l1_R_addr); + __opset(1, rs4); + __asm__ volatile("div.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // (x - mean(x)) / sqrt(x_num): R bank + + // variance(x) = sum(((x - mean(x) / sqrt(x_num))^2) + rs1 = ((uint64_t) vector_size << 16); + rs3 = ((uint64_t) 0x2); + __set_spm_addr_B(l1_R_addr); + __opset(0, rs3); + __asm__ volatile ("mm.v %[r1]\n" : : [r1] "r"(rs1)); // variance(x): SVR_2 + + // Restore L1SPM addr + __set_spm_addr_A(l1_A_addr); + __set_spm_addr_B(l1_B_addr); +} + + + +//================================= +// Special function - normalization +//================================= +// [layernorm] +// layernorm(x) = (x-mean(x))/sqrt(var(x)+epsilon) +// input : L1SPM_A_BANK (l1_A_addr) +// result: L1SPM_R_BANK (l1_R_addr) +__attribute__((noinline)) void __layernorm(uint32_t vector_size, uint32_t l1_A_addr, uint32_t l1_B_addr, uint32_t l1_R_addr, uint16_t epsilon) +{ + uint64_t rs1, rs2, rs3, rs4; + + // varience(x) + // mean : SVR_1 + // variance: SVR_2 + __var(vector_size, l1_A_addr, l1_B_addr, l1_R_addr); + + // variance(x) + epsilon + if ((epsilon != 0x0000) || (epsilon != 0x8000)) { + rs1 = ((uint64_t) 0x2); + rs2 = ((uint64_t) epsilon); + rs3 = ((uint64_t) 0x2); + __opset(0, rs3); + __asm__ volatile("add.is %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // variance + epsilon: SVR_2 + } + + // sqrt(variance(x) + epsilon) + rs1 = ((uint64_t) 0x2); + rs3 = ((uint64_t) 0x3); + __opset(0, rs3); + __asm__ volatile("sqrt.i %[r1]\n" : : [r1] "r"(rs1)); // sqrt(variance(x) + epsilon): SVR_3 + + // x - mean(x) + rs1 = ((uint64_t) vector_size); + rs4 = ((uint64_t) 0x184); + __opset(1, rs4); + __asm__ volatile("sub.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // x - mean(x): R bank + + // layernorm_eps(x) = (x-mean(x))/sqrt(var(x)+epsilon) + rs4 = ((uint64_t) 0x18C); + __set_spm_addr_A(l1_R_addr); + __opset(1, rs4); + __asm__ volatile("div.vs %[r1], %[r2]\n" : : [r1] "r"(rs1) ,[r2] "r"(rs2)); // layernorm_eps(x): R bank + + // Restore L1SPM addr + __set_spm_addr_A(l1_A_addr); +} + + +// [layernorm_aff] +// layernorm_aff(x) = ((x-mean(x))/var(x)+epsilon) * scale + shift +// input : L1SPM_A_BANK (l1_A_addr) +// scale : L1SPM_B_BANK (l1_B_addr) +// shift : L1SPM_C_BANK (l1_C_addr) +// result: L1SPM_R_BANK (l1_R_addr) +__attribute__((noinline)) void __layernorm_aff(uint32_t vector_size, uint32_t l1_A_addr, uint32_t l1_B_addr, uint32_t l1_C_addr, uint32_t l1_R_addr, uint16_t epsilon) +{ + uint64_t rs1, rs2, rs3, rs4; + + // variance(x) + // mean : SVR_1 + // variance: SVR_2 + __var(vector_size, l1_A_addr, l1_B_addr, l1_R_addr); + + // variance(x) + epsilon + if ((epsilon != 0x0000) || (epsilon != 0x8000)) { + rs1 = ((uint64_t) 0x2); + rs2 = ((uint64_t) epsilon); + rs3 = ((uint64_t) 0x2); + __opset(0, rs3); + __asm__ volatile("add.is %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // variance + epsilon: SVR_2 + } + + // sqrt(variance(x) + epsilon) + rs1 = ((uint64_t) 0x2); + rs3 = ((uint64_t) 0x3); + __opset(0, rs3); + __asm__ volatile("sqrt.i %[r1]\n" : : [r1] "r"(rs1)); // sqrt(variance(x) + epsilon): SVR_3 + + // x - mean(x) + rs1 = ((uint64_t) vector_size); + rs4 = ((uint64_t) 0x184); + __opset(1, rs4); + __asm__ volatile("sub.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // x - mean(x): R bank + + // layernorm_eps(x) = (x-mean(x))/sqrt(variance(x)+epsilon) + rs4 = ((uint64_t) 0x18C); + __set_spm_addr_A(l1_R_addr); + __opset(1, rs4); + __asm__ volatile("div.vs %[r1], %[r2]\n" : : [r1] "r"(rs1) ,[r2] "r"(rs2)); // layernorm_eps(x): R bank + + // layernorm_eps(x) * scale + __asm__ volatile("mul.vv %[r1]\n" : : [r1] "r"(rs1)); // layernorm(x) * scale: R bank + + // layernorm_eps_aff(x) = layernorm_eps(x) * scale + shift + __set_spm_addr_B(l1_C_addr); + __asm__ volatile("add.vv %[r1]\n" : : [r1] "r"(rs1)); // layernorm_aff(x): R bank + + // Restore L1SPM addr + __set_spm_addr_A(l1_A_addr); + __set_spm_addr_B(l1_B_addr); +} + + +// [rmsnorm] +// rmsnorm(x) = x / R(MS(x)+epsilon) +// input : L1SPM_A_BANK (l1_A_addr) +// result: L1SPM_R_BANK (l1_R_addr) +__attribute__((noinline)) void __rmsnorm(uint32_t vector_size, uint32_t l1_A_addr, uint32_t l1_B_addr, uint32_t l1_R_addr, uint16_t epsilon) +{ + uint64_t rs1, rs2, rs3, rs4; + + // store x_num to SVR + rs1 = ((uint64_t) SGPR_0); + rs2 = ((uint64_t) __int16_to_fp16(vector_size)); + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // x_num: SVR_0 + + // sqrt(x_num) + rs1 = ((uint64_t) 0x0); + __asm__ volatile("sqrt.i %[r1]\n" : : [r1] "r"(rs1)); // sqrt(x_num): SVR_0 + + // x/sqrt(x_num) + rs1 = ((uint64_t) vector_size); + rs4 = ((uint64_t) 0x180); + __opset(1, rs4); + __asm__ volatile("div.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // x/sqrt(x_num): R bank + + // MS(x) = mean((x/sqrt(x_num))^2) + rs1 = ((uint64_t) vector_size << 16); + rs3 = ((uint64_t) 0x1); + __set_spm_addr_A(l1_R_addr); + __set_spm_addr_B(l1_R_addr); + __opset(0, rs3); + __asm__ volatile ("mm.v %[r1]\n" : : [r1] "r"(rs1)); // mean((x/sqrt(x_num))^2): SVR_1 + + // MS(x) + epsilon + if ((epsilon != 0x0000) || (epsilon != 0x8000)) { + rs1 = ((uint64_t) 0x1); + rs2 = ((uint64_t) epsilon); + __asm__ volatile("add.is %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // MS(x) + epsilon: SVR_0 + } + + // R(MS(x) + epsilon) = sqrt(MS(x) + epsilon) + rs1 = ((uint64_t) 0x0); + __asm__ volatile("sqrt.i %[r1]\n" : : [r1] "r"(rs1)); // RMS(x): SVR_0 + + // rmsnorm_eps(x) = x / R(MS(x) + epsilon) + rs1 = ((uint64_t) vector_size); + rs4 = ((uint64_t) 0x180); + __set_spm_addr_A(l1_A_addr); + __opset(1, rs4); + __asm__ volatile("div.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // rmsnorm_eps(x): R bank + + // Restore L1SPM addr + __set_spm_addr_B(l1_B_addr); +} + + +// [rmsnorm_scale] +// rmsnorm_scale(x) = (x/R(MS(x)+epsilon))* scale +// input : L1SPM_A_BANK (l1_A_addr) +// scale : L1SPM_B_BANK (l1_B_addr) +// result: L1SPM_R_BANK (l1_R_addr) +__attribute__((noinline)) void __rmsnorm_scale(uint32_t vector_size, uint32_t l1_A_addr, uint32_t l1_B_addr, uint32_t l1_R_addr, uint16_t epsilon) +{ + uint64_t rs1, rs2, rs3, rs4; + + // store x_num to SVR + rs1 = ((uint64_t) SGPR_0); + rs2 = ((uint64_t) __int16_to_fp16(vector_size)); + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // x_num: SVR_0 + + // sqrt(x_num) + rs1 = ((uint64_t) 0x0); + __asm__ volatile("sqrt.i %[r1]\n" : : [r1] "r"(rs1)); // sqrt(x_num): SVR_0 + + // x/sqrt(x_num) + rs1 = ((uint64_t) vector_size); + rs4 = ((uint64_t) 0x180); + __opset(1, rs4); + __asm__ volatile("div.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // x/sqrt(x_num): R bank + + // MS(x) = mean((x/sqrt(x_num))^2) + rs1 = ((uint64_t) vector_size << 16); + rs3 = ((uint64_t) 0x1); + __set_spm_addr_A(l1_R_addr); + __set_spm_addr_B(l1_R_addr); + __opset(0, rs3); + __asm__ volatile ("mm.v %[r1]\n" : : [r1] "r"(rs1)); // mean((x/sqrt(x_num))^2): SVR_1 + + // MS(x) + epsilon + if ((epsilon != 0x0000) || (epsilon != 0x8000)) { + rs1 = ((uint64_t) 0x1); + rs2 = ((uint64_t) epsilon); + __asm__ volatile("add.is %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // MS(x) + epsilon: SVR_0 + } + + // R(MS(x) + epsilon) = sqrt(MS(x) + epsilon) + rs1 = ((uint64_t) 0x0); + __asm__ volatile("sqrt.i %[r1]\n" : : [r1] "r"(rs1)); // RMS(x): SVR_0 + + // rmsnorm_eps(x) = x / R(MS(x) + epsilon) + rs1 = ((uint64_t) vector_size); + rs4 = ((uint64_t) 0x180); + __set_spm_addr_A(l1_A_addr); + __opset(1, rs4); + __asm__ volatile("div.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // rmsnorm_eps(x): R bank + + // rmsnorm_eps_scale(x) = (x/R(MS(x) + epsilon))* scale + __set_spm_addr_A(l1_R_addr); + __set_spm_addr_B(l1_B_addr); + __asm__ volatile("mul.vv %[r1]\n" : : [r1] "r"(rs1)); // rmsnorm_eps_scale(x): R Bank + + // Restore L1SPM addr + __set_spm_addr_A(l1_A_addr); +} + + +// [batchnorm] +// batchnorm(x) = (x-mean)/sqrt(var_epsilon) +// input : L1SPM_A_BANK (l1_A_addr) +// mean : fp16 value +// var : fp16 value +// result: L1SPM_R_BANK (l1_R_addr) +__attribute__((noinline)) void __batchnorm(uint32_t vector_size, uint16_t mean, uint16_t var, uint32_t l1_A_addr, uint32_t l1_R_addr, uint16_t epsilon) +{ + uint64_t rs1, rs2, rs4; + + // x - mean + rs1 = ((uint64_t) vector_size); + rs2 = ((uint64_t) mean); + __set_spm_addr_R(l1_A_addr); + __asm__ volatile("sub.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // x - mean result: A bank + + // store variacne to SVR + rs1 = ((uint64_t) SGPR_0); + rs2 = ((uint64_t) var); + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // var: SVR_0 + + // var + epsilon + if ((epsilon != 0x0000) || (epsilon != 0x8000)) { + rs1 = ((uint64_t) 0x0); + rs2 = ((uint64_t) epsilon); + __asm__ volatile("add.is %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // var + epsilon: SVR_0 + } + + // sqrt(var + epsilon) + rs1 = ((uint64_t) 0x0); + __asm__ volatile("sqrt.i %[r1]\n" : : [r1] "r"(rs1)); // sqrt(var + epsilon): SVR_0 + + // batchnorm_eps(x) = (x - mean) / sqrt(var + epsilon) + rs1 = ((uint64_t) vector_size); + rs4 = ((uint64_t) 0x180); + __set_spm_addr_R(l1_R_addr); + __opset(1, rs4); + __asm__ volatile("div.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // batchnorm_eps(x): R bank +} + + +// [batchnorm_aff] +// batchnorm_aff(x) = (x-mean)/sqrt(var+epsilon)*scale + shift +// input : L1SPM_A_BANK (l1_A_addr) +// mean : fp16 value +// var : fp16 value +// scale : fp16 value +// shift : fp16 value +// result: L1SPM_R_BANK (l1_R_addr) +__attribute__((noinline)) void __batchnorm_aff(uint32_t vector_size, uint16_t mean, uint16_t var, uint16_t scale, uint16_t shift, uint32_t l1_A_addr, uint32_t l1_R_addr, uint16_t epsilon) +{ + uint64_t rs1, rs2, rs4; + + // x - mean + rs1 = ((uint64_t) vector_size); + rs2 = ((uint64_t) mean); + __set_spm_addr_R(l1_A_addr); + __asm__ volatile("sub.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // x - mean result: A bank + + // store variacne to SVR + rs1 = ((uint64_t) SGPR_0); + rs2 = ((uint64_t) var); + __asm__ volatile("wrspr %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // var: SVR_0 + + // var + epsilon + if ((epsilon != 0x0000) || (epsilon != 0x8000)) { + rs1 = ((uint64_t) 0x0); + rs2 = ((uint64_t) epsilon); + __asm__ volatile("add.is %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // var + epsilon: SVR_0 + } + + // sqrt(var + epsilon) + rs1 = ((uint64_t) 0x0); + __asm__ volatile("sqrt.i %[r1]\n" : : [r1] "r"(rs1)); // sqrt(var + epsilon): SVR_0 + + // batchnorm_eps(x) = (x - mean) / sqrt(var + epsilon) + rs1 = ((uint64_t) vector_size); + rs4 = ((uint64_t) 0x180); + __opset(1, rs4); + __asm__ volatile("div.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // batchnorm_eps(x): A bank + + // batchnorm_eps(x) * scale + rs2 = ((uint64_t) scale); + __asm__ volatile("mul.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // batchnorm_eps(x) * scale: A Bank + + // batchnorm_eps_aff(x) = batchnorm_eps(x) * scale + shift + rs2 = ((uint64_t) shift); + __set_spm_addr_R(l1_R_addr); + __asm__ volatile("add.vs %[r1], %[r2]\n" : : [r1] "r"(rs1), [r2] "r"(rs2)); // batchnorm_eps_aff(x): R Bank +} + + + +#endif diff --git a/depend/ggml-gtx/src/pcie_irq_host.cpp b/depend/ggml-gtx/src/pcie_irq_host.cpp new file mode 100644 index 0000000..aa8428c --- /dev/null +++ b/depend/ggml-gtx/src/pcie_irq_host.cpp @@ -0,0 +1,1120 @@ +/** + * @file pcie_irq_host.cpp + * @brief GTX PCIe Host-side Interrupt Implementation + * + * Implements vfio-user binary protocol client for communicating with + * GTX_ISS simulator running with USE_VFIO_USER=1. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "pcie_irq_host.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __linux__ +#include +#include +#include +#include +#include +#include +#include +#include +#include +/* Memory barrier for GCC/Clang */ +#define MEMORY_BARRIER() __sync_synchronize() +#else +/* Memory barrier for Windows (placeholder) */ +#define MEMORY_BARRIER() do {} while(0) +#endif + +/*============================================================================= + * vfio-user Protocol Definitions (Linux only) + *============================================================================*/ + +#ifdef __linux__ + +/* vfio-user protocol version */ +#define LIB_VFIO_USER_MAJOR 0 +#define LIB_VFIO_USER_MINOR 1 + +/* vfio-user commands */ +enum vfio_user_command { + VFIO_USER_VERSION = 1, + VFIO_USER_DMA_MAP = 2, + VFIO_USER_DMA_UNMAP = 3, + VFIO_USER_DEVICE_GET_INFO = 4, + VFIO_USER_DEVICE_GET_REGION_INFO = 5, + VFIO_USER_DEVICE_GET_REGION_IO_FDS = 6, + VFIO_USER_DEVICE_GET_IRQ_INFO = 7, + VFIO_USER_DEVICE_SET_IRQS = 8, + VFIO_USER_REGION_READ = 9, + VFIO_USER_REGION_WRITE = 10, + VFIO_USER_DMA_READ = 11, + VFIO_USER_DMA_WRITE = 12, + VFIO_USER_DEVICE_RESET = 13, + VFIO_USER_MAX, +}; + +/* vfio-user message header flags */ +#define VFIO_USER_F_TYPE_COMMAND (0x0) +#define VFIO_USER_F_TYPE_REPLY (0x1) +#define VFIO_USER_F_NO_REPLY (0x10) +#define VFIO_USER_F_ERROR (0x20) + +/* vfio-user message header */ +struct vfio_user_header { + uint16_t msg_id; + uint16_t cmd; + uint32_t msg_size; + uint32_t flags; + uint32_t error_no; +} __attribute__((packed)); + +/* vfio-user version */ +struct vfio_user_version { + uint16_t major; + uint16_t minor; + uint8_t data[]; +} __attribute__((packed)); + +/* vfio-user device info */ +struct vfio_user_device_info { + uint32_t argsz; + uint32_t flags; + uint32_t num_regions; + uint32_t num_irqs; +} __attribute__((packed)); + +/* vfio-user region access */ +struct vfio_user_region_access { + uint64_t offset; + uint32_t region; + uint32_t count; + uint8_t data[]; +} __attribute__((packed)); + +/* PCIe region indices */ +#define VFU_PCI_DEV_BAR0_REGION_IDX 0 +#define VFU_PCI_DEV_BAR1_REGION_IDX 1 +#define VFU_PCI_DEV_BAR2_REGION_IDX 2 +#define VFU_PCI_DEV_BAR3_REGION_IDX 3 +#define VFU_PCI_DEV_BAR4_REGION_IDX 4 +#define VFU_PCI_DEV_BAR5_REGION_IDX 5 +#define VFU_PCI_DEV_ROM_REGION_IDX 6 +#define VFU_PCI_DEV_CFG_REGION_IDX 7 +#define VFU_PCI_DEV_VGA_REGION_IDX 8 +#define VFU_PCI_DEV_NUM_REGIONS 9 + +#endif /* __linux__ */ + +/*============================================================================= + * Debug Logging + *============================================================================*/ + +static bool g_irq_debug = false; +static std::atomic g_irq_connection_broken(false); + +#define IRQ_LOG(...) \ + do { if (g_irq_debug) fprintf(stderr, "[GTX-IRQ] " __VA_ARGS__); } while(0) + +#define IRQ_LOG_ERROR(...) \ + fprintf(stderr, "[GTX-IRQ ERROR] " __VA_ARGS__) + +static void irq_log_connection_broken_once(const char * msg) { + bool expected = false; + if (g_irq_connection_broken.compare_exchange_strong(expected, true)) { + IRQ_LOG_ERROR("%s\n", msg); + } +} + +static bool irq_is_connection_broken_errno(int err) { + return err == EPIPE || err == ECONNRESET || err == ENOTCONN; +} + +/*============================================================================= + * IRQ Context Structure + *============================================================================*/ + +struct gtx_pcie_irq_ctx { + /* Mode */ + enum { + MODE_VFIO_USER, + MODE_HARDWARE, + MODE_MAPPED + } mode; + + /* BAR0 mapping */ + volatile uint8_t* bar0_base; + size_t bar0_size; + bool bar0_owned; /* true if we should unmap on destroy */ + + /* vfio-user socket (for simulator) */ + int vfio_socket; + + /* MSI handlers */ + struct { + gtx_msi_handler_t handler; + void* user_data; + } msi_handlers[GTX_MSI_MAX]; + + /* MSI eventfds (one per vector for epoll integration) */ + int msi_eventfds[GTX_MSI_MAX]; + + /* Pending MSI mask (atomic for thread safety) */ + std::atomic pending_msi; + + /* MSI polling thread */ + std::thread msi_thread; + std::atomic msi_thread_running; + + /* Mutex for handler registration */ + std::mutex handler_mutex; + + gtx_pcie_irq_ctx() { + mode = MODE_MAPPED; + bar0_base = nullptr; + bar0_size = 0; + bar0_owned = false; + vfio_socket = -1; + pending_msi = 0; + msi_thread_running = false; + + memset(msi_handlers, 0, sizeof(msi_handlers)); + for (int i = 0; i < GTX_MSI_MAX; i++) { + msi_eventfds[i] = -1; + } + } + + ~gtx_pcie_irq_ctx() { + /* Stop MSI thread */ + if (msi_thread_running) { + msi_thread_running = false; + if (msi_thread.joinable()) { + msi_thread.join(); + } + } + + /* Close eventfds */ + for (int i = 0; i < GTX_MSI_MAX; i++) { + if (msi_eventfds[i] >= 0) { +#ifdef __linux__ + close(msi_eventfds[i]); +#endif + msi_eventfds[i] = -1; + } + } + + /* Close vfio socket */ + if (vfio_socket >= 0) { +#ifdef __linux__ + close(vfio_socket); +#endif + vfio_socket = -1; + } + + /* Unmap BAR0 if owned */ + if (bar0_owned && bar0_base) { +#ifdef __linux__ + munmap(const_cast(bar0_base), bar0_size); +#endif + bar0_base = nullptr; + } + } +}; + +/*============================================================================= + * vfio-user Binary Protocol Helpers + *============================================================================*/ + +#ifdef __linux__ + +static std::atomic g_msg_id(1); + +/** + * Send vfio-user message with iovec + */ +static int vfio_send_iovec(int sock, uint16_t msg_id, bool is_reply, + enum vfio_user_command cmd, + struct iovec *iovecs, size_t nr_iovecs, + int *fds, int fd_count) { + struct vfio_user_header hdr; + struct msghdr msg; + size_t i; + + memset(&hdr, 0, sizeof(hdr)); + hdr.msg_id = msg_id; + hdr.cmd = cmd; + hdr.flags = is_reply ? VFIO_USER_F_TYPE_REPLY : VFIO_USER_F_TYPE_COMMAND; + + /* Calculate total message size */ + hdr.msg_size = sizeof(hdr); + for (i = 1; i < nr_iovecs; i++) { + hdr.msg_size += iovecs[i].iov_len; + } + + /* Set header as first iovec */ + iovecs[0].iov_base = &hdr; + iovecs[0].iov_len = sizeof(hdr); + + memset(&msg, 0, sizeof(msg)); + msg.msg_iov = iovecs; + msg.msg_iovlen = nr_iovecs; + + /* Add file descriptors if provided */ + char cmsg_buf[CMSG_SPACE(sizeof(int) * 32)]; + if (fds != NULL && fd_count > 0) { + msg.msg_control = cmsg_buf; + msg.msg_controllen = CMSG_SPACE(sizeof(int) * fd_count); + + struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(int) * fd_count); + memcpy(CMSG_DATA(cmsg), fds, sizeof(int) * fd_count); + } + + ssize_t ret = sendmsg(sock, &msg, MSG_NOSIGNAL); + if (ret == -1) { + if (irq_is_connection_broken_errno(errno)) { + irq_log_connection_broken_once("vfio-user connection closed"); + } else { + IRQ_LOG_ERROR("sendmsg failed: %s\n", strerror(errno)); + } + return -1; + } + if ((size_t)ret < hdr.msg_size) { + IRQ_LOG_ERROR("short write: expected=%u, actual=%zd\n", hdr.msg_size, ret); + return -1; + } + + return 0; +} + +/** + * Send vfio-user message with data buffer + */ +static int vfio_send(int sock, uint16_t msg_id, bool is_reply, + enum vfio_user_command cmd, + void *data, size_t data_len) { + struct iovec iovecs[2] = { + { nullptr, 0 }, /* Header placeholder */ + { data, data_len } + }; + return vfio_send_iovec(sock, msg_id, is_reply, cmd, iovecs, 2, nullptr, 0); +} + +/** + * Receive vfio-user message header and optional data + */ +static int vfio_recv(int sock, struct vfio_user_header *hdr, bool is_reply, + uint16_t *expected_msg_id, void *data, size_t *len, + int *fds, size_t *nr_fds) { + struct iovec iov = { hdr, sizeof(*hdr) }; + struct msghdr msg; + char cmsg_buf[CMSG_SPACE(sizeof(int) * 32)]; + + memset(&msg, 0, sizeof(msg)); + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + if (nr_fds != NULL && *nr_fds > 0) { + msg.msg_control = cmsg_buf; + msg.msg_controllen = sizeof(cmsg_buf); + } + + ssize_t ret = recvmsg(sock, &msg, 0); + if (ret < 0) { + if (irq_is_connection_broken_errno(errno)) { + irq_log_connection_broken_once("vfio-user connection closed"); + } else { + IRQ_LOG_ERROR("recvmsg failed: %s\n", strerror(errno)); + } + return -1; + } + if (ret == 0) { + irq_log_connection_broken_once("vfio-user connection closed"); + return -1; + } + if ((size_t)ret < sizeof(*hdr)) { + IRQ_LOG_ERROR("short header read: %zd\n", ret); + return -1; + } + + /* Parse file descriptors */ + if (nr_fds != NULL) { + *nr_fds = 0; + for (struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg); + cmsg != NULL; + cmsg = CMSG_NXTHDR(&msg, cmsg)) { + if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) { + size_t fd_size = cmsg->cmsg_len - CMSG_LEN(0); + *nr_fds = fd_size / sizeof(int); + if (fds != NULL) { + memcpy(fds, CMSG_DATA(cmsg), fd_size); + } + break; + } + } + } + + /* Validate reply */ + if (is_reply) { + if (expected_msg_id != NULL && hdr->msg_id != *expected_msg_id) { + IRQ_LOG_ERROR("msg_id mismatch: expected=%u, got=%u\n", + *expected_msg_id, hdr->msg_id); + return -1; + } + if ((hdr->flags & 0xf) != VFIO_USER_F_TYPE_REPLY) { + IRQ_LOG_ERROR("expected reply, got command\n"); + return -1; + } + if (hdr->flags & VFIO_USER_F_ERROR) { + IRQ_LOG_ERROR("server error: %u\n", hdr->error_no); + errno = hdr->error_no; + return -1; + } + } + + /* Read data if present */ + if (len != NULL && *len > 0 && hdr->msg_size > sizeof(*hdr)) { + size_t data_size = hdr->msg_size - sizeof(*hdr); + if (data_size > *len) { + data_size = *len; + } + ret = recv(sock, data, data_size, MSG_WAITALL); + if (ret < 0) { + IRQ_LOG_ERROR("recv data failed: %s\n", strerror(errno)); + return -1; + } + *len = ret; + } + + return 0; +} + +/** + * Send and receive vfio-user message (synchronous) + */ +static int vfio_msg(int sock, uint16_t msg_id, enum vfio_user_command cmd, + void *send_data, size_t send_len, + void *recv_data, size_t recv_len) { + struct iovec iovecs[2] = { + { nullptr, 0 }, + { send_data, send_len } + }; + + int ret = vfio_send_iovec(sock, msg_id, false, cmd, iovecs, 2, nullptr, 0); + if (ret < 0) { + return ret; + } + + struct vfio_user_header hdr; + size_t len = recv_len; + ret = vfio_recv(sock, &hdr, true, &msg_id, recv_data, &len, nullptr, nullptr); + if (ret < 0) { + return ret; + } + return (int) len; +} + +/** + * Negotiate vfio-user protocol version + */ +static int vfio_negotiate(int sock) { + /* Send version */ + struct vfio_user_version ver = { LIB_VFIO_USER_MAJOR, LIB_VFIO_USER_MINOR }; + const char *caps = "{\"capabilities\":{\"max_msg_fds\":8,\"max_data_xfer_size\":1048576}}"; + size_t caps_len = strlen(caps) + 1; + + struct iovec iovecs[3] = { + { nullptr, 0 }, + { &ver, sizeof(ver) }, + { (void*)caps, caps_len } + }; + + uint16_t msg_id = g_msg_id++; + int ret = vfio_send_iovec(sock, msg_id, false, VFIO_USER_VERSION, + iovecs, 3, nullptr, 0); + if (ret < 0) { + IRQ_LOG_ERROR("Failed to send version\n"); + return -1; + } + + /* Receive version response */ + struct vfio_user_header hdr; + char recv_buf[1024]; + size_t recv_len = sizeof(recv_buf); + ret = vfio_recv(sock, &hdr, true, &msg_id, recv_buf, &recv_len, nullptr, nullptr); + if (ret < 0) { + IRQ_LOG_ERROR("Failed to receive version\n"); + return -1; + } + + IRQ_LOG("vfio-user version negotiated\n"); + return 0; +} + +/** + * Get device info + */ +static int vfio_get_device_info(int sock, struct vfio_user_device_info *info) { + info->argsz = sizeof(*info); + + uint16_t msg_id = g_msg_id++; + int ret = vfio_msg(sock, msg_id, VFIO_USER_DEVICE_GET_INFO, + info, sizeof(*info), info, sizeof(*info)); + if (ret < 0) { + IRQ_LOG_ERROR("Failed to get device info\n"); + return -1; + } + + IRQ_LOG("Device info: flags=%u, regions=%u, irqs=%u\n", + info->flags, info->num_regions, info->num_irqs); + return 0; +} + +/** + * Write to device region (for doorbell) + */ +static int vfio_region_write(int sock, uint32_t region, uint64_t offset, + void *data, size_t count) { + size_t msg_size = sizeof(struct vfio_user_region_access) + count; + uint8_t *buf = (uint8_t*)alloca(msg_size); + + struct vfio_user_region_access *access = (struct vfio_user_region_access *)buf; + access->offset = offset; + access->region = region; + access->count = count; + memcpy(access->data, data, count); + + struct vfio_user_region_access reply; + uint16_t msg_id = g_msg_id++; + int ret = vfio_msg(sock, msg_id, VFIO_USER_REGION_WRITE, + buf, msg_size, &reply, sizeof(reply)); + if (ret < 0) { + if (!g_irq_connection_broken.load()) { + IRQ_LOG_ERROR("Region write failed: msg_id=%u region=%u offset=0x%lx count=%zu msg_size=%zu\n", + msg_id, region, (unsigned long) offset, count, msg_size); + } + return -1; + } + + IRQ_LOG("Region write: region=%u offset=0x%lx count=%zu\n", + region, (unsigned long)offset, count); + return 0; +} + +/** + * Read from device region + */ +static int vfio_region_read(int sock, uint32_t region, uint64_t offset, + void *data, size_t count) { + struct vfio_user_region_access req; + req.offset = offset; + req.region = region; + req.count = count; + + size_t reply_size = sizeof(struct vfio_user_region_access) + count; + uint8_t *reply_buf = (uint8_t*)alloca(reply_size); + + uint16_t msg_id = g_msg_id++; + int ret = vfio_msg(sock, msg_id, VFIO_USER_REGION_READ, + &req, sizeof(req), reply_buf, reply_size); + if (ret < 0) { + if (!g_irq_connection_broken.load()) { + IRQ_LOG_ERROR("Region read failed\n"); + } + return -1; + } + + if (g_irq_debug && region == VFU_PCI_DEV_BAR2_REGION_IDX && offset == 0x8000) { + const uint32_t *rw = (const uint32_t *) reply_buf; + IRQ_LOG("BAR2 slot reply ret=%d raw32=[%08x %08x %08x %08x %08x %08x %08x %08x]\n", + ret, rw[0], rw[1], rw[2], rw[3], rw[4], rw[5], rw[6], rw[7]); + } + + /* libvfio-user replies with raw region bytes, while the in-tree custom + * vfio path can reply with vfio_user_region_access + payload. + * Accept either encoding. + */ + if ((size_t)ret >= count && (size_t)ret < sizeof(struct vfio_user_region_access) + count) { + memcpy(data, reply_buf, count); + } else { + struct vfio_user_region_access *reply = (struct vfio_user_region_access *)reply_buf; + memcpy(data, reply->data, count); + } + + if (g_irq_debug && region == VFU_PCI_DEV_BAR2_REGION_IDX && offset == 0x8000 && count >= 16) { + const uint32_t *w = (const uint32_t *) data; + IRQ_LOG("BAR2 slot read raw ret=%d words=[%08x %08x %08x %08x]\n", + ret, w[0], w[1], w[2], w[3]); + } + + IRQ_LOG("Region read: region=%u offset=0x%lx count=%zu\n", + region, (unsigned long)offset, count); + return 0; +} + +/** + * Configure IRQ eventfds + */ +static int vfio_set_irqs(int sock, uint32_t index, uint32_t start, uint32_t count, + int *fds) { + struct vfio_irq_set irq_set; + irq_set.argsz = sizeof(irq_set); + irq_set.flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER; + irq_set.index = index; + irq_set.start = start; + irq_set.count = count; + + struct iovec iovecs[2] = { + { nullptr, 0 }, + { &irq_set, sizeof(irq_set) } + }; + + uint16_t msg_id = g_msg_id++; + int ret = vfio_send_iovec(sock, msg_id, false, VFIO_USER_DEVICE_SET_IRQS, + iovecs, 2, fds, count); + if (ret < 0) { + IRQ_LOG_ERROR("Failed to send SET_IRQS\n"); + return -1; + } + + struct vfio_user_header hdr; + ret = vfio_recv(sock, &hdr, true, &msg_id, nullptr, nullptr, nullptr, nullptr); + if (ret < 0) { + IRQ_LOG_ERROR("Failed to receive SET_IRQS response\n"); + return -1; + } + + IRQ_LOG("IRQs configured: index=%u start=%u count=%u\n", index, start, count); + return 0; +} + +#endif /* __linux__ */ + +/*============================================================================= + * MSI Polling Thread (for vfio-user mode) + *============================================================================*/ + +#ifdef __linux__ + +/* PCIe addresses in simulator memory map */ +#define PCIE_SLAVE_BASE 0x60000000 +#define PCIE_CFG_BASE 0x40A00000 +#define PCIE_DBI_MSI 0x40000000 + 0x200D70 + +/** + * MSI eventfd monitoring thread + * Polls all registered MSI eventfds and calls handlers when triggered + */ +static void msi_poll_thread(gtx_pcie_irq_ctx* ctx) { + IRQ_LOG("MSI poll thread started\n"); + + struct pollfd pfds[GTX_MSI_MAX]; + int nfds = 0; + + while (ctx->msi_thread_running) { + /* Build poll array from active eventfds */ + nfds = 0; + for (int i = 0; i < GTX_MSI_MAX; i++) { + if (ctx->msi_eventfds[i] >= 0) { + pfds[nfds].fd = ctx->msi_eventfds[i]; + pfds[nfds].events = POLLIN; + pfds[nfds].revents = 0; + nfds++; + } + } + + if (nfds == 0) { + /* No eventfds registered, just sleep */ + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + continue; + } + + /* Poll with timeout */ + int ret = poll(pfds, nfds, 100); /* 100ms timeout */ + if (ret < 0) { + if (errno == EINTR) continue; + IRQ_LOG_ERROR("poll failed: %s\n", strerror(errno)); + break; + } + + if (ret == 0) { + /* Timeout, continue */ + continue; + } + + /* Check which eventfds were triggered */ + int pfd_idx = 0; + for (int i = 0; i < GTX_MSI_MAX && pfd_idx < nfds; i++) { + if (ctx->msi_eventfds[i] < 0) continue; + + if (pfds[pfd_idx].revents & POLLIN) { + /* Read and clear eventfd */ + uint64_t val; + (void)read(ctx->msi_eventfds[i], &val, sizeof(val)); + + /* Set pending bit */ + ctx->pending_msi.fetch_or(1U << i); + + /* Call handler if registered */ + std::lock_guard lock(ctx->handler_mutex); + if (ctx->msi_handlers[i].handler) { + IRQ_LOG("MSI %d triggered, calling handler\n", i); + ctx->msi_handlers[i].handler(i, ctx->msi_handlers[i].user_data); + } + } + pfd_idx++; + } + } + + IRQ_LOG("MSI poll thread stopped\n"); +} + +#endif /* __linux__ */ + +/*============================================================================= + * Initialization + *============================================================================*/ + +extern "C" { + +gtx_pcie_irq_ctx* gtx_pcie_irq_create_vfio(const char* socket_path) { +#ifndef __linux__ + IRQ_LOG_ERROR("vfio-user mode only supported on Linux\n"); + return nullptr; +#else + g_irq_connection_broken.store(false); + + if (!socket_path) { + IRQ_LOG_ERROR("socket_path is NULL\n"); + return nullptr; + } + + /* Check if socket exists */ + struct stat st; + if (stat(socket_path, &st) != 0) { + IRQ_LOG_ERROR("Socket not found: %s\n", socket_path); + return nullptr; + } + + /* Create socket and connect */ + int sock = socket(AF_UNIX, SOCK_STREAM, 0); + if (sock < 0) { + IRQ_LOG_ERROR("Failed to create socket: %s\n", strerror(errno)); + return nullptr; + } + + struct sockaddr_un addr; + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, socket_path, sizeof(addr.sun_path) - 1); + + if (connect(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) { + IRQ_LOG_ERROR("Failed to connect to %s: %s\n", socket_path, strerror(errno)); + close(sock); + return nullptr; + } + + IRQ_LOG("Connected to vfio-user socket: %s\n", socket_path); + + /* Negotiate vfio-user protocol version */ + if (vfio_negotiate(sock) < 0) { + IRQ_LOG_ERROR("Protocol negotiation failed\n"); + close(sock); + return nullptr; + } + + /* Get device info */ + struct vfio_user_device_info dev_info; + memset(&dev_info, 0, sizeof(dev_info)); + if (vfio_get_device_info(sock, &dev_info) < 0) { + IRQ_LOG_ERROR("Failed to get device info\n"); + close(sock); + return nullptr; + } + + /* Create context */ + gtx_pcie_irq_ctx* ctx = new gtx_pcie_irq_ctx(); + ctx->mode = gtx_pcie_irq_ctx::MODE_VFIO_USER; + ctx->vfio_socket = sock; + + /* Create eventfds for MSI vectors */ + for (int i = 0; i < GTX_MSI_MAX; i++) { + ctx->msi_eventfds[i] = eventfd(0, EFD_NONBLOCK); + if (ctx->msi_eventfds[i] < 0) { + IRQ_LOG_ERROR("Failed to create eventfd %d: %s\n", i, strerror(errno)); + } + } + + /* Configure MSI IRQs with eventfds */ + /* VFU_DEV_MSI_IRQ = 1 (MSI interrupt type, supports up to 32 vectors per CLAUDE.md) */ + #define VFU_DEV_MSI_IRQ 1 + if (dev_info.num_irqs > 1) { + /* Register MSI vectors with eventfds (up to 32 as per CLAUDE.md) */ + int msi_fds[GTX_MSI_MAX]; + int num_msi = GTX_MSI_MAX; + for (int i = 0; i < num_msi; i++) { + msi_fds[i] = ctx->msi_eventfds[i]; + } + + if (vfio_set_irqs(sock, VFU_DEV_MSI_IRQ, 0, num_msi, msi_fds) < 0) { + IRQ_LOG("Failed to configure MSI IRQs (may not be supported)\n"); + /* Not a fatal error - device may not support MSI */ + } + } + + /* Start MSI poll thread */ + ctx->msi_thread_running = true; + ctx->msi_thread = std::thread(msi_poll_thread, ctx); + + IRQ_LOG("vfio-user context created successfully\n"); + + return ctx; +#endif +} + +gtx_pcie_irq_ctx* gtx_pcie_irq_create_hw(const char* device_path, int group_id) { + (void)device_path; + (void)group_id; + + /* TODO: Implement real VFIO hardware support */ + IRQ_LOG_ERROR("Hardware VFIO mode not implemented\n"); + return nullptr; +} + +gtx_pcie_irq_ctx* gtx_pcie_irq_create_mapped(volatile void* bar0_base, size_t bar0_size) { + if (!bar0_base || bar0_size == 0) { + IRQ_LOG_ERROR("Invalid BAR0 mapping\n"); + return nullptr; + } + + gtx_pcie_irq_ctx* ctx = new gtx_pcie_irq_ctx(); + ctx->mode = gtx_pcie_irq_ctx::MODE_MAPPED; + ctx->bar0_base = (volatile uint8_t*)bar0_base; + ctx->bar0_size = bar0_size; + ctx->bar0_owned = false; + + IRQ_LOG("Created mapped IRQ context: bar0=%p size=%zu\n", bar0_base, bar0_size); + + return ctx; +} + +void gtx_pcie_irq_destroy(gtx_pcie_irq_ctx* ctx) { + if (ctx) { + delete ctx; + } +} + +/*============================================================================= + * MSI Handling + *============================================================================*/ + +int gtx_pcie_irq_register_msi(gtx_pcie_irq_ctx* ctx, int vector, + gtx_msi_handler_t handler, void* user_data) { + if (!ctx || vector < 0 || vector >= GTX_MSI_MAX) { + return -1; + } + + std::lock_guard lock(ctx->handler_mutex); + ctx->msi_handlers[vector].handler = handler; + ctx->msi_handlers[vector].user_data = user_data; + + IRQ_LOG("Registered MSI handler for vector %d\n", vector); + return 0; +} + +void gtx_pcie_irq_unregister_msi(gtx_pcie_irq_ctx* ctx, int vector) { + if (!ctx || vector < 0 || vector >= GTX_MSI_MAX) { + return; + } + + std::lock_guard lock(ctx->handler_mutex); + ctx->msi_handlers[vector].handler = nullptr; + ctx->msi_handlers[vector].user_data = nullptr; +} + +int gtx_pcie_irq_wait_msi(gtx_pcie_irq_ctx* ctx, int vector, int timeout_ms) { + if (!ctx) { + return -1; + } + + const uint32_t mask = (vector >= 0) ? (1U << vector) : 0xFFFFFFFF; + + auto consume_pending = [&]() -> int { + uint32_t pending = ctx->pending_msi.load() & mask; + if (!pending) { + return -1; + } + + int found = __builtin_ctz(pending); + ctx->pending_msi.fetch_and(~(1U << found)); + return found; + }; + + int found = consume_pending(); + if (found >= 0) { + return found; + } + +#ifdef __linux__ + /* Use eventfd if available */ + if (vector >= 0 && vector < GTX_MSI_MAX && ctx->msi_eventfds[vector] >= 0) { + struct pollfd pfd = { ctx->msi_eventfds[vector], POLLIN, 0 }; + int ret = poll(&pfd, 1, timeout_ms); + if (ret > 0 && (pfd.revents & POLLIN)) { + uint64_t val; + (void)read(ctx->msi_eventfds[vector], &val, sizeof(val)); + return vector; + } + + return consume_pending(); + } +#endif + + /* Fallback: poll pending_msi */ + auto start = std::chrono::steady_clock::now(); + while (true) { + int found = consume_pending(); + if (found >= 0) { + return found; + } + + auto now = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast(now - start).count(); + if (timeout_ms >= 0 && elapsed >= timeout_ms) { + return -1; /* Timeout */ + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } +} + +int gtx_pcie_irq_poll_msi(gtx_pcie_irq_ctx* ctx) { + if (!ctx) { + return -1; + } + + uint32_t pending = ctx->pending_msi.load(); + if (pending) { + int found = __builtin_ctz(pending); + ctx->pending_msi.fetch_and(~(1U << found)); + return found; + } + + return -1; +} + +uint32_t gtx_pcie_irq_get_pending_msi(gtx_pcie_irq_ctx* ctx) { + return ctx ? ctx->pending_msi.load() : 0; +} + +void gtx_pcie_irq_clear_msi(gtx_pcie_irq_ctx* ctx, int vector) { + if (ctx && vector >= 0 && vector < GTX_MSI_MAX) { + ctx->pending_msi.fetch_and(~(1U << vector)); + } +} + +/*============================================================================= + * Doorbell (Host -> Device) + *============================================================================*/ + +int gtx_pcie_irq_send_doorbell(gtx_pcie_irq_ctx* ctx, int source) { + if (!ctx || source < 0 || source >= GTX_DOORBELL_MAX) { + return -1; + } + + return gtx_pcie_irq_send_doorbell_mask(ctx, 1U << source); +} + +int gtx_pcie_irq_send_doorbell_mask(gtx_pcie_irq_ctx* ctx, uint32_t sources) { + if (!ctx) { + return -1; + } + + switch (ctx->mode) { + case gtx_pcie_irq_ctx::MODE_VFIO_USER: +#ifdef __linux__ + if (ctx->vfio_socket >= 0) { + /* Write to ELBI doorbell via vfio-user REGION_WRITE */ + int ret = vfio_region_write(ctx->vfio_socket, + VFU_PCI_DEV_BAR0_REGION_IDX, + GTX_BAR0_ELBI_DOORBELL, + &sources, sizeof(sources)); + if (ret < 0) { + IRQ_LOG_ERROR("Failed to send doorbell via vfio-user\n"); + return -1; + } + + IRQ_LOG("Doorbell sent via vfio-user: 0x%08x\n", sources); + return 0; + } +#endif + IRQ_LOG_ERROR("vfio socket not connected\n"); + return -1; + + case gtx_pcie_irq_ctx::MODE_MAPPED: + if (ctx->bar0_base && GTX_BAR0_ELBI_DOORBELL < ctx->bar0_size) { + volatile uint32_t* doorbell = + (volatile uint32_t*)(ctx->bar0_base + GTX_BAR0_ELBI_DOORBELL); + *doorbell = sources; + __sync_synchronize(); + IRQ_LOG("Doorbell sent (mapped): 0x%08x\n", sources); + return 0; + } + IRQ_LOG_ERROR("BAR0 not mapped or offset out of range\n"); + return -1; + + case gtx_pcie_irq_ctx::MODE_HARDWARE: + /* TODO: Use real VFIO write */ + IRQ_LOG_ERROR("Hardware mode not implemented\n"); + return -1; + } + + return -1; +} + +/*============================================================================= + * vfio-user RPC Communication + *============================================================================*/ + +int gtx_pcie_irq_region_write(gtx_pcie_irq_ctx* ctx, uint32_t region, + uint64_t offset, const void* data, size_t size) { + if (!ctx || !data || size == 0) { + return -1; + } + +#ifdef __linux__ + if (ctx->mode == gtx_pcie_irq_ctx::MODE_VFIO_USER && ctx->vfio_socket >= 0) { + return vfio_region_write(ctx->vfio_socket, region, offset, + const_cast(data), size); + } +#endif + + /* Fallback to mapped mode */ + if (ctx->mode == gtx_pcie_irq_ctx::MODE_MAPPED && ctx->bar0_base && region == 0) { + if (offset + size <= ctx->bar0_size) { + memcpy(const_cast(ctx->bar0_base) + offset, data, size); + MEMORY_BARRIER(); + return 0; + } + } + + IRQ_LOG_ERROR("region_write: unsupported mode or invalid region\n"); + return -1; +} + +int gtx_pcie_irq_region_read(gtx_pcie_irq_ctx* ctx, uint32_t region, + uint64_t offset, void* data, size_t size) { + if (!ctx || !data || size == 0) { + return -1; + } + +#ifdef __linux__ + if (ctx->mode == gtx_pcie_irq_ctx::MODE_VFIO_USER && ctx->vfio_socket >= 0) { + return vfio_region_read(ctx->vfio_socket, region, offset, data, size); + } +#endif + + /* Fallback to mapped mode */ + if (ctx->mode == gtx_pcie_irq_ctx::MODE_MAPPED && ctx->bar0_base && region == 0) { + if (offset + size <= ctx->bar0_size) { + memcpy(data, const_cast(ctx->bar0_base) + offset, size); + return 0; + } + } + + IRQ_LOG_ERROR("region_read: unsupported mode or invalid region\n"); + return -1; +} + +int gtx_pcie_irq_get_socket(gtx_pcie_irq_ctx* ctx) { + if (!ctx) { + return -1; + } + return ctx->vfio_socket; +} + +/*============================================================================= + * Event Loop Integration + *============================================================================*/ + +int gtx_pcie_irq_get_eventfd(gtx_pcie_irq_ctx* ctx, int vector) { + if (!ctx || vector < 0 || vector >= GTX_MSI_MAX) { + return -1; + } + return ctx->msi_eventfds[vector]; +} + +int gtx_pcie_irq_process(gtx_pcie_irq_ctx* ctx) { + if (!ctx) { + return 0; + } + + int count = 0; + uint32_t pending = ctx->pending_msi.exchange(0); + + if (pending) { + std::lock_guard lock(ctx->handler_mutex); + for (int i = 0; i < GTX_MSI_MAX && pending; i++) { + if (pending & (1U << i)) { + if (ctx->msi_handlers[i].handler) { + ctx->msi_handlers[i].handler(i, ctx->msi_handlers[i].user_data); + count++; + } + pending &= ~(1U << i); + } + } + } + + return count; +} + +/*============================================================================= + * Debug + *============================================================================*/ + +void gtx_pcie_irq_print_status(gtx_pcie_irq_ctx* ctx) { + if (!ctx) { + fprintf(stderr, "GTX IRQ: context is NULL\n"); + return; + } + + const char* mode_str = "unknown"; + switch (ctx->mode) { + case gtx_pcie_irq_ctx::MODE_VFIO_USER: mode_str = "vfio-user"; break; + case gtx_pcie_irq_ctx::MODE_HARDWARE: mode_str = "hardware"; break; + case gtx_pcie_irq_ctx::MODE_MAPPED: mode_str = "mapped"; break; + } + + fprintf(stderr, "GTX IRQ Status:\n"); + fprintf(stderr, " Mode: %s\n", mode_str); + fprintf(stderr, " BAR0: %p (size=%zu)\n", ctx->bar0_base, ctx->bar0_size); + fprintf(stderr, " vfio socket: %d\n", ctx->vfio_socket); + fprintf(stderr, " Pending MSI: 0x%08x\n", ctx->pending_msi.load()); + fprintf(stderr, " MSI thread: %s\n", ctx->msi_thread_running ? "running" : "stopped"); + + fprintf(stderr, " Registered handlers:\n"); + for (int i = 0; i < GTX_MSI_MAX; i++) { + if (ctx->msi_handlers[i].handler) { + fprintf(stderr, " Vector %d: handler=%p\n", i, (void*)ctx->msi_handlers[i].handler); + } + } +} + +/* Enable debug logging */ +void gtx_pcie_irq_set_debug(bool enable) { + g_irq_debug = enable; +} + +} /* extern "C" */ diff --git a/depend/ggml-gtx/src/pcie_irq_host.h b/depend/ggml-gtx/src/pcie_irq_host.h new file mode 100644 index 0000000..f390f85 --- /dev/null +++ b/depend/ggml-gtx/src/pcie_irq_host.h @@ -0,0 +1,317 @@ +/** + * @file pcie_irq_host.h + * @brief GTX PCIe Host-side Interrupt Handling + * Supports both vfio-user (simulator) and real hardware + * + * SPDX-License-Identifier: Apache-2.0 + * + * ## Interrupt Architecture + * + * ### Host -> Device (ELBI Doorbell) + * - Write to BAR0 + 0x0E40 sets doorbell bits + * - Device RISC-V receives external interrupt + * - Device reads status, handles, clears + * + * ### Device -> Host (MSI/MSI-X) + * - Device writes to DBI MSI_GEN register + * - PCIe controller generates MSI TLP + * - Host receives via eventfd (vfio-user) or IRQ (hardware) + * + * ## Usage (vfio-user mode) + * + * ```cpp + * #include "pcie_irq_host.h" + * + * gtx_pcie_irq_ctx ctx; + * gtx_pcie_irq_init(&ctx, "$HOME/.gtx-vfio.sock"); + * + * // Register MSI handler + * gtx_pcie_irq_register_msi(&ctx, 0, on_infer_done, user_data); + * + * // Send doorbell to device + * gtx_pcie_irq_send_doorbell(&ctx, GTX_DOORBELL_CMD_READY); + * + * // Wait for MSI (blocking) + * gtx_pcie_irq_wait_msi(&ctx, 0, 5000); // 5 second timeout + * + * gtx_pcie_irq_cleanup(&ctx); + * ``` + */ + +#ifndef GTX_PCIE_IRQ_HOST_H +#define GTX_PCIE_IRQ_HOST_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/*============================================================================= + * Constants + *============================================================================*/ + +/* BAR0 offsets */ +#define GTX_BAR0_ELBI_DOORBELL 0x0E40 +#define GTX_BAR0_ELBI_MASK 0x0E44 +#define GTX_BAR0_ELBI_CLEAR 0x0E48 + +/* Doorbell sources (Host -> Device) */ +#define GTX_DOORBELL_CMD_READY 0 +#define GTX_DOORBELL_DMA_DONE 1 +#define GTX_DOORBELL_CONFIG_UPDATE 2 +#define GTX_DOORBELL_ABORT 3 +#define GTX_DOORBELL_SYNC 4 +#define GTX_DOORBELL_MSG_AVAILABLE 5 +#define GTX_DOORBELL_MAX 32 + +/* MSI vectors (Device -> Host) - matches GGML_GTX_PCIE_MSI_* in pcie-elbi-regs.h */ +#define GTX_MSI_CMD_ACK 0 +#define GTX_MSI_INFER_DONE 1 +#define GTX_MSI_ERROR 2 +#define GTX_MSI_DMA_COMPLETE 3 +#define GTX_MSI_MSG_SENT 4 +#define GTX_MSI_STATUS_CHANGE 5 +#define GTX_MSI_HEARTBEAT 6 +#define GTX_MSI_ABORT_ACK 7 +#define GTX_MSI_MAX 32 + +/*============================================================================= + * Types + *============================================================================*/ + +/* MSI handler callback */ +typedef void (*gtx_msi_handler_t)(int vector, void* user_data); + +/* IRQ context (opaque) */ +typedef struct gtx_pcie_irq_ctx gtx_pcie_irq_ctx; + +/*============================================================================= + * Initialization + *============================================================================*/ + +/** + * @brief Create IRQ context for vfio-user mode + * @param socket_path vfio-user socket path (e.g., "$HOME/.gtx-vfio.sock") + * @return IRQ context, or NULL on error + * + * Connects to the GTX_ISS simulator via vfio-user socket. + */ +gtx_pcie_irq_ctx* gtx_pcie_irq_create_vfio(const char* socket_path); + +/** + * @brief Create IRQ context for real hardware + * @param device_path Device path (e.g., "/dev/vfio/42") + * @param group_id VFIO group ID + * @return IRQ context, or NULL on error + */ +gtx_pcie_irq_ctx* gtx_pcie_irq_create_hw(const char* device_path, int group_id); + +/** + * @brief Create IRQ context with existing BAR mapping + * @param bar0_base BAR0 mapped address + * @param bar0_size BAR0 size + * @return IRQ context, or NULL on error + */ +gtx_pcie_irq_ctx* gtx_pcie_irq_create_mapped(volatile void* bar0_base, size_t bar0_size); + +/** + * @brief Destroy IRQ context + * @param ctx IRQ context + */ +void gtx_pcie_irq_destroy(gtx_pcie_irq_ctx* ctx); + +/*============================================================================= + * MSI Handling (Device -> Host) + *============================================================================*/ + +/** + * @brief Register MSI handler + * @param ctx IRQ context + * @param vector MSI vector (0-31) + * @param handler Callback function + * @param user_data User data passed to callback + * @return 0 on success, -1 on error + */ +int gtx_pcie_irq_register_msi(gtx_pcie_irq_ctx* ctx, int vector, + gtx_msi_handler_t handler, void* user_data); + +/** + * @brief Unregister MSI handler + * @param ctx IRQ context + * @param vector MSI vector + */ +void gtx_pcie_irq_unregister_msi(gtx_pcie_irq_ctx* ctx, int vector); + +/** + * @brief Wait for MSI (blocking) + * @param ctx IRQ context + * @param vector MSI vector to wait for (-1 for any) + * @param timeout_ms Timeout in milliseconds (-1 for infinite) + * @return Vector number received, or -1 on timeout/error + */ +int gtx_pcie_irq_wait_msi(gtx_pcie_irq_ctx* ctx, int vector, int timeout_ms); + +/** + * @brief Poll for MSI (non-blocking) + * @param ctx IRQ context + * @return Vector number if MSI pending, or -1 if none + */ +int gtx_pcie_irq_poll_msi(gtx_pcie_irq_ctx* ctx); + +/** + * @brief Get pending MSI mask + * @param ctx IRQ context + * @return Bitmask of pending MSI vectors + */ +uint32_t gtx_pcie_irq_get_pending_msi(gtx_pcie_irq_ctx* ctx); + +/** + * @brief Clear pending MSI + * @param ctx IRQ context + * @param vector MSI vector to clear + */ +void gtx_pcie_irq_clear_msi(gtx_pcie_irq_ctx* ctx, int vector); + +/*============================================================================= + * Doorbell (Host -> Device) + *============================================================================*/ + +/** + * @brief Send doorbell to device + * @param ctx IRQ context + * @param source Doorbell source (0-31) + * @return 0 on success, -1 on error + * + * Writes (1 << source) to BAR0 + 0x0E40. + */ +int gtx_pcie_irq_send_doorbell(gtx_pcie_irq_ctx* ctx, int source); + +/** + * @brief Send multiple doorbells + * @param ctx IRQ context + * @param sources Bitmask of sources + * @return 0 on success, -1 on error + */ +int gtx_pcie_irq_send_doorbell_mask(gtx_pcie_irq_ctx* ctx, uint32_t sources); + +/*============================================================================= + * Convenience Functions + *============================================================================*/ + +/** + * @brief Notify device: command ready + */ +static inline int gtx_notify_cmd_ready(gtx_pcie_irq_ctx* ctx) { + return gtx_pcie_irq_send_doorbell(ctx, GTX_DOORBELL_CMD_READY); +} + +/** + * @brief Notify device: DMA done + */ +static inline int gtx_notify_dma_done(gtx_pcie_irq_ctx* ctx) { + return gtx_pcie_irq_send_doorbell(ctx, GTX_DOORBELL_DMA_DONE); +} + +/** + * @brief Notify device: abort request + */ +static inline int gtx_notify_abort(gtx_pcie_irq_ctx* ctx) { + return gtx_pcie_irq_send_doorbell(ctx, GTX_DOORBELL_ABORT); +} + +/** + * @brief Wait for inference done MSI + */ +static inline int gtx_wait_infer_done(gtx_pcie_irq_ctx* ctx, int timeout_ms) { + return gtx_pcie_irq_wait_msi(ctx, GTX_MSI_INFER_DONE, timeout_ms); +} + +/** + * @brief Wait for command ACK MSI + */ +static inline int gtx_wait_cmd_ack(gtx_pcie_irq_ctx* ctx, int timeout_ms) { + return gtx_pcie_irq_wait_msi(ctx, GTX_MSI_CMD_ACK, timeout_ms); +} + +/*============================================================================= + * vfio-user RPC Communication + *============================================================================*/ + +/** + * @brief Write data to device BAR region via vfio-user + * @param ctx IRQ context + * @param region BAR region index (0 for BAR0, 2 for BAR2) + * @param offset Offset within BAR + * @param data Data to write + * @param size Data size in bytes + * @return 0 on success, -1 on error + */ +int gtx_pcie_irq_region_write(gtx_pcie_irq_ctx* ctx, uint32_t region, + uint64_t offset, const void* data, size_t size); + +/** + * @brief Read data from device BAR region via vfio-user + * @param ctx IRQ context + * @param region BAR region index (0 for BAR0, 2 for BAR2) + * @param offset Offset within BAR + * @param data Buffer to receive data + * @param size Data size in bytes + * @return 0 on success, -1 on error + */ +int gtx_pcie_irq_region_read(gtx_pcie_irq_ctx* ctx, uint32_t region, + uint64_t offset, void* data, size_t size); + +/** + * @brief Get vfio-user socket fd (for advanced use) + * @param ctx IRQ context + * @return Socket file descriptor, or -1 if not in vfio-user mode + */ +int gtx_pcie_irq_get_socket(gtx_pcie_irq_ctx* ctx); + +/*============================================================================= + * Event Loop Integration + *============================================================================*/ + +/** + * @brief Get eventfd for MSI notifications + * @param ctx IRQ context + * @param vector MSI vector + * @return eventfd file descriptor, or -1 if not supported + * + * Allows integration with epoll/select event loops. + */ +int gtx_pcie_irq_get_eventfd(gtx_pcie_irq_ctx* ctx, int vector); + +/** + * @brief Process pending interrupts + * @param ctx IRQ context + * @return Number of interrupts processed + * + * Calls registered handlers for pending MSIs. + */ +int gtx_pcie_irq_process(gtx_pcie_irq_ctx* ctx); + +/*============================================================================= + * Debug + *============================================================================*/ + +/** + * @brief Print IRQ status + * @param ctx IRQ context + */ +void gtx_pcie_irq_print_status(gtx_pcie_irq_ctx* ctx); + +/** + * @brief Enable/disable debug logging for IRQ subsystem + * @param enable true to enable debug output, false to disable + */ +void gtx_pcie_irq_set_debug(bool enable); + +#ifdef __cplusplus +} +#endif + +#endif /* GTX_PCIE_IRQ_HOST_H */ diff --git a/depend/ggml-gtx/src/sc_print.c b/depend/ggml-gtx/src/sc_print.c new file mode 100644 index 0000000..4dbf301 --- /dev/null +++ b/depend/ggml-gtx/src/sc_print.c @@ -0,0 +1,653 @@ +/// Copyright by Syntacore LLC © 2016, 2017. See LICENSE for details +/// @file +/// +/// Printf implementation for baremetal RISC-V +/// Based on Marco Paland's printf implementation (MIT License) + +#include +#include +#include +#include "ggml-gtx/sc_print.h" + +//============================================================================== +// Configuration +//============================================================================== + +// Output port for simulator/hardware +#define SC_SIM_OUTPORT (0x000FFFE0) + +// Buffer sizes +#define PRINTF_NTOA_BUFFER_SIZE 32U +#define PRINTF_FTOA_BUFFER_SIZE 32U + +// Enable float support +#define PRINTF_SUPPORT_FLOAT + +// Enable long long support +#define PRINTF_SUPPORT_LONG_LONG + +//============================================================================== +// Internal flags +//============================================================================== + +#define FLAGS_ZEROPAD (1U << 0U) +#define FLAGS_LEFT (1U << 1U) +#define FLAGS_PLUS (1U << 2U) +#define FLAGS_SPACE (1U << 3U) +#define FLAGS_HASH (1U << 4U) +#define FLAGS_UPPERCASE (1U << 5U) +#define FLAGS_CHAR (1U << 6U) +#define FLAGS_SHORT (1U << 7U) +#define FLAGS_LONG (1U << 8U) +#define FLAGS_LONG_LONG (1U << 9U) +#define FLAGS_PRECISION (1U << 10U) +#define FLAGS_WIDTH (1U << 11U) + +//============================================================================== +// Low-level output +//============================================================================== + +// + +static void sc_puts(const char *str, size_t len) { + volatile char *out_ptr = (volatile char *)SC_SIM_OUTPORT; + for (size_t i = 0; i < len; i++) { + *out_ptr = str[i]; + } +} + +#undef putchar +int putchar(int ch) { + static char buf[64] __attribute__((aligned(64))); + static int buflen = 0; + + buf[buflen++] = (char)ch; + + if (ch == '\n' || buflen == (int)sizeof(buf)) { + sc_puts(buf, buflen); + buflen = 0; + } + + return 0; +} + +// Output function type +typedef void (*out_fct_type)(char character, void *buffer, size_t idx, size_t maxlen); + +// Internal character output +static void _out_char(char character, void *buffer, size_t idx, size_t maxlen) { + (void)buffer; (void)idx; (void)maxlen; + if (character) { + putchar(character); + } +} + +// Internal buffer output +static void _out_buffer(char character, void *buffer, size_t idx, size_t maxlen) { + if (idx < maxlen) { + ((char *)buffer)[idx] = character; + } +} + +// Internal null output +static void _out_null(char character, void *buffer, size_t idx, size_t maxlen) { + (void)character; (void)buffer; (void)idx; (void)maxlen; +} + +//============================================================================== +// Helper functions +//============================================================================== + +static inline unsigned int _strlen(const char *str) { + const char *s; + for (s = str; *s; ++s); + return (unsigned int)(s - str); +} + +static inline int _is_digit(char ch) { + return (ch >= '0') && (ch <= '9'); +} + +static inline unsigned int _atoi(const char **str) { + unsigned int i = 0U; + while (_is_digit(**str)) { + i = i * 10U + (unsigned int)(*((*str)++) - '0'); + } + return i; +} + +//============================================================================== +// Number formatting +//============================================================================== + +static size_t _ntoa_format(out_fct_type out, char *buffer, size_t idx, size_t maxlen, + char *buf, size_t len, int negative, unsigned int base, + unsigned int prec, unsigned int width, unsigned int flags) { + const size_t start_idx = idx; + + // Pad leading zeros + while (!(flags & FLAGS_LEFT) && (len < prec) && (len < PRINTF_NTOA_BUFFER_SIZE)) { + buf[len++] = '0'; + } + while (!(flags & FLAGS_LEFT) && (flags & FLAGS_ZEROPAD) && (len < width) && (len < PRINTF_NTOA_BUFFER_SIZE)) { + buf[len++] = '0'; + } + + // Handle hash + if (flags & FLAGS_HASH) { + if (((len == prec) || (len == width)) && (len > 0U)) { + len--; + if ((base == 16U) && (len > 0U)) { + len--; + } + } + if ((base == 16U) && !(flags & FLAGS_UPPERCASE) && (len < PRINTF_NTOA_BUFFER_SIZE)) { + buf[len++] = 'x'; + } + if ((base == 16U) && (flags & FLAGS_UPPERCASE) && (len < PRINTF_NTOA_BUFFER_SIZE)) { + buf[len++] = 'X'; + } + if (len < PRINTF_NTOA_BUFFER_SIZE) { + buf[len++] = '0'; + } + } + + // Handle sign + if ((len == width) && (negative || (flags & FLAGS_PLUS) || (flags & FLAGS_SPACE))) { + len--; + } + if (len < PRINTF_NTOA_BUFFER_SIZE) { + if (negative) { + buf[len++] = '-'; + } else if (flags & FLAGS_PLUS) { + buf[len++] = '+'; + } else if (flags & FLAGS_SPACE) { + buf[len++] = ' '; + } + } + + // Pad spaces up to given width + if (!(flags & FLAGS_LEFT) && !(flags & FLAGS_ZEROPAD)) { + for (size_t i = len; i < width; i++) { + out(' ', buffer, idx++, maxlen); + } + } + + // Reverse string + for (size_t i = 0U; i < len; i++) { + out(buf[len - i - 1U], buffer, idx++, maxlen); + } + + // Append pad spaces up to given width + if (flags & FLAGS_LEFT) { + while (idx - start_idx < width) { + out(' ', buffer, idx++, maxlen); + } + } + + return idx; +} + +static size_t _ntoa_long(out_fct_type out, char *buffer, size_t idx, size_t maxlen, + unsigned long value, int negative, unsigned long base, + unsigned int prec, unsigned int width, unsigned int flags) { + char buf[PRINTF_NTOA_BUFFER_SIZE]; + size_t len = 0U; + + // Write if precision != 0 and value is != 0 + if (!(flags & FLAGS_PRECISION) || value) { + do { + const char digit = (char)(value % base); + buf[len++] = digit < 10 ? '0' + digit : (flags & FLAGS_UPPERCASE ? 'A' : 'a') + digit - 10; + value /= base; + } while (value && (len < PRINTF_NTOA_BUFFER_SIZE)); + } + + return _ntoa_format(out, buffer, idx, maxlen, buf, len, negative, (unsigned int)base, prec, width, flags); +} + +#if defined(PRINTF_SUPPORT_LONG_LONG) +static size_t _ntoa_long_long(out_fct_type out, char *buffer, size_t idx, size_t maxlen, + unsigned long long value, int negative, unsigned long long base, + unsigned int prec, unsigned int width, unsigned int flags) { + char buf[PRINTF_NTOA_BUFFER_SIZE]; + size_t len = 0U; + + // Write if precision != 0 and value is != 0 + if (!(flags & FLAGS_PRECISION) || value) { + do { + const char digit = (char)(value % base); + buf[len++] = digit < 10 ? '0' + digit : (flags & FLAGS_UPPERCASE ? 'A' : 'a') + digit - 10; + value /= base; + } while (value && (len < PRINTF_NTOA_BUFFER_SIZE)); + } + + return _ntoa_format(out, buffer, idx, maxlen, buf, len, negative, (unsigned int)base, prec, width, flags); +} +#endif + +#if defined(PRINTF_SUPPORT_FLOAT) +static size_t _ftoa(out_fct_type out, char *buffer, size_t idx, size_t maxlen, + double value, unsigned int prec, unsigned int width, unsigned int flags) { + char buf[PRINTF_FTOA_BUFFER_SIZE]; + size_t len = 0U; + double diff = 0.0; + + // Powers of 10 + static const double pow10[] = {1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000}; + + // Test for negative + int negative = 0; + if (value < 0) { + negative = 1; + value = 0 - value; + } + + // Set default precision to 6, if not set explicitly + if (!(flags & FLAGS_PRECISION)) { + prec = 6U; + } + // Limit precision to 9 + while ((len < PRINTF_FTOA_BUFFER_SIZE) && (prec > 9U)) { + buf[len++] = '0'; + prec--; + } + + int whole = (int)value; + double tmp = (value - whole) * pow10[prec]; + unsigned long frac = (unsigned long)tmp; + diff = tmp - frac; + + if (diff > 0.5) { + ++frac; + if (frac >= (unsigned long)pow10[prec]) { + frac = 0; + ++whole; + } + } else if ((diff == 0.5) && ((frac == 0U) || (frac & 1U))) { + ++frac; + } + + // For very large numbers, just return 0 + if (value > (double)0x7FFFFFFF) { + return 0U; + } + + if (prec == 0U) { + diff = value - (double)whole; + if (diff > 0.5) { + ++whole; + } else if ((diff == 0.5) && (whole & 1)) { + ++whole; + } + } else { + unsigned int count = prec; + while (len < PRINTF_FTOA_BUFFER_SIZE) { + --count; + buf[len++] = (char)(48U + (frac % 10U)); + if (!(frac /= 10U)) { + break; + } + } + while ((len < PRINTF_FTOA_BUFFER_SIZE) && (count-- > 0U)) { + buf[len++] = '0'; + } + if (len < PRINTF_FTOA_BUFFER_SIZE) { + buf[len++] = '.'; + } + } + + // Do whole part + while (len < PRINTF_FTOA_BUFFER_SIZE) { + buf[len++] = (char)(48 + (whole % 10)); + if (!(whole /= 10)) { + break; + } + } + + // Pad leading zeros + while (!(flags & FLAGS_LEFT) && (flags & FLAGS_ZEROPAD) && (len < width) && (len < PRINTF_FTOA_BUFFER_SIZE)) { + buf[len++] = '0'; + } + + // Handle sign + if ((len == width) && (negative || (flags & FLAGS_PLUS) || (flags & FLAGS_SPACE))) { + len--; + } + if (len < PRINTF_FTOA_BUFFER_SIZE) { + if (negative) { + buf[len++] = '-'; + } else if (flags & FLAGS_PLUS) { + buf[len++] = '+'; + } else if (flags & FLAGS_SPACE) { + buf[len++] = ' '; + } + } + + // Pad spaces up to given width + if (!(flags & FLAGS_LEFT) && !(flags & FLAGS_ZEROPAD)) { + for (size_t i = len; i < width; i++) { + out(' ', buffer, idx++, maxlen); + } + } + + // Reverse string + for (size_t i = 0U; i < len; i++) { + out(buf[len - i - 1U], buffer, idx++, maxlen); + } + + // Append pad spaces up to given width + if (flags & FLAGS_LEFT) { + while (idx < width) { + out(' ', buffer, idx++, maxlen); + } + } + + return idx; +} +#endif + +//============================================================================== +// Internal vsnprintf +//============================================================================== + +static int _vsnprintf(out_fct_type out, char *buffer, const size_t maxlen, const char *format, va_list va) { + unsigned int flags, width, precision, n; + size_t idx = 0U; + + if (!buffer) { + out = _out_null; + } + + while (*format) { + // Format specifier? %[flags][width][.precision][length] + if (*format != '%') { + out(*format, buffer, idx++, maxlen); + format++; + continue; + } else { + format++; + } + + // Evaluate flags + flags = 0U; + do { + switch (*format) { + case '0': flags |= FLAGS_ZEROPAD; format++; n = 1U; break; + case '-': flags |= FLAGS_LEFT; format++; n = 1U; break; + case '+': flags |= FLAGS_PLUS; format++; n = 1U; break; + case ' ': flags |= FLAGS_SPACE; format++; n = 1U; break; + case '#': flags |= FLAGS_HASH; format++; n = 1U; break; + default: n = 0U; break; + } + } while (n); + + // Evaluate width field + width = 0U; + if (_is_digit(*format)) { + width = _atoi(&format); + } else if (*format == '*') { + const int w = va_arg(va, int); + if (w < 0) { + flags |= FLAGS_LEFT; + width = (unsigned int)-w; + } else { + width = (unsigned int)w; + } + format++; + } + + // Evaluate precision field + precision = 0U; + if (*format == '.') { + flags |= FLAGS_PRECISION; + format++; + if (_is_digit(*format)) { + precision = _atoi(&format); + } else if (*format == '*') { + const int prec = va_arg(va, int); + precision = prec > 0 ? (unsigned int)prec : 0U; + format++; + } + } + + // Evaluate length field + switch (*format) { + case 'l': + flags |= FLAGS_LONG; + format++; + if (*format == 'l') { + flags |= FLAGS_LONG_LONG; + format++; + } + break; + case 'h': + flags |= FLAGS_SHORT; + format++; + if (*format == 'h') { + flags |= FLAGS_CHAR; + format++; + } + break; + case 'z': + flags |= (sizeof(size_t) == sizeof(long) ? FLAGS_LONG : FLAGS_LONG_LONG); + format++; + break; + case 'j': + flags |= (sizeof(intmax_t) == sizeof(long) ? FLAGS_LONG : FLAGS_LONG_LONG); + format++; + break; + case 't': + flags |= (sizeof(ptrdiff_t) == sizeof(long) ? FLAGS_LONG : FLAGS_LONG_LONG); + format++; + break; + default: + break; + } + + // Evaluate specifier + switch (*format) { + case 'd': + case 'i': + case 'u': + case 'x': + case 'X': + case 'o': + case 'b': { + // Set the base + unsigned int base; + if (*format == 'x' || *format == 'X') { + base = 16U; + } else if (*format == 'o') { + base = 8U; + } else if (*format == 'b') { + base = 2U; + flags &= ~FLAGS_HASH; + } else { + base = 10U; + flags &= ~FLAGS_HASH; + } + // Uppercase + if (*format == 'X') { + flags |= FLAGS_UPPERCASE; + } + + // No plus or space flag for u, x, X, o, b + if ((*format != 'i') && (*format != 'd')) { + flags &= ~(FLAGS_PLUS | FLAGS_SPACE); + } + + // Convert the integer + if ((*format == 'i') || (*format == 'd')) { + // Signed + if (flags & FLAGS_LONG_LONG) { +#if defined(PRINTF_SUPPORT_LONG_LONG) + const long long value = va_arg(va, long long); + idx = _ntoa_long_long(out, buffer, idx, maxlen, + (unsigned long long)(value > 0 ? value : 0 - value), + value < 0, base, precision, width, flags); +#endif + } else if (flags & FLAGS_LONG) { + const long value = va_arg(va, long); + idx = _ntoa_long(out, buffer, idx, maxlen, + (unsigned long)(value > 0 ? value : 0 - value), + value < 0, base, precision, width, flags); + } else { + const int value = (flags & FLAGS_CHAR) ? (char)va_arg(va, int) : + (flags & FLAGS_SHORT) ? (short int)va_arg(va, int) : + va_arg(va, int); + idx = _ntoa_long(out, buffer, idx, maxlen, + (unsigned int)(value > 0 ? value : 0 - value), + value < 0, base, precision, width, flags); + } + } else { + // Unsigned + if (flags & FLAGS_LONG_LONG) { +#if defined(PRINTF_SUPPORT_LONG_LONG) + idx = _ntoa_long_long(out, buffer, idx, maxlen, + va_arg(va, unsigned long long), 0, base, + precision, width, flags); +#endif + } else if (flags & FLAGS_LONG) { + idx = _ntoa_long(out, buffer, idx, maxlen, + va_arg(va, unsigned long), 0, base, + precision, width, flags); + } else { + const unsigned int value = (flags & FLAGS_CHAR) ? (unsigned char)va_arg(va, unsigned int) : + (flags & FLAGS_SHORT) ? (unsigned short int)va_arg(va, unsigned int) : + va_arg(va, unsigned int); + idx = _ntoa_long(out, buffer, idx, maxlen, value, 0, base, precision, width, flags); + } + } + format++; + break; + } + +#if defined(PRINTF_SUPPORT_FLOAT) + case 'f': + case 'F': + idx = _ftoa(out, buffer, idx, maxlen, va_arg(va, double), precision, width, flags); + format++; + break; +#endif + + case 'c': { + unsigned int l = 1U; + // Pre padding + if (!(flags & FLAGS_LEFT)) { + while (l++ < width) { + out(' ', buffer, idx++, maxlen); + } + } + // Char output + out((char)va_arg(va, int), buffer, idx++, maxlen); + // Post padding + if (flags & FLAGS_LEFT) { + while (l++ < width) { + out(' ', buffer, idx++, maxlen); + } + } + format++; + break; + } + + case 's': { + const char *p = va_arg(va, char *); + if (p == NULL) { + p = "(null)"; + } + unsigned int l = _strlen(p); + // Pre padding + if (flags & FLAGS_PRECISION) { + l = (l < precision ? l : precision); + } + if (!(flags & FLAGS_LEFT)) { + while (l++ < width) { + out(' ', buffer, idx++, maxlen); + } + } + // String output + while ((*p != 0) && (!(flags & FLAGS_PRECISION) || precision--)) { + out(*(p++), buffer, idx++, maxlen); + } + // Post padding + if (flags & FLAGS_LEFT) { + while (l++ < width) { + out(' ', buffer, idx++, maxlen); + } + } + format++; + break; + } + + case 'p': { + width = sizeof(void *) * 2U; + flags |= FLAGS_ZEROPAD | FLAGS_UPPERCASE; +#if defined(PRINTF_SUPPORT_LONG_LONG) + const int is_ll = sizeof(uintptr_t) == sizeof(long long); + if (is_ll) { + idx = _ntoa_long_long(out, buffer, idx, maxlen, + (uintptr_t)va_arg(va, void *), 0, 16U, + precision, width, flags); + } else { +#endif + idx = _ntoa_long(out, buffer, idx, maxlen, + (unsigned long)((uintptr_t)va_arg(va, void *)), 0, 16U, + precision, width, flags); +#if defined(PRINTF_SUPPORT_LONG_LONG) + } +#endif + format++; + break; + } + + case '%': + out('%', buffer, idx++, maxlen); + format++; + break; + + default: + out(*format, buffer, idx++, maxlen); + format++; + break; + } + } + + // Termination + out((char)0, buffer, idx < maxlen ? idx : maxlen - 1U, maxlen); + + return (int)idx; +} + +//============================================================================== +// Public API +//============================================================================== + +int sc_printf(const char *format, ...) { + va_list va; + va_start(va, format); + char buffer[1]; + const int ret = _vsnprintf(_out_char, buffer, (size_t)-1, format, va); + va_end(va); + return ret; +} + +int sc_sprintf(char *buffer, const char *format, ...) { + va_list va; + va_start(va, format); + const int ret = _vsnprintf(_out_buffer, buffer, (size_t)-1, format, va); + va_end(va); + return ret; +} + +int sc_snprintf(char *buffer, size_t count, const char *format, ...) { + va_list va; + va_start(va, format); + const int ret = _vsnprintf(_out_buffer, buffer, count, format, va); + va_end(va); + return ret; +} + +int sc_vsnprintf(char *buffer, size_t count, const char *format, va_list va) { + return _vsnprintf(_out_buffer, buffer, count, format, va); +} diff --git a/depend/llama b/depend/llama index c66ee8e..5fde0fa 160000 --- a/depend/llama +++ b/depend/llama @@ -1 +1 @@ -Subproject commit c66ee8e2c118b9f04a3a500a672f906833cb8875 +Subproject commit 5fde0fae463cb6def4747d05dcadb21495b4ce93 diff --git a/docs/mmdet-detectors.md b/docs/mmdet-detectors.md index 5710eb0..57af7d5 100644 --- a/docs/mmdet-detectors.md +++ b/docs/mmdet-detectors.md @@ -14,7 +14,7 @@ that way serves every family that shares its structure. ``` image ──▶ backbone + neck (compiled ggml graph) ──▶ FPN levels │ - head │ tools/detect/head.cpp + head │ src/visp/mmdet_head.cpp ▼ raw cls / bbox │ @@ -80,17 +80,19 @@ python tools/frontend/mmdet/mmdet_to_pt.py \ Outputs: -`backbone.pt` -: The backbone and neck as a traceable module. The head is kept as an attribute so that its - weights are preserved in the `state_dict`, but it does not participate in the forward pass. +**`backbone.pt`** -`backbone.postproc.h` -: Everything the C++ side needs to reconstruct the head and decode its output: anchor - generator settings, bbox coder statistics, head convolution layout, and the pre-processing - normalisation taken from the config's `data_preprocessor` — emitted as a generated - `mmdet_params()` function. These values are constants once the architecture is chosen, so - they are compiled into the runner rather than read at run time. - See [Configuration reference](#configuration-reference). + The backbone and neck as a traceable module. The head is kept as an attribute so that its + weights are preserved in the `state_dict`, but it does not participate in the forward pass. + +**`backbone.postproc.h`** + + Everything the C++ side needs to reconstruct the head and decode its output: anchor + generator settings, bbox coder statistics, head convolution layout, and the pre-processing + normalisation taken from the config's `data_preprocessor` — emitted as a generated + `mmdet_params()` function. These values are constants once the architecture is chosen, so + they are compiled into the runner rather than read at run time. + See [Configuration reference](#configuration-reference). Those two are what you read. Two more land beside them, and they are not optional: saving a module pickles its classes by module name, so whatever opens the `.pt` has to be able to @@ -151,24 +153,41 @@ Graph contract If a level cannot be found the runner stops and reports the missing `out_`, which means the generated graph did not name its outputs as expected. -### Step 3 — Build the runner +### Step 3 — Register and build + +```sh +python tools/install_arch.py output/MMDetBackbone --mmdet --postproc backbone.postproc.h +cmake --build build -j8 +``` + +Registration copies the generated graph and the parameters header into `src/visp/arch/` and +writes a registration unit that carries the head configuration, so `vision-cli` finds the +detector by name. The parameters header is renamed per architecture, which is what lets more +than one detector live in the same executable. + +### Step 3b — The verification runner + +`vision-cli` covers the normal path. The separate runner exists for comparison work: it accepts +an already pre-processed `.bin` tensor as input, and `MMDET_DUMP_HEAD` writes the tensors before +decoding. ```sh bash tools/build/build_mmdet_cpp.sh output/MMDetBackbone backbone.postproc.h ``` -The script compiles three translation units together, with the generated parameters -included as a header, and links them against `libvisioncpp`: +The script compiles the runner and the generated graph, with the generated parameters included +as a header, and links them against `libvisioncpp`, which now carries the head assembler and the +decode sequence: -- `tools/verify/backbone/run_mmdet.cpp` — the runner, -- `tools/detect/head.cpp` — the head component, +- `tools/verify/mmdet/backbone/run_mmdet.cpp` — the verification runner, - `output/MMDetBackbone/MMDetBackbone.cpp` — the generated graph, - `backbone.postproc.h` — the generated parameters. -`build_mmdet_cpp.sh [params.h] [arch_name]` -: `gen_dir` is the directory holding the generated `.cpp`, `.h` and `.gguf`. The parameters - header is found in `gen_dir` when it is there, and named explicitly otherwise. - `arch_name` defaults to the base name of the `.cpp` found there. +**`build_mmdet_cpp.sh [params.h] [arch_name]`** + + `gen_dir` is the directory holding the generated `.cpp`, `.h` and `.gguf`. The parameters + header is found in `gen_dir` when it is there, and named explicitly otherwise. + `arch_name` defaults to the base name of the `.cpp` found there. The library is looked up in `build/`, which is where [Building](../README.md#building) puts it. If you configured elsewhere, name that directory: @@ -183,33 +202,39 @@ The result is `/run_mmdet`. ### Step 4 — Run ```sh -output/MMDetBackbone/run_mmdet \ - output/MMDetBackbone/MMDetBackbone.gguf \ - image.jpg \ - detected.png \ - 512 +build/bin/vision-cli mmdetbackbone \ + -m output/MMDetBackbone/MMDetBackbone.gguf \ + -i image.jpg \ + -o detected.png ``` +The input resolution is fixed at registration, so it is not passed on the command line. The +verification runner takes it as an argument because it also accepts raw tensors. + ``` run_mmdet [size=512] ``` -`` -: Weights produced in step 2. +**``** + + Weights produced in step 2. + +**``** -`` -: An image (`.jpg`, `.jpeg`, `.png`, `.bmp`) or a pre-processed tensor (`.bin`). - Images are resized and normalised in-process using `preprocess()`; the mean, standard - deviation and channel order are compiled in. A `.bin` file is taken as-is and must - contain `3 × size × size` `float32` values in CWHN order. + An image (`.jpg`, `.jpeg`, `.png`, `.bmp`) or a pre-processed tensor (`.bin`). + Images are resized and normalised in-process using `preprocess()`; the mean, standard + deviation and channel order are compiled in. A `.bin` file is taken as-is and must + contain `3 × size × size` `float32` values in CWHN order. -`` -: Where to write the result. The extension decides what is written: `.bin` gives raw - detections, anything else gives the input image with the boxes drawn on it. +**``** -`[size]` -: Input resolution. Must match the value passed to `--size` in step 1 and the shape the graph - was compiled for. + Where to write the result. The extension decides what is written: `.bin` gives raw + detections, anything else gives the input image with the boxes drawn on it. + +**`[size]`** + + Input resolution. Must match the value passed to `--size` in step 1 and the shape the graph + was compiled for. ### Output @@ -256,13 +281,13 @@ import numpy as np d = np.fromfile("boxes.bin", dtype="float32").reshape(-1, 6) ``` -`tools/verify/draw_boxes.py` draws such a file afterwards, with class names and scores as text. +`tools/verify/common/draw_boxes.py` draws such a file afterwards, with class names and scores as text. ## Models whose head survives tracing Everything above splits the model because an MMDetection head cannot be traced. Most other -detectors have no such problem — an ultralytics YOLO or a torchvision model traces whole, and +models have no such problem — a torchvision model, for example, traces whole, and then none of the steps above apply. A compiler emits the entire graph, `install_arch.py` drops it into `src/visp/arch/` with a registration unit beside it, and `vision-cli` dispatches on the architecture name recorded in the GGUF: @@ -274,12 +299,13 @@ started inside `vision.cpp/` finds no project there: it builds a second virtual then fails to spawn. ```sh -uv run g2c --model "ultralytics.YOLO('yolo26m.pt')" --name Yolo26m \ - --output out --input-shape 1,3,640,640 -python vision.cpp/tools/install_arch.py out --name Yolo26m --detect-yolo +uv run g2c --model "torchvision.models.resnet18(weights='DEFAULT')" --name Resnet18 \ + --output out --input-shape 1,3,224,224 +python vision.cpp/tools/install_arch.py out --name Resnet18 \ + --mean 123.675,116.28,103.53 --std 58.395,57.12,57.375 cmake --build vision.cpp/build -j4 -./vision.cpp/build/bin/vision-cli yolo26m -m out/Yolo26m.gguf \ - -i vision.cpp/tests/input/cat-and-hat.jpg -o detected.png +./vision.cpp/build/bin/vision-cli -m out/Resnet18.gguf \ + -i vision.cpp/tests/input/cat-and-hat.jpg -o out/resnet18 ``` > ⚠️ **Read the `→ gguf:` line, not the exit code.** A `g2c` run that prints `done` without a @@ -290,22 +316,16 @@ cmake --build vision.cpp/build -j4 > so a static link discards the whole translation unit. The build succeeds and the command > silently disappears from `vision-cli --help`. -`--detect-yolo` supplies what the GGUF does not carry — class count, strides, whether the head -is NMS-free — so the result comes back as boxes: `vision-cli` draws them and prints their -coordinates and scores. Registered without the flag, the same command writes the graph -outputs as raw `float32` files instead, which is what a numerical comparison needs. The `.bin` -rule described under *Output* above belongs to `run_mmdet`; a registered detector writes an -image whatever the output path is called. +`` is the architecture name that `install_arch.py` prints after registration. A model +registered this way writes its graph outputs as raw `float32` files (`..bin`), which +is what a numerical comparison needs. The `.bin` rule described under *Output* above belongs to +`run_mmdet`. An MMDetection detector registered with `--mmdet` is decoded instead: `vision-cli` +draws the boxes and prints their coordinates and scores. The full version of this route — options, failure modes, how to measure it — is `docs/vision-cpp-mmdet-guide-en.md` in the compiler checkout that carries this repository as a submodule. -Measured on `yolo26m` at 640, against ultralytics on the same pixels: the three detections -above 0.25 agree in class and score (cat 0.918, couch 0.771, tv 0.681) with box coordinates -within 0.1 px, and the pre-decode tensors are at relative L1 1.7e-03 on the boxes and -4.7e-04 on the class logits. - ## What decodes to boxes Every family that has been verified, the decoder each one uses, and the measured error are in @@ -324,7 +344,7 @@ If you need to know how far to trust the number, read the method section above i ## Detection heads Head components take FPN features and produce the raw per-level tensors that the decoders -expect. They are declared in `tools/detect/head.h`. +expect. They are declared in `src/visp/mmdet_head.h`. ### `anchor_head_forward` @@ -385,9 +405,9 @@ reg_denom`). ### Adding a head -1. Add a `_head_forward` function to `tools/detect/head.cpp` that turns FPN features - into raw per-level tensors. Use library primitives (`conv_2d`, `group_norm`, - `conv_2d_deform`); do not add framework-specific code to `src/visp`. +1. Add a `_head_forward` function to `src/visp/mmdet_head.cpp` that turns FPN features + into raw per-level tensors, and dispatch to it from `mmdet_head_forward`. Build on library + primitives (`conv_2d`, `group_norm`, `conv_2d_deform`). 2. Extract the head's structural parameters in `mmdet_wrap.postproc_cfg`; they are emitted into the generated parameters header. 3. Connect the raw output to the matching decoder in `visp/postproc.h`, or add one if the @@ -409,35 +429,40 @@ struct detection { ### Pre-processing -`std::vector preprocess(uint8_t const* img, int img_h, int img_w, int img_c, int out_size, float const mean[3], float const std[3], bool to_rgb, int* out_w = nullptr, int* out_h = nullptr)` -: Resize to `out_size × out_size` and normalise to `(v - mean) / std`, optionally swapping - channel order. Returns a CWHN `float32` tensor ready for the graph input. +**`std::vector preprocess(uint8_t const* img, int img_h, int img_w, int img_c, int out_size, float const mean[3], float const std[3], bool to_rgb, int* out_w = nullptr, int* out_h = nullptr)`** + + Resize to `out_size × out_size` and normalise to `(v - mean) / std`, optionally swapping + channel order. Returns a CWHN `float32` tensor ready for the graph input. ### Dense heads -`std::vector detect_anchor(cls_scores, bbox_preds, feat_hw, det_params const& p, score_factors = nullptr)` -: Anchor-based decoding: anchor generation, delta decoding, per-level top-k, score - thresholding and NMS. Used by RetinaNet, ATSS, PAA and other delta-coded heads. - `score_factors` is the optional centerness/IoU branch. MMDetection thresholds and takes - top-k on the class score **alone** and multiplies the factor in afterwards, so passing it - here rather than folding it into `cls_scores` keeps the surviving set the same. - -`std::vector detect_fcos(cls_scores, bbox_preds, centerness, feat_hw, fcos_params const& p)` -: Anchor-free distance decoding. `centerness` may be empty — GFL and VFNet fold quality into - the class score and have no such branch. `bbox_preds` are already pixel distances: the head - component applies the DFL integral — DFL is Distribution Focal Loss, which predicts each - box edge as a distribution over bins and recovers the distance by integrating it — plus the - stride multiply and the exponent, so this function - must not apply a stride again. `point_offset` is 0.5 for FCOS and 0 for the heads built on - an `AnchorGenerator`. - -`std::vector detect_yolox(cls, box, obj, feat_hw, yolox_params const& p)` -: Grid-based decoding with an objectness branch; score is `sigmoid(cls) * sigmoid(obj)`. - -`std::vector detect_detr(float const* cls, float const* bbox, detr_params const& p)` -: Set prediction. Takes query logits and normalised `cxcywh` boxes, applies top-k, and - performs no NMS. Set `use_sigmoid` for Deformable-DETR-style heads. +**`std::vector detect_anchor(cls_scores, bbox_preds, feat_hw, det_params const& p, score_factors = nullptr)`** + + Anchor-based decoding: anchor generation, delta decoding, per-level top-k, score + thresholding and NMS. Used by RetinaNet, ATSS, PAA and other delta-coded heads. + `score_factors` is the optional centerness/IoU branch. MMDetection thresholds and takes + top-k on the class score **alone** and multiplies the factor in afterwards, so passing it + here rather than folding it into `cls_scores` keeps the surviving set the same. + +**`std::vector detect_fcos(cls_scores, bbox_preds, centerness, feat_hw, fcos_params const& p)`** + + Anchor-free distance decoding. `centerness` may be empty — GFL and VFNet fold quality into + the class score and have no such branch. `bbox_preds` are already pixel distances: the head + component applies the DFL integral — DFL is Distribution Focal Loss, which predicts each + box edge as a distribution over bins and recovers the distance by integrating it — plus the + stride multiply and the exponent, so this function + must not apply a stride again. `point_offset` is 0.5 for FCOS and 0 for the heads built on + an `AnchorGenerator`. + +**`std::vector detect_yolox(cls, box, obj, feat_hw, yolox_params const& p)`** + + Grid-based decoding with an objectness branch; score is `sigmoid(cls) * sigmoid(obj)`. + +**`std::vector detect_detr(float const* cls, float const* bbox, detr_params const& p)`** + + Set prediction. Takes query logits and normalised `cxcywh` boxes, applies top-k, and + performs no NMS. Set `use_sigmoid` for Deformable-DETR-style heads. `det_params` carries the anchor generator (`strides`, `octave_base_scale`, `octave_scales`, `ratios`, `center_offset`), the bbox coder (`means`, `stds`), and the test-time thresholds @@ -446,32 +471,38 @@ image. ### Two-stage components -`std::vector detect_rpn(rpn_cls, rpn_bbox, feat_hw, rpn_params const& p)` -: Region proposals with their objectness scores. NMS runs **per level**, not per class — - MMDetection passes level ids to `batched_nms` — and there is no pre-NMS score threshold. - The returned `label` is the level the proposal came from. +**`std::vector detect_rpn(rpn_cls, rpn_bbox, feat_hw, rpn_params const& p)`** + + Region proposals with their objectness scores. NMS runs **per level**, not per class — + MMDetection passes level ids to `batched_nms` — and there is no pre-NMS score threshold. + The returned `label` is the level the proposal came from. + +**`std::vector rpn_proposals(rpn_cls, rpn_bbox, feat_hw, rpn_params const& p)`** + + The same computation with the scores dropped: `M × 4` boxes in image coordinates, + `M ≤ max_per_img`, which is the form the RoIAlign stage takes. -`std::vector rpn_proposals(rpn_cls, rpn_bbox, feat_hw, rpn_params const& p)` -: The same computation with the scores dropped: `M × 4` boxes in image coordinates, - `M ≤ max_per_img`, which is the form the RoIAlign stage takes. +**`std::vector roi_align(feats, feat_hw, float const* rois, int m, roi_align_params const& p)`** -`std::vector roi_align(feats, feat_hw, float const* rois, int m, roi_align_params const& p)` -: MMCV-compatible RoIAlign (`aligned = true`, adaptive `sampling_ratio`). Level assignment - follows `clamp(floor(log2(sqrt(w*h) / finest_scale + 1e-6)), 0, L-1)`. - Returns `M × C × out × out` in NCHW order. + MMCV-compatible RoIAlign (`aligned = true`, adaptive `sampling_ratio`). Level assignment + follows `clamp(floor(log2(sqrt(w*h) / finest_scale + 1e-6)), 0, L-1)`. + Returns `M × C × out × out` in NCHW order. -`std::vector detect_roi(float const* scores, float const* bbox_deltas, float const* proposals, int n, roi_params const& p)` -: Final RoI-head decoding: class-wise delta decoding and per-class NMS. `scores` are - post-softmax with background last. Set `class_agnostic` when `bbox_pred` has four columns - instead of `num_classes * 4`. +**`std::vector detect_roi(float const* scores, float const* bbox_deltas, float const* proposals, int n, roi_params const& p)`** + + Final RoI-head decoding: class-wise delta decoding and per-class NMS. `scores` are + post-softmax with background last. Set `class_agnostic` when `bbox_pred` has four columns + instead of `num_classes * 4`. ### Masks and keypoints -`std::vector paste_mask(float const* mask_logit, int mh, int mw, detection const& box, float thr = 0.5f, int* out_h = nullptr, int* out_w = nullptr)` -: Sigmoid, resize to the box, threshold. Returns a binary mask covering the box. +**`std::vector paste_mask(float const* mask_logit, int mh, int mw, detection const& box, float thr = 0.5f, int* out_h = nullptr, int* out_w = nullptr)`** + + Sigmoid, resize to the box, threshold. Returns a binary mask covering the box. + +**`std::vector decode_keypoints(float const* heatmap, int k, int hm_h, int hm_w, float stride)`** -`std::vector decode_keypoints(float const* heatmap, int k, int hm_h, int hm_w, float stride)` -: Per-keypoint argmax over a heatmap; returns `k × 3` as `(x, y, score)`. + Per-keypoint argmax over a heatmap; returns `k × 3` as `(x, y, score)`. ### Building blocks diff --git a/docs/using-the-cli.md b/docs/using-the-cli.md index a5b023f..73afd7a 100644 --- a/docs/using-the-cli.md +++ b/docs/using-the-cli.md @@ -23,36 +23,44 @@ The command selects the model, `-m` says which weights to load, `-i` and `-o` ar ## Options -`-m, --model ` -: The `.gguf` weights. Omit it and each command looks for its own default name — - `MobileSAM-F16.gguf`, `BiRefNet-lite-F16.gguf`, and so on — under `models/`, - `$VISION_MODEL_DIR`, `$XDG_DATA_HOME/visioncpp`, `~/.local/share/visioncpp` and the - install directory, in that order. +**`-m, --model `** -`-i, --input [ ...]` -: Input image. `migan` takes two — the image and the mask. + The `.gguf` weights. Omit it and each command looks for its own default name — + `MobileSAM-F16.gguf`, `BiRefNet-lite-F16.gguf`, and so on — under `models/`, + `$VISION_MODEL_DIR`, `$XDG_DATA_HOME/visioncpp`, `~/.local/share/visioncpp` and the + install directory, in that order. -`-o, --output ` -: Output file. Defaults to `output.png`. Images are always written as **PNG**, whatever - the name says — `-o out.jpg` produces a PNG file called `out.jpg`, which some viewers - refuse to open. Give it a `.png` name. +**`-i, --input [ ...]`** -`-p, --prompt [ ...]` -: Prompt for models that take one. `sam` accepts a point (`x y`) or a box - (`x1 y1 x2 y2`) in pixels, origin top-left. + Input image. `migan` takes two — the image and the mask. -`-b, --backend ` -: Which device to run on. Defaults to automatic — GPU if the build has Vulkan and a device is - available, CPU otherwise. +**`-o, --output `** -`--composite ` -: Also write the input image combined with the resulting mask, instead of the mask alone. + Output file. Defaults to `output.png`. Images are always written as **PNG**, whatever + the name says — `-o out.jpg` produces a PNG file called `out.jpg`, which some viewers + refuse to open. Give it a `.png` name. -`--tile ` -: Split large inputs into tiles of this size. Used by `esrgan` to keep memory bounded. +**`-p, --prompt [ ...]`** -`-h, --help` -: Print the command list and exit. + Prompt for models that take one. `sam` accepts a point (`x y`) or a box + (`x1 y1 x2 y2`) in pixels, origin top-left. + +**`-b, --backend `** + + Which device to run on. Defaults to automatic — GPU if the build has Vulkan and a device is + available, CPU otherwise. + +**`--composite `** + + Also write the input image combined with the resulting mask, instead of the mask alone. + +**`--tile `** + + Split large inputs into tiles of this size. Used by `esrgan` to keep memory bounded. + +**`-h, --help`** + + Print the command list and exit. ## Getting weights diff --git a/infer_yolo.sh b/infer_yolo.sh deleted file mode 100644 index 7fd37fb..0000000 --- a/infer_yolo.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -./wsl_build/bin/vision-cli yolov9t -m models/yolov9t_converted.gguf -i tests/input/cat-and-hat.jpg -o ./yolov9t_detections.jpg 2>&1 \ No newline at end of file diff --git a/scripts/dump_yolov9t_compare.py b/scripts/dump_yolov9t_compare.py deleted file mode 100644 index ff52808..0000000 --- a/scripts/dump_yolov9t_compare.py +++ /dev/null @@ -1,319 +0,0 @@ -import argparse -import os -import re -import sys -from typing import Dict, List, Tuple - -import numpy as np -import torch -import torch.nn as nn -from PIL import Image - -# Reuse the custom model and weight mapping utilities -from yolov9 import YOLOv9t_Seq -import sys, subprocess - - -def read_cwhn_txt_tensor(path: str) -> torch.Tensor: - """Read a text tensor saved by C++ save_input_to_txt (C,H,W,N layout).""" - with open(path, "r") as f: - header1 = f.readline().strip() - if not header1.startswith("# input shape") and not header1.startswith("# shape"): - # The file may start directly with numbers - # rewind to beginning - f.seek(0) - data = np.fromstring(f.read().strip().replace("\n", " "), sep=" ") - return torch.from_numpy(data.astype(np.float32)) - - # Parse shape line: e.g. "# input shape C,H,W,N = 3,640,640,1" - m = re.search(r"C,H,W,N\s*=\s*([0-9]+),([0-9]+),([0-9]+),([0-9]+)", header1) - if not m: - raise ValueError(f"Failed to parse shape header in {path}: '{header1}'") - C, H, W, N = map(int, m.groups()) - header2 = f.readline() # type line - # Remaining are whitespace-separated values - data = np.fromstring(f.read().strip().replace("\n", " "), sep=" ") - expected = C * H * W * N - if data.size != expected: - raise ValueError(f"Data size mismatch in {path}: {data.size} vs expected {expected}") - x = torch.from_numpy(data.astype(np.float32)).view(C, H, W, N) - return x - - -def write_cwhn_txt_tensor(t: torch.Tensor, path: str, *, comment_name: str = "input") -> None: - """Write tensor to txt in the same format as C++ save_input_to_txt: C,H,W,N with header.""" - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - # Ensure C,H, W,N - if t.dim() == 4 and t.shape[-1] in (1,): - cwhn = t - elif t.dim() == 4: # likely NCHW - # N,C,H,W -> C,H,W,N - cwhn = t.permute(1, 2, 3, 0).contiguous() - elif t.dim() == 3: # C,H,W -> C,H,W,1 - cwhn = t.unsqueeze(-1) - else: - raise ValueError(f"Unsupported tensor shape for writing: {tuple(t.shape)}") - - C, H, W, N = cwhn.shape - with open(path, "w") as f: - f.write(f"# {comment_name} shape C,H,W,N = {C},{H},{W},{N}\n") - f.write("# type = f32\n") - flat = cwhn.reshape(-1).cpu().numpy() - # Write space-separated with newline at end - for i, v in enumerate(flat): - end = "\n" if i + 1 == flat.size else " " - f.write(("%g" % float(v)) + end) - - -# Add: build runner reflecting build_code.sh - - -def run_build(build_script: str = "/mnt/e/7_RISCV/vision.cpp/build_code.sh") -> None: - if not os.path.exists(build_script): - raise FileNotFoundError(f"Build script not found: {build_script}") - print(f"[build] Running: {build_script}") - subprocess.run(["bash", build_script], check=True) - - -# Add: C++ vision-cli runner reflecting excute.sh - - -def run_cpp_vision_cli( - cpp_bin: str, - image_path: str, - model_path: str, - output_path: str, - dump_all: bool, - dump_keys: List[int] = None, - extra_args: str = "", -) -> None: - if not os.path.exists(cpp_bin): - raise FileNotFoundError(f"vision-cli not found: {cpp_bin}") - cmd = [ - cpp_bin, - "yolov9t", - "-m", - model_path, - "-i", - image_path, - "-o", - output_path, - ] - if dump_all or not dump_keys: - cmd.append("--dump-all") - else: - cmd.append("--dump-keys") - cmd += [str(k) for k in (dump_keys or [])] - if extra_args: - cmd += extra_args.split() - repo_root = "/mnt/e/7_RISCV/vision.cpp" - print("[cpp] Running:", " ".join(cmd)) - subprocess.run(cmd, check=True, cwd=repo_root) - - -def preprocess_like_cpp(img_path: str, input_size: int = 640) -> torch.Tensor: - """Replicate the C++ yolov9t_process_input: scale longest side to input_size, paste top-left, to [0,1]. - - Returns CHW float32 in [0,1]. - """ - img = Image.open(img_path).convert("RGB") - W, H = img.size - s = float(input_size) / float(max(H, W)) - new_w, new_h = int(round(W * s)), int(round(H * s)) - img_resized = img.resize((new_w, new_h), Image.BILINEAR) - - canvas = Image.new("RGB", (input_size, input_size), (0, 0, 0)) - canvas.paste(img_resized, (0, 0)) # top-left paste to match C++ behavior - arr = np.asarray(canvas).astype(np.float32) / 255.0 # H,W,C - chw = torch.from_numpy(arr).permute(2, 0, 1).contiguous() # C,H,W - return chw - - -@torch.no_grad() -def run_and_dump( - img_path: str, - weight_path: str, - out_path: str, - dump_all: bool, - dump_keys: List[int], - use_cpp_input_txt: str = "", -) -> None: - model = YOLOv9t_Seq(ch=3, nc=80) - # Load Ultralytics weights with detect head mapping - try: - model.load_ultralytics_weights_fixed(weight_path) - except Exception: - # fallback - model.load_ultralytics_weights(weight_path) - model.eval() - - base = os.path.splitext(out_path)[0] - - # Prepare input - if use_cpp_input_txt: - x_cwhn = read_cwhn_txt_tensor(use_cpp_input_txt) # C,H,W,N - if x_cwhn.dim() != 4 or x_cwhn.shape[-1] != 1: - raise ValueError("C++ input txt must be C,H,W,N with N=1") - x = x_cwhn.permute(3, 0, 1, 2).contiguous() # N,C,H,W - # Save back what we read (for symmetry) - write_cwhn_txt_tensor(x_cwhn, base + "_input.txt", comment_name="input") - else: - chw = preprocess_like_cpp(img_path, input_size=640) - x = chw.unsqueeze(0) # N,C,H,W - write_cwhn_txt_tensor( - x.permute(1, 2, 3, 0).contiguous(), base + "_input.txt", comment_name="input" - ) - - # Forward - inf_out, raw_outs = model(x) - - # Dump features along the way by running the layers manually similar to forward, or re-run with hooks - # We'll register forward hooks to capture layer outputs matching indices used in C++ - save_indices = [3, 6, 9, 15, 18, 21] - features: Dict[int, torch.Tensor] = {} - - def make_hook(idx: int): - def hook(module, inp, out): - # ensure C,H,W,N layout for saving - if isinstance(out, (list, tuple)): - t = out[0] - else: - t = out - if t.dim() == 4: # N,C,H,W - write_cwhn_txt_tensor( - t.permute(1, 2, 3, 0).contiguous(), - f"{base}_features_layer_{idx}.txt", - comment_name=f"layer_{idx}", - ) - features[idx] = t.detach().cpu() - - return hook - - hooks = [] - try: - for idx in save_indices: - if idx < len(model.model): - h = model.model[idx].register_forward_hook(make_hook(idx)) - hooks.append(h) - - # Re-run a forward pass to trigger hooks - _ = model(x) - finally: - for h in hooks: - h.remove() - - # Dump raw outputs per scale similar to C++ flat [C, H*W, 1, 1] -> treat as [C, num_anchors, 1, 1] - for i, t in enumerate(raw_outs): - if t.dim() == 3: # [C, H*W, N]? Some implementations flatten already - C, HW, N = t.shape - cwhn = t.view(C, HW, 1).unsqueeze(-1) # C, HW, 1, 1 - elif t.dim() == 4: # [N, C, H, W] - N, C, H, W = t.shape - cwhn = t.permute(1, 2, 3, 0).contiguous() # C,H,W,N - else: - # Fallback flatten - flat = t.reshape(t.shape[0], -1, 1, 1) if t.dim() >= 2 else t.reshape(-1, 1, 1, 1) - cwhn = flat - write_cwhn_txt_tensor(cwhn, f"{base}_features_raw_{i}.txt", comment_name=f"raw_{i}") - - # Dump predictions (post-DFL + cls depending on Detect) - if isinstance(inf_out, torch.Tensor): - # Try to reshape like C++: [C, num_anchors, 1, 1] - if inf_out.dim() == 3: # [N, num_anchors, C] - N, A, C = inf_out.shape - pred = inf_out.permute(2, 1, 0).contiguous().view(C, A, 1, 1) - elif inf_out.dim() == 2: # [num_anchors, C] - A, C = inf_out.shape - pred = inf_out.t().contiguous().view(C, A, 1, 1) - elif inf_out.dim() == 4 and inf_out.shape[0] == 1: # [1, C, H, W] - pred = inf_out.squeeze(0).permute(0, 1, 2).unsqueeze(-1) # C,H,W,1 - else: - # Fallback: flatten to [C, A, 1, 1] - flat = ( - inf_out.reshape(inf_out.shape[-1], -1) - if inf_out.dim() > 1 - else inf_out.reshape(1, -1) - ) - pred = flat.t().contiguous().view(flat.shape[1], flat.shape[0], 1, 1) - write_cwhn_txt_tensor(pred, f"{base}_features_predictions.txt", comment_name="predictions") - - -def parse_args(argv: List[str]) -> argparse.Namespace: - ap = argparse.ArgumentParser(description="Dump YOLOv9t PyTorch features to compare with C++") - ap.add_argument("-i", "--input", required=True, help="Input image path") - ap.add_argument("-w", "--weights", required=True, help="Ultralytics yolov9t.pt path") - ap.add_argument( - "-o", "--output", required=True, help="Output image path (base name used for dumps)" - ) - ap.add_argument( - "--dump-all", action="store_true", help="Dump all predefined feature layers and outputs" - ) - ap.add_argument( - "--dump-keys", - type=int, - nargs="*", - default=[], - help="Specific feature layer indices to dump", - ) - ap.add_argument( - "--use-cpp-input", - default="", - help="Use preprocessed input txt from C++ (path to *_input.txt)", - ) - # Added flags to mirror shell scripts - ap.add_argument("--build-cpp", action="store_true", help="Build C++ project via build_code.sh") - ap.add_argument( - "--run-cpp", action="store_true", help="Run C++ vision-cli (excute.sh-like) before Python" - ) - ap.add_argument( - "--cpp-bin", - default="/mnt/e/7_RISCV/vision.cpp/build/bin/vision-cli", - help="Path to vision-cli binary", - ) - ap.add_argument( - "--cpp-model", default="models/yolov9t_converted.gguf", help="GGUF model path for C++ run" - ) - ap.add_argument( - "--cpp-output", default=None, help="Output path for C++ run; defaults to --output" - ) - ap.add_argument( - "--cpp-extra", default="", help="Extra args for vision-cli (e.g., --backend gpu)" - ) - return ap.parse_args(argv) - - -def main(argv: List[str]) -> int: - args = parse_args(argv) - # Optionally build C++ - if getattr(args, "build_cpp", False): - run_build() - # Optionally run C++ to generate dumps and reuse its input - use_cpp_input = args.use_cpp_input - if getattr(args, "run_cpp", False): - cpp_out = args.cpp_output or args.output - run_cpp_vision_cli( - cpp_bin=args.cpp_bin, - image_path=args.input, - model_path=args.cpp_model, - output_path=cpp_out, - dump_all=args.dump_all or (not args.dump_keys), - dump_keys=args.dump_keys if args.dump_keys else None, - extra_args=args.cpp_extra, - ) - base_cpp = os.path.splitext(cpp_out)[0] - use_cpp_input = base_cpp + "_input.txt" - # Proceed with Python dump side - run_and_dump( - img_path=args.input, - weight_path=args.weights, - out_path=args.output, - dump_all=args.dump_all, - dump_keys=args.dump_keys, - use_cpp_input_txt=use_cpp_input, - ) - print("Done.") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/yolov9t.py b/scripts/yolov9t.py deleted file mode 100644 index c7cbb9e..0000000 --- a/scripts/yolov9t.py +++ /dev/null @@ -1,361 +0,0 @@ -# ultralytics yolov9t를 로드해서 sequential 모델로 변환 -from collections import OrderedDict -import torch - -try: - from modules import * -except: - from scripts.modules import * - - -class YOLOv9t_Seq(nn.Module): - """YOLOv9-t model implementation""" - - def __init__(self, ch=3, nc=80): - super().__init__() - self.nc = nc - - # Sequential model with all layers - self.model = nn.Sequential( - Conv(ch, 16, 3, 2), # 0-P1/2 - Conv(16, 32, 3, 2), # 1-P2/4 - ELAN1(32, 32, 32, 16), # 2 - AConv(32, 64), # 3-P3/8 - RepNCSPELAN4(64, 64, 64, 32, 3), # 4 - AConv(64, 96), # 5-P4/16 - RepNCSPELAN4(96, 96, 96, 48, 3), # 6 - AConv(96, 128), # 7-P5/32 - RepNCSPELAN4(128, 128, 128, 64, 3), # 8 - SPPELAN(128, 128, 64), # 9 - nn.Upsample(None, 2, "nearest"), # 10 - Concat(1), # 11-cat backbone P4(5) - RepNCSPELAN4(224, 96, 96, 48, 3), # 12 - nn.Upsample(None, 2, "nearest"), # 13 - Concat(1), # 14 cat backbone P3(3) - RepNCSPELAN4(160, 64, 64, 32, 3), # 15 - N3 output - AConv(64, 48), # 16 - Concat(1), # 17 cat head P4() - RepNCSPELAN4(144, 96, 96, 48, 3), # 18 - (P4/16-medium) - AConv(96, 64), # 19 - Concat(1), # 20 cat head P5 - RepNCSPELAN4(192, 128, 128, 64, 3), # 21 - (P5/32-large) - ) - - # Detection head - self.detect = Detect(nc, (64, 96, 128)) # 22 - self.stride = 32 - # Save indices for feature extraction - self.save_indices = [3, 6, 9, 15, 18, 21] # P3, P4, P5, N3, N4, N5 - - self._initialize_weights() - - def _initialize_weights(self): - """Initialize model weights""" - for m in self.modules(): - if isinstance(m, nn.Conv2d): - nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu") - if m.bias is not None: - nn.init.constant_(m.bias, 0) - elif isinstance(m, nn.BatchNorm2d): - nn.init.constant_(m.weight, 1) - nn.init.constant_(m.bias, 0) - - def forward(self, x): - """Forward pass with intermediate feature extraction""" - features = {} - # Pass through each layer and save specific outputs - for i in range(len(self.model)): - layer = self.model[i] - - # Handle Concat operations - if isinstance(layer, Concat): - if i == 11: # Upsample(10) + P4(6) - x = layer([x, features[6]]) - elif i == 14: # Upsample(13) + P3(3) - x = layer([x, features[4]]) - elif i == 17: # AConv(16) + P4(12) ← 수정! - x = layer([x, features[12]]) # layer 6이 아니라 12! - elif i == 20: # AConv(19) + P5(9) ← 여기는 맞음 - x = layer([x, features[9]]) - else: - x = layer(x) - layer_name = layer.__class__.__name__ - - # Save features needed for concat or output - if i in [4, 6, 9, 12, 15, 18, 21]: - features[i] = x - - # Multi-scale detection outputs - outputs = [features[15], features[18], features[21]] - - - # Detection head - if self.training: - return self.detect(outputs) - else: - inference_out, raw_outputs = self.detect(outputs) - return inference_out, raw_outputs - - def load_ultralytics_weights(self, weight_path): - """ - Load weights from Ultralytics YOLOv9 checkpoint with improved mapping - - Args: - weight_path (str): Path to .pt file (e.g., 'yolov9t.pt') - """ - print(f"Loading weights from {weight_path}...") - - # Load checkpoint - checkpoint = torch.load(weight_path, map_location="cpu") - - # Extract state dict - if isinstance(checkpoint, dict): - if "model" in checkpoint: - source_model = checkpoint["model"] - if hasattr(source_model, "state_dict"): - state_dict = source_model.state_dict() - elif hasattr(source_model, "float"): # It's a model object - state_dict = source_model.float().state_dict() - else: - state_dict = source_model - elif "state_dict" in checkpoint: - state_dict = checkpoint["state_dict"] - else: - state_dict = checkpoint - else: - state_dict = checkpoint.state_dict() - - # Print source model structure for debugging - print("\n=== Analyzing Source Model Structure ===") - source_keys = list(state_dict.keys()) - - # Group keys by layer - layer_groups = {} - for key in source_keys: - parts = key.split(".") - if len(parts) >= 2: - if parts[0] == "model": - layer_idx = parts[1] - if layer_idx not in layer_groups: - layer_groups[layer_idx] = [] - layer_groups[layer_idx].append(key) - - # Show structure of problematic layers - for idx in ["2", "4", "6", "8", "12", "15", "18", "21", "22"]: - if idx in layer_groups: - print(f"\nLayer {idx} keys (showing first 5):") - for key in layer_groups[idx][:5]: - print(f" {key}") - - # Load weights with improved mapping - self._load_state_dict_with_improved_mapping(state_dict) - - print("\nWeights loaded successfully!") - - def _load_state_dict_with_improved_mapping(self, source_state_dict): - """Load state dict with improved key mapping including Detect head""" - target_state_dict = self.state_dict() - mapped_state_dict = OrderedDict() - - def find_source_key(target_key, source_keys): - """Find matching source key for target key""" - - # Direct match - if target_key in source_keys: - return target_key - - # Pattern 1: detect.* -> model.22.* - if target_key.startswith("detect."): - alt_key = target_key.replace("detect.", "model.22.") - if alt_key in source_keys: - return alt_key - - # Pattern 2: model.X.cv4 -> model.X.cv4 (already correct) - if ".cv4." in target_key and target_key in source_keys: - return target_key - - # Pattern 3: Try fuzzy matching for similar keys - target_parts = target_key.split(".") - for source_key in source_keys: - source_parts = source_key.split(".") - - # Match based on last 3 components and shape - if len(target_parts) >= 3 and len(source_parts) >= 3: - target_suffix = ".".join(target_parts[-3:]) - source_suffix = ".".join(source_parts[-3:]) - - if target_suffix == source_suffix: - # Check shape compatibility - try: - if ( - target_state_dict[target_key].shape - == source_state_dict[source_key].shape - ): - return source_key - except: - pass - - return None - - source_keys = set(source_state_dict.keys()) - matched_count = 0 - missing_keys = [] - - print("\n=== Mapping Details ===") - detect_mapped = 0 - backbone_mapped = 0 - - for target_key in target_state_dict.keys(): - source_key = find_source_key(target_key, source_keys) - - if source_key: - # Verify shape compatibility - if target_state_dict[target_key].shape == source_state_dict[source_key].shape: - mapped_state_dict[target_key] = source_state_dict[source_key] - matched_count += 1 - - # Count detect vs backbone - if target_key.startswith("detect."): - detect_mapped += 1 - elif target_key.startswith("model."): - backbone_mapped += 1 - else: - print( - f"Shape mismatch: {target_key} {target_state_dict[target_key].shape} != {source_key} {source_state_dict[source_key].shape}" - ) - missing_keys.append(target_key) - else: - missing_keys.append(target_key) - # Debug: show what we're looking for - if target_key.startswith("detect."): - attempted_key = target_key.replace("detect.", "model.22.") - if attempted_key in source_keys: - print(f"✓ Found mapping: {target_key} -> {attempted_key}") - else: - print(f"✗ Missing: {target_key} (tried: {attempted_key})") - - # Load mapped weights - self.load_state_dict(mapped_state_dict, strict=False) - - print(f"\n=== Loading Results ===") - print(f"Backbone parameters: {backbone_mapped}") - print(f"Detect head parameters: {detect_mapped}") - print(f"Total mapped: {matched_count}/{len(target_state_dict)}") - print(f"Missing: {len(missing_keys)}") - - if missing_keys and len(missing_keys) <= 20: - print("\nMissing keys:") - for key in missing_keys: - print(f" {key}") - - return matched_count, missing_keys - - # 또는 더 직접적인 방법 - def load_ultralytics_weights_fixed(self, weight_path): - """Load weights with explicit Detect head mapping""" - print(f"Loading weights from {weight_path}...") - - checkpoint = torch.load(weight_path, map_location="cpu", weights_only=False) - - if isinstance(checkpoint, dict) and "model" in checkpoint: - source_model = checkpoint["model"] - if hasattr(source_model, "state_dict"): - state_dict = source_model.state_dict() - else: - state_dict = source_model - else: - state_dict = checkpoint - - # Create new state dict with mapped keys - new_state_dict = OrderedDict() - - for source_key, source_param in state_dict.items(): - # Map model.22.* to detect.* - if source_key.startswith("model.22."): - target_key = source_key.replace("model.22.", "detect.") - new_state_dict[target_key] = source_param - else: - new_state_dict[source_key] = source_param - - # Now load with the remapped keys - target_state = self.state_dict() - final_state = OrderedDict() - - matched = 0 - missing = [] - - for target_key in target_state.keys(): - if target_key in new_state_dict: - if target_state[target_key].shape == new_state_dict[target_key].shape: - final_state[target_key] = new_state_dict[target_key] - matched += 1 - else: - print(f"Shape mismatch: {target_key}") - missing.append(target_key) - else: - missing.append(target_key) - - self.load_state_dict(final_state, strict=False) - - print(f"\n✓ Loaded: {matched}/{len(target_state)} parameters") - print(f"✗ Missing: {len(missing)} parameters") - - if missing: - print("\nMissing keys (first 10):") - for key in missing[:10]: - print(f" {key}") - - print("\nWeights loaded!") - - -# 실행해서 구조 확인 -def load_yolov9t(): - - """Load custom YOLOv9t with all weights including Detect head""" - from collections import OrderedDict - - # Load Ultralytics model - from ultralytics import YOLO - YOLO('yolov9t.pt') # Ensure model is downloaded - checkpoint = torch.load("yolov9t.pt", map_location="cpu", weights_only=False) - - if isinstance(checkpoint, dict) and "model" in checkpoint: - source_model = checkpoint["model"] - if hasattr(source_model, "state_dict"): - source_state = source_model.state_dict() - else: - source_state = source_model - else: - source_state = checkpoint - - # Create custom model - model = YOLOv9t_Seq(nc=80, ch=3) - target_state = model.state_dict() - - # Map weights with Detect head fix - new_state = OrderedDict() - - for target_key in target_state.keys(): - source_key = target_key - - # Map detect.* to model.22.* - if target_key.startswith("detect."): - source_key = target_key.replace("detect.", "model.22.") - - if source_key in source_state: - if target_state[target_key].shape == source_state[source_key].shape: - new_state[target_key] = source_state[source_key] - print(f"✓ Mapped: {target_key} <- {source_key}") - - # Load weights - model.load_state_dict(new_state, strict=False) - - # Save with correct mapping - torch.save(model.state_dict(), "yolov9t_converted.pth") - print(f"\nSaved {len(new_state)}/{len(target_state)} parameters") - print(new_state.keys()) - return model - - -if __name__ == "__main__": - model = load_yolov9t() diff --git a/src/cli/cli.cpp b/src/cli/cli.cpp index bc0726b..0d81066 100644 --- a/src/cli/cli.cpp +++ b/src/cli/cli.cpp @@ -1,7 +1,14 @@ +#include +#include +#include + #include "util/math.h" #include "util/string.h" #include "visp/arch_registry.h" +#include "visp/runtime_graph.h" +#include "visp/mmlab.h" #include "visp/draw.h" +#include "visp/mmdet_run.h" #include "visp/postproc.h" #include "visp/vision.h" @@ -33,6 +40,8 @@ struct cli_args { // std::string_view device = 0; // -d --device // ggml_type float_type = GGML_TYPE_COUNT; // -f32 -f16 + char const* input_tensor = nullptr; // normalized F32 CWHN input for reproducible validation + bool raw_outputs = false; char const* composite = nullptr; // --composite int tile_size = -1; // --tile }; @@ -56,6 +65,8 @@ Usage: vision-cli [options] -p, --prompt [ ...] Prompt (eg. pixel coordinates) -b, --backend Backend type (default: auto) -h, --help Print usage and exit + --input-tensor Normalized F32 tensor (MMLab runners) + --raw-outputs Dump graph tensors before detection head --composite Composite input image with mask --tile Tile size to split large images @@ -173,6 +184,10 @@ cli_args cli_parse(int argc, char** argv) { } else { throw except("Unknown backend type '{}', must be one of: cpu, gpu", backend_arg); } + } else if (arg == "--input-tensor") { + r.input_tensor = next_arg(argc, argv, i); + } else if (arg == "--raw-outputs") { + r.raw_outputs = true; } else if (arg == "--composite") { r.composite = next_arg(argc, argv, i); } else if (arg == "--tile") { @@ -318,10 +333,18 @@ std::tuple load_model_weights( model_file file = model_load(model_path_str.c_str()); model_weights weights = model_init(file.n_tensors() + n_tensors); + if (file.arch() == "mmlab") { + preferred_layout = runtime_graph_weight_layout(file, preferred_layout); + } + if (file.arch() == "mmlab-config") preferred_layout = tensor_data_layout::whcn; if (preferred_layout == tensor_data_layout::unknown) { preferred_layout = file.tensor_layout(); } - model_transfer(file, weights, dev, dev.preferred_float_type(), preferred_layout); + // GTX(NPU) 백엔드는 f16 을 선호한다고 답하지만, 가중치를 f16 으로 올리면 f32 활성과 + // 섞여 값이 어긋난다. 장치는 내부에서 f16 으로 변환하므로 여기서는 f32 로 통일한다. + ggml_type transfer_type = dev.type() == backend_type::cpu ? dev.preferred_float_type() + : GGML_TYPE_F32; + model_transfer(file, weights, dev, transfer_type, preferred_layout); printf("done (%s)\n", t.elapsed_str()); ggml_type ftype = file.float_type(); @@ -347,6 +370,264 @@ void compute_timed(compute_graph const& g, backend_device const& b) { printf("complete (%s)\n", t.elapsed_str()); } +// 장치에 커널이 없는 op(pool_2d·sub·sqrt·div·max 등)를 CPU 로 내려보낸다. +// ggml_backend_sched 가 op 단위로 백엔드를 고르므로 장치가 못 받는 노드만 CPU 가 맡는다. +// 폴백이 없으면 그 노드가 실패한 채로 값을 다음 노드에 넘겨 결과가 조용히 틀린다. +// CPU 백엔드로 실행할 때는 갈라 보낼 곳이 없으므로 기존 경로를 그대로 쓴다. +// 입력 전송은 그래프 할당 뒤에 해야 하므로 이 함수 안에서 함께 처리한다. +// sched 와 폴백 CPU 백엔드의 수명을 호출한 쪽으로 넘긴다. 출력 텐서가 sched 버퍼에 있어 +// 결과를 다 읽기 전에 해제하면 안 된다. +struct sched_run { + ggml_backend_sched_t sched = nullptr; + ggml_backend_t cpu = nullptr; + + sched_run() = default; + sched_run(ggml_backend_sched_t s, ggml_backend_t c) : sched(s), cpu(c) {} + sched_run(sched_run const&) = delete; + sched_run& operator=(sched_run const&) = delete; + sched_run(sched_run&& o) noexcept : sched(o.sched), cpu(o.cpu) { + o.sched = nullptr; + o.cpu = nullptr; + } + ~sched_run() { + if (sched) { + ggml_backend_sched_free(sched); + } + if (cpu) { + ggml_backend_free(cpu); + } + } +}; + +// 그래프를 할당한다. 장치 백엔드로 실행할 때는 ggml_backend_sched 로 할당해 op 단위로 +// 백엔드를 고를 수 있게 한다. 장치에 커널이 없는 op(pool_2d·sub·sqrt·div·max 등)는 +// CPU 가 맡는다. 폴백이 없으면 그 노드가 실패한 채로 값을 다음 노드에 넘겨 결과가 +// 조용히 틀린다. +// +// 반환한 sched_run 은 출력을 다 읽을 때까지 살려 둔다. 출력 텐서가 sched 버퍼 안에 있다. +sched_run graph_allocate_split(compute_graph& g, backend_device const& b) { + // 백엔드마다 build flag 가 달라 **같은 모델이라도 그래프 모양이 달라진다.** + // CPU 와 장치의 값을 댈 때 그래프가 같은지 먼저 확인하지 않으면, 커널이 틀린 것과 + // 그래프가 다른 것을 구별할 수 없다. G2C_NPU_OP_CENSUS=1 로 op 분포를 찍는다. + if (std::getenv("G2C_NPU_OP_CENSUS")) { + std::map census; + for (int i = 0; i < ggml_graph_n_nodes(g.graph); ++i) { + ++census[ggml_op_name(ggml_graph_node(g.graph, i)->op)]; + } + printf("- op census (%s, %d nodes):", b.type() == backend_type::cpu ? "cpu" : "dev", + ggml_graph_n_nodes(g.graph)); + for (auto const& [name, n] : census) { + printf(" %s=%d", name.c_str(), n); + } + printf("\n"); + // 장치의 supports_op 은 **축마다** ne <= UINT16_MAX 만 본다. 축이 다 작아도 + // 총 원소가 65535 를 넘는 텐서는 그 검사를 통과한다. 몇 개인지 센다. + { + int over = 0, total = 0; + int64_t worst = 0; + std::map over_by_op; + for (int i = 0; i < ggml_graph_n_nodes(g.graph); ++i) { + ggml_tensor* n = ggml_graph_node(g.graph, i); + ++total; + int64_t ne = ggml_nelements(n); + if (ne > 65535) { + ++over; + ++over_by_op[ggml_op_name(n->op)]; + worst = std::max(worst, ne); + } + } + printf("- 원소 65535 초과 노드: %d / %d (최대 %lld):", over, total, + (long long)worst); + for (auto const& [name, n] : over_by_op) { + printf(" %s=%d", name.c_str(), n); + } + printf("\n"); + } + // stride 까지 찍는다. 장치의 conv 문지기는 nb[0] 만 보므로, 바깥 축이 뒤섞인 + // 텐서도 통과한다. 그 상황인지는 nb 를 눈으로 봐야 갈린다. + for (int i = 0; i < ggml_graph_n_nodes(g.graph); ++i) { + ggml_tensor* n = ggml_graph_node(g.graph, i); + if (n->op != GGML_OP_CONV_2D) { + continue; + } + auto dump = [](char const* tag, ggml_tensor* t) { + printf(" %s ne=[%lld,%lld,%lld,%lld] nb=[%zu,%zu,%zu,%zu] %s%s\n", tag, + (long long)t->ne[0], (long long)t->ne[1], (long long)t->ne[2], + (long long)t->ne[3], t->nb[0], t->nb[1], t->nb[2], t->nb[3], + ggml_type_name(t->type), ggml_is_contiguous(t) ? " cont" : " NOTCONT"); + }; + printf(" conv[%d]\n", i); + dump("w ", n->src[0]); + dump("x ", n->src[1]); + dump("out", n); + } + } + if (b.type() == backend_type::cpu) { + compute_graph_allocate(g, b); + return sched_run{}; + } + ggml_backend_t b_dev = (ggml_backend_t)b; + ggml_backend_t b_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); + if (!b_cpu) { + throw except("CPU backend unavailable for fallback"); + } + ggml_backend_t backs[2] = {b_dev, b_cpu}; + ggml_cgraph* cg = g.graph; + sched_run hold(ggml_backend_sched_new(backs, nullptr, 2, ggml_graph_size(cg), false, true), + b_cpu); + + // sched 를 켜면 장치의 MUL 이 브로드캐스트 경로에서 깨진다. SOFT_MAX 와 모양이 다른 + // MUL/ADD 는 CPU 로 못 박는다. G2C_NPU_NOPIN_BCAST 로 이 핀을 끌 수 있다. + bool no_pin_bcast = std::getenv("G2C_NPU_NOPIN_BCAST") != nullptr; + char const* pin_ops = std::getenv("G2C_NPU_PIN_OPS"); + int64_t pin_big = 0; + if (char const* s_big = std::getenv("G2C_NPU_PIN_BIG")) { + pin_big = std::atoll(s_big); + } + // 장치에 **남길** op 만 지정한다. G2C_NPU_PIN_OPS 의 반대다. + // sched 는 전송을 줄이려고 그래프를 재분할하므로, op 하나를 CPU 로 빼면 이웃이 함께 + // 따라간다. 그래서 "이것만 장치"를 표현하려면 나머지를 전부 못박아야 한다. + char const* only_ops = std::getenv("G2C_NPU_ONLY_OPS"); + // 같은 op 이 여럿일 때 어느 것이 틀렸는지 이분으로 좁힌다. 노드 번호를 쉼표로 잇는다. + // 번호는 G2C_NPU_OP_CENSUS=1 이 찍는 conv[i] 의 i 와 같다. + std::vector pin_nodes; + if (char const* s = std::getenv("G2C_NPU_PIN_NODES")) { + int v = -1; + for (char const* p = s;; ++p) { + if (*p >= '0' && *p <= '9') { + v = (v < 0 ? 0 : v * 10) + (*p - '0'); + } else { + if (v >= 0) { + pin_nodes.push_back(v); + } + v = -1; + if (*p == '\0') { + break; + } + } + } + } + int pinned = 0; + for (int i = 0; i < ggml_graph_n_nodes(cg); ++i) { + ggml_tensor* node = ggml_graph_node(cg, i); + bool to_cpu = node->op == GGML_OP_SOFT_MAX; + if (!to_cpu && std::find(pin_nodes.begin(), pin_nodes.end(), i) != pin_nodes.end()) { + to_cpu = true; + } + // 원소 수가 이 값을 넘는 노드를 CPU 로 뺀다. 장치의 계산이 크기에서 깨지는지 + // 확인할 때 쓴다 — `gtx_smoke` 는 65536 부터 graph_compute 가 실패한다. + if (!to_cpu && pin_big > 0 && ggml_nelements(node) > pin_big) { + to_cpu = true; + } + // 뷰 계열은 계산이 없고 부모의 버퍼를 그대로 쓴다. 여기에 백엔드를 못박으면 + // sched 가 분할을 못 만들고 죽는다(실측: 코어 덤프). 그대로 둔다. + bool is_view = node->op == GGML_OP_RESHAPE || node->op == GGML_OP_VIEW || + node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE || + node->op == GGML_OP_NONE; + if (!to_cpu && only_ops != nullptr && !is_view) { + std::string want = ggml_op_name(node->op); + for (auto& c : want) c = (char) std::toupper((unsigned char) c); + std::string list = only_ops; + for (auto& c : list) c = (char) std::toupper((unsigned char) c); + bool keep = false; + size_t at = list.find(want); + while (at != std::string::npos) { + bool lb = at == 0 || list[at - 1] == ','; + size_t end = at + want.size(); + bool rb = end == list.size() || list[end] == ','; + if (lb && rb) { keep = true; break; } + at = list.find(want, at + 1); + } + if (!keep) { + to_cpu = true; + } else { + // 남길 op 은 **장치로 명시 배정**한다. 안 하면 이웃이 전부 CPU 일 때 + // sched 가 지역성을 우선해 이것도 CPU 로 데려간다(실측). + ggml_backend_sched_set_tensor_backend(hold.sched, node, b_dev); + } + } + if (!to_cpu && !no_pin_bcast && (node->op == GGML_OP_MUL || node->op == GGML_OP_ADD) && + node->src[0] && node->src[1] && !ggml_are_same_shape(node->src[0], node->src[1])) { + to_cpu = true; + } + // 어느 op 이 장치에서 틀린 값을 내는지 좁힐 때 쓴다. 쉼표로 이은 op 이름을 + // G2C_NPU_PIN_OPS 에 주면 그 종류를 통째로 CPU 로 뺀다. + // G2C_NPU_PIN_OPS=CONV_2D,MUL_MAT + // 이름은 ggml_op_name() 이 내는 것과 같다. 대소문자를 가리지 않는다. + if (!to_cpu && pin_ops != nullptr) { + std::string want = ggml_op_name(node->op); + for (auto& c : want) c = (char) std::toupper((unsigned char) c); + std::string list = pin_ops; + for (auto& c : list) c = (char) std::toupper((unsigned char) c); + size_t at = list.find(want); + while (at != std::string::npos) { + bool lb = at == 0 || list[at - 1] == ','; + size_t end = at + want.size(); + bool rb = end == list.size() || list[end] == ','; + if (lb && rb) { to_cpu = true; break; } + at = list.find(want, at + 1); + } + } + if (to_cpu) { + ggml_backend_sched_set_tensor_backend(hold.sched, node, b_cpu); + ++pinned; + } + } + printf("- CPU-pinned (SOFT_MAX, broadcast MUL/ADD%s): %d nodes\n", + pin_ops ? ", G2C_NPU_PIN_OPS" : "", pinned); + + if (!ggml_backend_sched_alloc_graph(hold.sched, cg)) { + throw except("ggml_backend_sched_alloc_graph failed"); + } + int n_dev = 0, n_cpu = 0, n_nodes = ggml_graph_n_nodes(cg); + std::map dev_ops; + int64_t dev_max_ne = 0; + for (int i = 0; i < n_nodes; ++i) { + ggml_tensor* nd = ggml_graph_node(cg, i); + ggml_backend_t sel = ggml_backend_sched_get_tensor_backend(hold.sched, nd); + if (sel == b_dev) { + ++n_dev; + ++dev_ops[ggml_op_name(nd->op)]; + dev_max_ne = std::max(dev_max_ne, ggml_nelements(nd)); + } else if (sel == b_cpu) { + ++n_cpu; + } + } + if (std::getenv("G2C_NPU_OP_CENSUS")) { + // 장치가 실제로 맡은 op 과 그중 가장 큰 텐서. 값이 맞는 구성과 틀리는 구성이 + // 무엇으로 갈리는지 보려면 개수가 아니라 **무엇을 맡았는지**를 봐야 한다. + printf("- 장치가 맡은 op (최대 원소 %lld):", (long long)dev_max_ne); + for (auto const& [name, n] : dev_ops) { + printf(" %s=%d", name.c_str(), n); + } + printf("\n"); + } + printf("- sched routing: %s=%d, CPU=%d / %d nodes\n", ggml_backend_name(b_dev), n_dev, n_cpu, + n_nodes); + + for (tensor_data const& buf : g.buffers) { + transfer_to_backend(buf); + } + g.buffers.clear(); + return hold; +} + +// graph_allocate_split 로 할당한 그래프를 실행한다. 입력 전송은 할당과 실행 사이에 한다. +void compute_timed_split(compute_graph const& g, backend_device const& b, sched_run const& r) { + if (!r.sched) { + compute_timed(g, b); + return; + } + timer t; + printf("Running inference... "); + enum ggml_status st = ggml_backend_sched_graph_compute(r.sched, g.graph); + printf("complete (%s)\n", t.elapsed_str()); + if (st != GGML_STATUS_SUCCESS) { + throw except("graph compute failed (status={})", (int)st); + } +} + + void composite_image_with_mask(image_view image, image_view mask, char const* output_path) { if (!output_path) { return; @@ -439,11 +720,11 @@ void run_sam(cli_args const& args) { sam_prediction output = sam_predict_mask(m, image_embed, prompt_embed); - compute_graph_allocate(graph, backend); + sched_run run_hold = graph_allocate_split(graph, backend); transfer_to_backend(image_tensor, image_data_); transfer_to_backend(point_tensor, span(prompt_data.v, 4)); - compute_timed(graph, backend); + compute_timed_split(graph, backend, run_hold); timer t_post; printf("Postprocessing output... "); @@ -469,6 +750,48 @@ void run_sam(cli_args const& args) { // // g2c 생성 arch (레지스트리 경유). **모델이 늘어도 이 함수는 안 바뀐다.** +// 좌표까지 찍는다. 그리기만 하면 결과를 수치로 확인할 방법이 없어서, 참조 구현과 +// 맞춰 보려면 디코더를 밖에서 다시 짜야 했다. 여기 이미 다 있는 값이다. +// `sx`/`sy` 는 모델 입력 좌표 → 원본 좌표 배율이다. +static void report_and_draw(image_data& image, std::vector dets, + arch_task const& task, float sx, float sy, char const* output) { + // 비율 유지 입력에서는 박스가 패딩 쪽으로 나갈 수 있다. mmdet 처럼 원본 안으로 자른다. + const float W = float(image.extent[0]), H = float(image.extent[1]); + for (detection& d : dets) { + d.x1 = std::clamp(d.x1 * sx, 0.0f, W) / sx; d.x2 = std::clamp(d.x2 * sx, 0.0f, W) / sx; + d.y1 = std::clamp(d.y1 * sy, 0.0f, H) / sy; d.y2 = std::clamp(d.y2 * sy, 0.0f, H) / sy; + } + printf(" %4s %9s %9s %9s %9s %8s %s\n", "#", "x1", "y1", "x2", "y2", "score", "class"); + for (size_t i = 0; i < dets.size(); ++i) { + detection const& d = dets[i]; + char const* name = (d.label >= 0 && size_t(d.label) < task.class_names.size()) + ? task.class_names[d.label].c_str() + : ""; + printf(" %4zu %9.2f %9.2f %9.2f %9.2f %8.4f %d %s\n", i, + d.x1 * sx, d.y1 * sy, d.x2 * sx, d.y2 * sy, d.score, d.label, name); + } + // 표는 전부 찍고, 그림에는 점수 0.3 이상만 그린다. mmdet test_cfg 의 score_thr(YOLOX 0.01)는 + // mAP 평가용이라 그대로 그리면 낮은 점수 박스가 그림을 덮는다. + constexpr float kDrawScoreMin = 0.3f; + std::erase_if(dets, [](detection const& d) { return d.score < kDrawScoreMin; }); + draw_detections(image_span(image), dets, task.class_names, sx, sy); + // 그린 박스를 `.json` 에도 남긴다. SMM Studio 는 이 파일을 읽어 동영상 프레임의 + // `detections` 를 채운다 — 없으면 null 이 된다. 좌표는 원본 이미지 픽셀이다. + std::string json_path = std::string(output) + ".json"; + FILE* f = fopen(json_path.c_str(), "w"); + if (!f) throw except("Cannot write {}", json_path); + fprintf(f, "["); + for (size_t i = 0; i < dets.size(); ++i) { + detection const& d = dets[i]; + std::string name = (d.label >= 0 && size_t(d.label) < task.class_names.size()) + ? task.class_names[d.label] : "class " + std::to_string(d.label); + fprintf(f, "%s{\"x1\":%.2f,\"y1\":%.2f,\"x2\":%.2f,\"y2\":%.2f,\"score\":%.4f,\"class_id\":%d,\"label\":\"%s\"}", + i ? "," : "", d.x1 * sx, d.y1 * sy, d.x2 * sx, d.y2 * sy, d.score, d.label, name.c_str()); + } + fprintf(f, "]\n"); + fclose(f); +} + void run_generated(cli_args const& args) { arch_entry const* e = arch_find(args.arch); ASSERT(e != nullptr, "arch not registered"); // 파싱에서 이미 확인했다 @@ -482,31 +805,88 @@ void run_generated(cli_args const& args) { throw except("Model arch is '{}' but command is '{}'", file.arch(), e->name); } - require_inputs(args.inputs, 1, ""); - image_data image = image_load(args.inputs[0]); + arch_task task = args.arch == "mmlab" ? runtime_graph_task(file) : e->task; + if(args.arch=="mmlab-config") task=mmlab::config_task(file); + if (args.input_tensor && args.arch != "mmlab" && args.arch != "mmlab-config") throw except("--input-tensor requires a MMLab runner"); + image_data image; + if (!args.input_tensor) { + require_inputs(args.inputs, 1, ""); + image = image_load(args.inputs[0]); + } + if (args.raw_outputs) task.kind = arch_kind::raw; + // 비정사각 모델은 `input_w`/`input_h` 를 싣는다. 0 이면 정사각(`input_size`). + const int IW = task.input_w ? task.input_w : task.input_size; + const int IH = task.input_h ? task.input_h : task.input_size; + // 리사이즈 + **등록된** mean/std. `install_arch.py --mean/--std` 가 박는다. + // 단순 리사이즈다 — 종횡비가 바뀌므로 박스를 되돌릴 때 x·y 배율을 따로 쓴다. + const int nch = args.input_tensor ? 3 : n_channels(image.format); + i64x4 input_shape = args.arch == "mmlab" ? runtime_graph_input_shape(file) : i64x4{3,IW,IH,1}; + if(args.arch=="mmlab-config") input_shape=mmlab::config_input_shape(file); + std::vector input_cwhn; + float fit_x = 1.0f, fit_y = 1.0f; // 원본 → 모델 입력 배율 + if (args.input_tensor) { + size_t count=1; + for(int i=0;i<4;++i) count *= size_t(input_shape[i]); + auto size=std::filesystem::file_size(args.input_tensor); + if(size != count*sizeof(float)) throw except("Input tensor byte size {} != {}",size,count*sizeof(float)); + input_cwhn.resize(count); + FILE* f=fopen(args.input_tensor,"rb"); + if(!f) throw except("Cannot read {}",args.input_tensor); + auto read=fread(input_cwhn.data(),sizeof(float),count,f); + fclose(f); + if(read!=count) throw except("Short input tensor read"); + } else { + if (input_shape[0]!=3 || input_shape[3]!=1) throw except("This model requires --input-tensor"); + if (task.keep_ratio) { + input_cwhn = preprocess_keep_ratio(image.data.get(), image.extent[1], image.extent[0], + nch, IW, IH, task.mean.data(), task.stdv.data(), + task.to_rgb, task.pad_value.data(), &fit_x, &fit_y); + } else { + input_cwhn = preprocess(image.data.get(), image.extent[1], image.extent[0], nch, IW, IH, + task.mean.data(), task.stdv.data(), task.to_rgb); + fit_x = float(IW) / float(image.extent[0]); + fit_y = float(IH) / float(image.extent[1]); + } + } - arch_task const& task = e->task; - const int SZ = task.input_size; - // 정사각 리사이즈 + **등록된** mean/std. `install_arch.py --mean/--std` 가 박는다. - // **letterbox 가 아니라 단순 리사이즈**다 — 종횡비가 바뀌므로 박스를 되돌릴 때 - // x·y 배율을 따로 쓴다. - const int nch = n_channels(image.format); - std::vector input_cwhn = preprocess(image.data.get(), image.extent[1], image.extent[0], - nch, SZ, task.mean.data(), task.stdv.data(), /*to_rgb=*/false); + // mmdet 검출기는 백본 뒤에 C++ 로 조립하는 head 가 붙고, DDQ 는 층 사이에 호스트 NMS 가 + // 끼어 그래프가 여러 벌로 갈린다. 그 경로 전체를 라이브러리가 맡는다. + if (task.kind == arch_kind::detect_mmdet) { + std::vector dets = + mmdet_run(backend, weights, file, e->forward, task.mmdet, + std::span(input_cwhn.data(), input_cwhn.size()), IW, IH); + printf("- detections: %zu\n", dets.size()); + if(args.input_tensor) { + std::string output=std::string(args.output)+".detections.json"; + FILE* f=fopen(output.c_str(),"w"); + if(!f) throw except("Cannot write {}",output); + fprintf(f,"["); + for(size_t i=0;i %s\n",output.c_str()); + } else { + report_and_draw(image, dets, task, 1.0f / fit_x, 1.0f / fit_y, args.output); + image_save(image, args.output); + printf("-> %s\n", args.output); + } + return; + } compute_graph graph = compute_graph_init(262144); model_ref m(weights, graph); - tensor input = compute_graph_input(m, GGML_TYPE_F32, {3, SZ, SZ, 1}, "x"); + tensor input = compute_graph_input(m, GGML_TYPE_F32, input_shape, "x"); ggml_build_forward_expand(graph, input); ggml_build_forward_expand(graph, e->forward(m, input, file)); - compute_graph_allocate(graph, backend); + // run_hold 는 출력을 다 읽을 때까지 살아 있어야 한다(sched 버퍼 수명). + sched_run run_hold = graph_allocate_split(graph, backend); transfer_to_backend(input, std::span(input_cwhn.data(), input_cwhn.size())); - compute_timed(graph, backend); + compute_timed_split(graph, backend, run_hold); - // 등록된 out_0.. 를 전부 읽는다. **개수를 강제하지 않는다** — 계열마다 분기 수가 다르다 - // (YOLO26 은 one2many/one2one 두 벌을 낸다). + // 등록된 out_0.. 를 전부 읽는다. **개수를 강제하지 않는다** — 계열마다 분기 수가 다르다. std::vector> outs; - std::vector> hw; for (int i = 0;; ++i) { tensor o = ggml_graph_get_tensor(graph, ("out_" + std::to_string(i)).c_str()); if (!o) { @@ -515,83 +895,18 @@ void run_generated(cli_args const& args) { std::vector d((size_t)ggml_nelements(o)); transfer_from_backend(o, std::span(d.data(), d.size())); outs.push_back(std::move(d)); - hw.push_back({(int)o->ne[2], (int)o->ne[1]}); } printf("- outputs: %zu\n", outs.size()); - if (task.kind != arch_kind::detect_yolo) { - for (size_t i = 0; i < outs.size(); ++i) { - std::string path = std::string(args.output) + "." + std::to_string(i) + ".bin"; - if (FILE* f = fopen(path.c_str(), "wb")) { - fwrite(outs[i].data(), sizeof(float), outs[i].size(), f); - fclose(f); - } - } - printf("-> raw outputs saved to %s..bin\n", args.output); - return; - } - - // 박스/점수 출력 고르기. 지정이 없으면 **뒤에서부터** 찾는다 — YOLO26 은 one2many 가 앞이고 - // 추론에 쓰는 one2one 이 뒤다. 앞 것을 쓰면 조용히 다른 분기를 재게 된다. - int bi = task.box_out, si = task.score_out; - if (bi < 0 || si < 0) { - for (int i = (int)outs.size() - 1; i >= 0; --i) { - if (si < 0 && hw[i].first == task.num_classes) { - si = i; - } else if (si >= 0 && bi < 0 && hw[i].first == 4) { - bi = i; - } - } + for (size_t i = 0; i < outs.size(); ++i) { + std::string path = std::string(args.output) + "." + std::to_string(i) + ".bin"; + FILE* f = fopen(path.c_str(), "wb"); + if(!f) throw except("Cannot write output {}",path); + auto written=fwrite(outs[i].data(),sizeof(float),outs[i].size(),f); + fclose(f); + if(written!=outs[i].size()) throw except("Short output write {}",path); } - if (bi < 0 || si < 0) { - throw except("Could not find box/score outputs (num_classes={})", task.num_classes); - } - const int n_anchor = hw[si].second; - printf("- decode: box=out_%d score=out_%d anchors=%d\n", bi, si, n_anchor); - - // 레벨별 격자는 stride 와 입력 크기로 정해진다. 합이 앵커 수와 안 맞으면 stride 가 틀린 것 — - // 그대로 디코드하면 박스가 통째로 엉뚱한 데 찍힌다. - std::vector> feat_hw; - int sum = 0; - for (float s : task.strides) { - const int g = int(float(SZ) / s); - feat_hw.push_back({g, g}); - sum += g * g; - } - if (sum != n_anchor) { - throw except("anchor mismatch: strides give {} but graph has {}", sum, n_anchor); - } - - yolo_dense_params dp; - dp.strides = task.strides; - dp.num_classes = task.num_classes; - dp.score_thr = task.score_thr; - dp.nms_thr = task.nms_thr; - dp.nms_free = task.nms_free; - dp.max_det = task.max_det; - dp.input_w = SZ; - dp.input_h = SZ; - std::vector dets = - detect_yolo_dense(outs[bi].data(), outs[si].data(), feat_hw, dp); - - const float sx = float(image.extent[0]) / float(SZ); - const float sy = float(image.extent[1]) / float(SZ); - - // 좌표까지 찍는다. 그리기만 하면 결과를 수치로 확인할 방법이 없어서, 참조 구현과 - // 맞춰 보려면 디코더를 밖에서 다시 짜야 했다. 여기 이미 다 있는 값이다. - printf(" %4s %9s %9s %9s %9s %8s %s\n", "#", "x1", "y1", "x2", "y2", "score", "class"); - for (size_t i = 0; i < dets.size(); ++i) { - detection const& d = dets[i]; - char const* name = (d.label >= 0 && size_t(d.label) < task.class_names.size()) - ? task.class_names[d.label].c_str() - : ""; - printf(" %4zu %9.2f %9.2f %9.2f %9.2f %8.4f %d %s\n", i, - d.x1 * sx, d.y1 * sy, d.x2 * sx, d.y2 * sy, d.score, d.label, name); - } - - draw_detections(image_span(image), dets, task.class_names, sx, sy); - image_save(image, args.output); - printf("-> %zu boxes drawn, saved to %s\n", dets.size(), args.output); + printf("-> raw outputs saved to %s..bin\n", args.output); } void run_birefnet(cli_args const& args) { @@ -617,13 +932,13 @@ void run_birefnet(cli_args const& args) { tensor input = compute_graph_input(m, GGML_TYPE_F32, {3, extent[0], extent[1], 1}); tensor output = birefnet_predict(m, input, params); - compute_graph_allocate(graph, backend); + sched_run run_hold = graph_allocate_split(graph, backend); transfer_to_backend(input, input_data); for (tensor_data const& buf : buffers) { transfer_to_backend(buf); } - compute_timed(graph, backend); + compute_timed_split(graph, backend, run_hold); tensor_data mask_data = transfer_from_backend(output); image_view mask_output(extent, mask_data.as_f32()); @@ -659,10 +974,10 @@ void run_depth_anything(cli_args const& args) { tensor input = compute_graph_input(m, GGML_TYPE_F32, {3, extent[0], extent[1], 1}); tensor output = depthany_predict(m, input, params); - compute_graph_allocate(graph, backend); + sched_run run_hold = graph_allocate_split(graph, backend); transfer_to_backend(input, input_data); - compute_timed(graph, backend); + compute_timed_split(graph, backend, run_hold); tensor_data output_data = transfer_from_backend(output); image_data depth_raw = depthany_process_output(output_data.as_f32(), image.extent, params); @@ -696,10 +1011,10 @@ void run_migan(cli_args const& args) { tensor input = compute_graph_input(m, GGML_TYPE_F32, input_shape); tensor output = migan_generate(m, input, params); - compute_graph_allocate(graph, backend); + sched_run run_hold = graph_allocate_split(graph, backend); transfer_to_backend(input, input_data); - compute_timed(graph, backend); + compute_timed_split(graph, backend, run_hold); tensor_data output_data = transfer_from_backend(output); image_data output_image = migan_process_output(output_data.as_f32(), image.extent, params); @@ -738,7 +1053,7 @@ void run_esrgan(cli_args const& args) { tensor input = compute_graph_input(m, GGML_TYPE_F32, input_shape); tensor output = esrgan_generate(m, input, params); - compute_graph_allocate(graph, backend); + sched_run run_hold = graph_allocate_split(graph, backend); timer total; printf( @@ -753,7 +1068,14 @@ void run_esrgan(cli_args const& args) { image_u8_to_f32(image, input_tile, f32x4(0), f32x4(1), tile_offset); transfer_to_backend(input, input_tile); - compute(graph, backend); + if (run_hold.sched) { + enum ggml_status st = ggml_backend_sched_graph_compute(run_hold.sched, graph.graph); + if (st != GGML_STATUS_SUCCESS) { + throw except("graph compute failed (status={})", (int)st); + } + } else { + compute(graph, backend); + } transfer_from_backend(output, output_tile); tile_merge(output_tile, output_image, tile_coord, tiles_out); diff --git a/src/cli/inference_yolov9t.cpp b/src/cli/inference_yolov9t.cpp deleted file mode 100644 index 2c8592f..0000000 --- a/src/cli/inference_yolov9t.cpp +++ /dev/null @@ -1,679 +0,0 @@ -#include -using namespace std; -#include "ggml.h" -#include "util/math.h" -#include "util/string.h" -#include "visp/arch/yolov9t.h" -#include "visp/ml.h" -#include "visp/nn.h" -#include "visp/vision.h" -#include -#include -#include -#include -#include -#include -#include -#include // FLT_MAX - -namespace visp { -using std::filesystem::path; - -enum class cli_command { none, sam, birefnet, depth_anything, migan, esrgan, yolov9t }; - -struct cli_args { - cli_command command = cli_command::none; - std::vector inputs; // -i --input - char const* output = "output.png"; // -o --output - char const* model = nullptr; // -m --model - std::vector prompt; // -p --prompt - // int threads = -1; // -t --threads - // bool verbose = false; // -v --verbose - std::optional bknd_type; // -b --backend - // std::string_view device = 0; // -d --device - // ggml_type float_type = GGML_TYPE_COUNT; // -f32 -f16 - - char const* composite = nullptr; // --composite - int tile_size = -1; // --tile - // Debug dump options - bool dump_all = true; // --dump-all - std::vector dump_keys; // --dump-keys ... -}; - -void print_usage() { - char const* const usage = R"( -Usage: vision-cli [options] - -Commands: - sam - MobileSAM image segmentation - birefnet - BirefNet background removal - depthany - Depth-Anything depth estimation - migan - MI-GAN inpainting - esrgan - ESRGAN/Real-ESRGAN upscaling - yolov9t - YOLOv9t object detection - -Options: - -i, --input [ ...] Input image(s) - -o, --output Output file (default: output.png) - -m, --model Model file (.gguf) - -p, --prompt [ ...] Prompt (eg. pixel coordinates) - -b, --backend Backend type (default: auto) - -h, --help Print usage and exit - --composite Composite input image with mask - --tile Tile size to split large images - -Examples: - vision-cli sam -m MobileSAM-F16.gguf -i image.jpg -p 100 200 -o mask.png - vision-cli birefnet -m BiRefNet-F16.gguf -i image.jpg -o mask.png --composite output.png - vision-cli migan -m MIGAN-F16.gguf -i image.jpg mask.png -o output.png - vision-cli esrgan -m ESRGAN-x4-F16.gguf -i image.jpg -o upscaled.png - vision-cli yolov9t -m yolov9t_converted-F16.gguf -i image.jpg -o detections.png -)"; - printf("%s", usage); -} - -char const* const short_usage = R"( -Usage: vision-cli [options] -See 'vision-cli --help' for more details. -)"; - -char const* next_arg(int argc, char** argv, int& i) { - if (++i < argc) { - return argv[i]; - } else { - throw except("Missing argument after {}", argv[i - 1]); - } -} - -std::vector collect_args(int argc, char** argv, int& i, char delim = '-') { - std::vector r; - do { - r.push_back(next_arg(argc, argv, i)); - } while (i + 1 < argc && argv[i + 1][0] != delim); - if (r.empty()) { - throw except("Missing argument after {}", argv[i - 1]); - } - return r; -} - -int parse_int(std::string_view arg) { - int value = 0; - auto [ptr, ec] = std::from_chars(arg.data(), arg.data() + arg.size(), value); - if (ec != std::errc()) { - throw except("Invalid integer argument: {}", arg); - } - return value; -} - -char const* validate_path(char const* arg) { - if (!exists(path(arg))) { - throw except("File not found: {}", arg); - } - return arg; -} - -void require_inputs(std::span inputs, int n_required, char const* names) { - if (inputs.size() != size_t(n_required)) { - throw except( - "Expected -i to be followed by {} inputs: {} - but found {}.", n_required, names, - inputs.size()); - } -} - -cli_args cli_parse(int argc, char** argv) { - cli_args r; - if (argc < 2) { - throw except("Missing command.\n{}", short_usage); - } - - std::string_view arg1 = argv[1]; - if (arg1 == "yolov9t") { - r.command = cli_command::yolov9t; - } else if (arg1 == "-h" || arg1 == "--help") { - print_usage(); - } else { - throw except("Unknown command: '{}'\n{}", arg1, short_usage); - } - - for (int i = 2; i < argc; ++i) { - std::string_view arg = argv[i]; - if (arg == "-i" || arg == "--input") { - r.inputs = collect_args(argc, argv, i); - for_each(r.inputs.begin(), r.inputs.end(), validate_path); - } else if (arg == "-o" || arg == "--output") { - r.output = next_arg(argc, argv, i); - } else if (arg == "-m" || arg == "--model") { - r.model = validate_path(next_arg(argc, argv, i)); - } else if (arg == "-p" || arg == "--prompt") { - r.prompt = collect_args(argc, argv, i, '-'); - } else if (arg == "-b" || arg == "--backend") { - std::string_view backend_arg = next_arg(argc, argv, i); - if (backend_arg == "cpu") { - r.bknd_type = backend_type::cpu; - } else if (backend_arg == "gpu") { - r.bknd_type = backend_type::gpu; - } else { - throw except("Unknown backend type '{}', must be one of: cpu, gpu", backend_arg); - } - } else if (arg == "--composite") { - r.composite = next_arg(argc, argv, i); - } else if (arg == "--tile") { - r.tile_size = parse_int(next_arg(argc, argv, i)); - } else if (arg == "--dump-all") { - r.dump_all = true; - } else if (arg == "--dump-keys") { - auto keys = collect_args(argc, argv, i); - for (char const* k : keys) { - r.dump_keys.push_back(parse_int(k)); - } - } else if (arg.starts_with("-")) { - throw except("Unknown argument: {}\n{}", arg, short_usage); - } - } - return r; -} - -void run_yolov9t(cli_args const&); - -} // namespace visp - -// -// main - -int main(int argc, char** argv) { - using namespace visp; - try { - ggml_time_init(); - - cli_args args = cli_parse(argc, argv); - switch (args.command) { - case cli_command::sam: - case cli_command::birefnet: - case cli_command::depth_anything: - case cli_command::migan: - case cli_command::esrgan: - printf("Command not implemented yet.\n"); - break; - case cli_command::yolov9t: - run_yolov9t(args); - break; - case cli_command::none: - break; - } - - } catch (std::exception const& e) { - printf("Error: %s\n", e.what()); - return 1; - } catch (...) { - return -1; - } - return 0; -} - -namespace visp { - -struct timer { - int64_t start; - fixed_string<16> string; - - timer() : start(ggml_time_us()) {} - - int64_t elapsed() const { return ggml_time_us() - start; } - float elapsed_ms() const { return float(elapsed()) / 1000.0f; } - - char const* elapsed_str() { - format(string, "{:.1f} ms", elapsed_ms()); - return string.c_str(); - } -}; - -// -// Common helpers - -backend_device backend_init(cli_args const& args) { - timer t; - printf("Initializing backend... "); - - backend_device b; - if (args.bknd_type) { - b = backend_init(*args.bknd_type); - } else { - b = backend_init(); - } - printf("done (%s)\n", t.elapsed_str()); - - ggml_backend_dev_t dev = ggml_backend_get_device(b); - char const* dev_name = ggml_backend_dev_name(dev); - char const* dev_desc = ggml_backend_dev_description(dev); - printf("- device: %s - %s\n", dev_name, dev_desc); - return b; -} - -char const* to_string(tensor_data_layout l) { - switch (l) { - case tensor_data_layout::cwhn: return "cwhn"; - case tensor_data_layout::whcn: return "whcn"; - default: return "unknown"; - } -} - -std::tuple load_model_weights( - cli_args const& args, - backend_device const& dev, - char const* default_model, - int n_tensors = 0, - tensor_data_layout preferred_layout = tensor_data_layout::unknown) { - - timer t; - char const* model_path = args.model ? args.model : default_model; - printf("Loading model weights from '%s'... ", model_path); - - model_file file = model_load(model_path); - model_weights weights = model_init(file.n_tensors() + n_tensors); - if (preferred_layout == tensor_data_layout::unknown) { - preferred_layout = file.tensor_layout(); - } - model_transfer(file, weights, dev, dev.preferred_float_type(), preferred_layout); - - printf("done (%s)\n", t.elapsed_str()); - printf("- float type: %s\n", ggml_type_name(weights.float_type())); - if (preferred_layout != tensor_data_layout::unknown) { - printf("- tensor layout: %s\n", to_string(preferred_layout)); - } - return {std::move(file), std::move(weights)}; -} - -void print_model_flags(model_ref const& m) { - bool flash_attn = !!(m.flags & model_build_flag::flash_attention); - printf("- flash attention: %s\n", flash_attn ? "on" : "off"); -} - -void compute_timed(compute_graph const& g, backend_device const& b) { - timer t; - printf("Running inference... "); - compute(g, b); - printf("complete (%s)\n", t.elapsed_str()); -} - - - -// YOLOv9t -void run_yolov9t(cli_args const& args) { - using namespace visp::yolov9t; - - backend_device backend = backend_init(args); - auto [file, weights] = load_model_weights( - args, backend, "../../../models/yolov9t_converted.gguf", 0, backend.preferred_layout()); - - require_inputs(args.inputs, 1, ""); - image_data image = image_load(args.inputs[0]); - yolov9t_params params = yolov9t_detect_params(file); - - int img_sz = check_img_size(params.input_size); - i32x2 extent = {img_sz, img_sz}; - printf("- model image size: %dx%d\n", extent[0], extent[1]); - printf("- original inference image size: %dx%d\n", image.extent[0], image.extent[1]); - - printf( - "- tensor layout: %s\n", - (backend.preferred_layout() == visp::tensor_data_layout::cwhn) ? "CWHN" : "WHCN"); - - // Build compute graph - compute_graph graph = compute_graph_init(); - model_ref m(weights, graph); - print_model_flags(m); - - // Prepare input - // image_data input_data = yolov9t_process_input2(image, params); - auto input_data = yolov9t_process_input(std::move(image), params); - - - - - - printf("- resized inference image size: %dx%d\n", input_data.img.extent[0], input_data.img.extent[1]); - - tensor input = compute_graph_input( - m, GGML_TYPE_F32, {3, params.input_size, params.input_size, 1}, "input"); - - ggml_build_forward_expand(m.graph, input); - - // Forward pass - DetectOutput d = yolov9t_forward(m, input); - - if (d.debug_cls_logits) { - ggml_build_forward_expand(m.graph, d.debug_cls_logits); - } - - // Build forward graph with predictions - ggml_build_forward_expand(m.graph, d.predictions_cls); - ggml_build_forward_expand(m.graph, d.predictions_bbox); - - - - - // Allocate and run - compute_graph_allocate(graph, backend); - - // Upload input tensor - transfer_to_backend(input, input_data.img); - - // 1) DFL projection (0..reg_max-1) - transfer_to_backend(d.dfl_proj, - std::span(d.dfl_proj_host_data.data(), d.dfl_proj_host_data.size())); - - // 2) Anchor points: [2, total_anchors] - - transfer_to_backend(d.anchor_points, - std::span(d.anchor_host_data.data(), d.anchor_host_data.size())); - - // 3) Stride tensor: [1, total_anchors] - transfer_to_backend(d.strides_points, - std::span(d.stride_host_data.data(), d.stride_host_data.size())); - - - transfer_to_backend(d.strides_points, - std::span(d.stride_host_data.data(), d.stride_host_data.size())); - - - compute_timed(graph, backend); - printf("Forward pass complete.\n"); - - // (Optional) 후처리 가능 - // std::vector detections = - // yolov9t_postprocess(d, params, image.extent); - // draw_detections(image, detections, get_coco_class_names()); - // image_save(image, args.output); - timer t_post; - // std::vector detections = non_max_suppression(outputs); - // scale_boxes(detections, {img_sz, img_sz}, orig_extent); - std::vector detections = - non_max_suppression(d, 0.25f, 0.45f, 300, 30000, 640); // conf, iou, max_det 등 - // scale_boxes(detections, {img_sz, img_sz}, image.extent); - scale_boxes(detections, - {img_sz, img_sz}, // model shape - image.extent, // original shape - input_data.gain, // ✅ from letterbox - input_data.pad_w, // ✅ from letterbox - input_data.pad_h); - - - - // Draw and save - image_data output_image = image_load(args.inputs[0]); - std::vector const& class_names = get_coco_class_names(); - draw_detections(output_image, detections, class_names); - image_save(output_image, args.output); - - printf("Postprocessing complete (%s)\n", t_post.elapsed_str()); - printf("Found %zu objects\n", detections.size()); - - printf("Inference finished.\n"); -} - - - - - - - - - - - - - - - - - -// void run_yolov9t(cli_args const& args) { -// using namespace visp::yolov9t; - -// backend_device backend = backend_init(args); -// auto [file, weights] = load_model_weights( -// args, backend, "../../../models/yolov9t_converted.gguf", 0, backend.preferred_layout()); - -// require_inputs(args.inputs, 1, ""); -// image_data image = image_load(args.inputs[0]); -// yolov9t_params params = yolov9t_detect_params(file); - -// int img_sz = check_img_size(params.input_size); -// i32x2 extent = {img_sz, img_sz}; -// printf("- model image size: %dx%d\n", extent[0], extent[1]); -// printf("- original inference image size: %dx%d\n", image.extent[0], image.extent[1]); -// printf("- tensor layout: %s\n", -// (backend.preferred_layout() == visp::tensor_data_layout::cwhn) ? "CWHN" : "WHCN"); - -// // Build graph -// compute_graph graph = compute_graph_init(); -// model_ref m(weights, graph); -// print_model_flags(m); - -// // Prepare input -// // 변경: letterbox + [0,1] 정규화 경로 -// // image_data in_f32 = yolov9t_process_input(std::move(img), params); -// image_data input_data = yolov9t_process_input(std::move(image), params); - -// // image_data input_data = yolov9t_process_input2(image, params); -// printf("- resized inference image size: %dx%d\n", input_data.extent[0], input_data.extent[1]); - -// // quick sanity on input range -// { -// // input_data는 f32로 [0,1]이어야 함 -// size_t n = (size_t)input_data.extent[0]*input_data.extent[1]*3; -// double mn=1e9, mx=-1e9, mean=0.0; -// size_t n_nan=0, n_inf=0; -// float* p = reinterpret_cast(input_data.data.get()); -// for (size_t i=0;i(d.dfl_proj_host_data.data(), d.dfl_proj_host_data.size())); -// transfer_to_backend(d.anchor_points, std::span(d.anchor_host_data.data(), d.anchor_host_data.size())); -// transfer_to_backend(d.strides_points, std::span(d.stride_host_data.data(), d.stride_host_data.size())); - -// // Sanity: dump proj/anchors/strides heads BEFORE compute -// dbg_print_shape("dfl_proj", d.dfl_proj); -// dbg_print_stats("dfl_proj", d.dfl_proj, 8); -// dbg_dump_anchors_and_strides(d.anchor_points, d.strides_points, 8); - - -// // compute 전에 -// auto ap_before = transfer_from_backend(d.anchor_points).as_f32(); -// auto st_before = transfer_from_backend(d.strides_points).as_f32(); - -// // Run -// timer t_fw; -// compute_timed(graph, backend); -// printf("Forward pass complete. (%s)\n", t_fw.elapsed_str()); - -// { -// auto ap_dev = transfer_from_backend(d.anchor_points).as_f32(); -// auto st_dev = transfer_from_backend(d.strides_points).as_f32(); -// const auto& ap_host = d.anchor_host_data; // [x0,y0,x1,y1,...] -// const auto& st_host = d.stride_host_data; // [s0,s1,...] - -// auto feq = [](float a, float b){ return std::fabs(a-b) < 1e-6f; }; -// bool anchors_ok = true; -// int A = (int)st_host.size(); - -// // 길이 먼저 체크 -// if ((int)ap_dev.size() != 2*A) anchors_ok = false; - -// for (int j = 0; anchors_ok && j < std::min(A, 32); ++j) { // 앞부분만 스팟 체크 -// float xh = ap_host[2*j+0], yh = ap_host[2*j+1]; -// float xd = ap_dev [2*j+0], yd = ap_dev [2*j+1]; -// if (!feq(xh, xd) || !feq(yh, yd)) anchors_ok = false; -// } -// bool strides_ok = true; -// if ((int)st_dev.size() != A) strides_ok = false; -// for (int j = 0; strides_ok && j < std::min(A, 32); ++j) -// if (!feq(st_host[j], st_dev[j])) strides_ok = false; - -// printf("[CHK] anchors equal (host vs device): %s\n", anchors_ok? "YES":"NO"); -// printf("[CHK] strides equal (host vs device): %s\n", strides_ok? "YES":"NO"); - -// // 디버그로 host/dev 헤드 각각 명확히 -// // Inferred correction for line 719 -// printf("HOST anchor head: "); -// for (int j=0;j<8 && jne[0]; -// int64_t na = d.predictions_cls->ne[1]; -// size_t cnt25=0, cnt50=0, cnt70=0, cnt90=0; -// for (int64_t j=0;j=0.25f) cnt25++; -// if (best>=0.50f) cnt50++; -// if (best>=0.70f) cnt70++; -// if (best>=0.90f) cnt90++; -// } -// printf("[DBG] candidate anchors by conf: >=0.25:%zu >=0.50:%zu >=0.70:%zu >=0.90:%zu (na=%lld)\n", -// cnt25, cnt50, cnt70, cnt90, (long long)na); -// } - -// // (기존) 간단 덤프 -// printf("[DEBUG] ======== cls tensor dump ========\n"); -// printf("cls tensor shape: [%ld, %ld, %ld, %ld]\n", -// d.predictions_cls->ne[0], d.predictions_cls->ne[1], -// d.predictions_cls->ne[2], d.predictions_cls->ne[3]); -// { -// tensor_data td_cls = transfer_from_backend(d.predictions_cls); -// auto cls = td_cls.as_f32().data(); -// for (int i = 0; i < 10; ++i) printf("cls[%d] = %.4f\n", i, cls[i]); -// } -// printf("[DEBUG] ====================================\n\n"); - -// { -// tensor_data td_anchor = transfer_from_backend(d.anchor_points); -// auto a = td_anchor.as_f32(); -// printf("anchor_points[0..7]: "); -// for (int i = 0; i < 8 && i < (int)a.size(); ++i) printf("%.2f ", a[i]); -// printf("\n"); -// } - -// // NMS + scale -// timer t_post; -// std::vector detections = -// non_max_suppression(d, 0.01f, 0.45f, 300, 30000, 640); -// scale_boxes(detections, {img_sz, img_sz}, image.extent); - -// // Summary of NMS -// size_t per_cls[80] = {0}; -// for (auto const& o : detections) if (0 <= o.class_id && o.class_id < 80) per_cls[o.class_id]++; -// size_t kept = detections.size(); -// printf("[DBG] NMS kept=%zu (max_det=300)\n", kept); -// for (int c=0;c<80;++c) if (per_cls[c]) printf(" class %d: %zu\n", c, per_cls[c]); - -// // Draw and save -// image_data output_image = image_load(args.inputs[0]); -// std::vector const& class_names = get_coco_class_names(); -// draw_detections(output_image, detections, class_names); - - - -// // ... compute_timed(graph, backend); - -// // compute 후 -// auto ap_after = transfer_from_backend(d.anchor_points).as_f32(); -// auto st_after = transfer_from_backend(d.strides_points).as_f32(); - -// auto same = [](auto a, auto b){ -// if (a.size()!=b.size()) return false; -// for (size_t i=0;i1e-6f) return false; -// return true; -// }; -// printf("[CHK] anchors equal after compute: %s\n", same(ap_before, ap_after) ? "YES" : "NO"); -// printf("[CHK] strides equal after compute: %s\n", same(st_before, st_after) ? "YES" : "NO"); - - -// image_save(output_image, args.output); - -// printf("Postprocessing complete (%s)\n", t_post.elapsed_str()); -// printf("Found %zu objects\n", detections.size()); -// if (top_anchor >= 0) { -// const char* cname = (top_class>=0 && (size_t)top_class < class_names.size()) ? class_names[top_class].c_str() : "unknown"; -// printf("[DBG] Global top-1: prob=%.4f, cls=%d(%s), anchor=%lld\n", -// top_prob, top_class, cname, (long long)top_anchor); -// } -// printf("Inference finished.\n"); -// } - - - - - - - - - - - - - - - - - - - - - - - - -} // namespace visp \ No newline at end of file diff --git a/src/visp/CMakeLists.txt b/src/visp/CMakeLists.txt index 43c7262..4a2e146 100644 --- a/src/visp/CMakeLists.txt +++ b/src/visp/CMakeLists.txt @@ -1,20 +1,46 @@ add_library(visioncpp) # arch/*.cpp 는 **자동으로 모은다.** 예전엔 손으로 나열했는데, g2c 생성물을 붙일 때마다 -# 한 줄 추가를 빠뜨리면 그 파일이 통째로 빌드에서 빠진 채 아무도 모른다 -# (`src/cli/inference_yolov9t.cpp` 가 실제로 그렇게 죽은 코드가 됐다). +# 한 줄 추가를 빠뜨리면 그 파일이 통째로 빌드에서 빠진 채 아무도 모른다. # ⚠️ CONFIGURE_DEPENDS 가 있어야 파일을 추가했을 때 CMake 가 다시 돈다. file(GLOB VISP_ARCH_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/arch/*.cpp") +# VISP_ARCHS 를 주면 그 이름만 빌드한다. 안 주면 위에서 모은 arch/ 전체 — 기본 동작은 그대로다. +# cmake -S . -B build -DVISP_ARCHS="ResNet;FamAnn" +# ⚠️ 없는 이름은 여기서 멈춘다. 조용히 빠지면 「등록했는데 vision-cli 에 안 보인다」로 +# 나타나고, 원인이 이 목록이라 찾기 어렵다. +if(VISP_ARCHS) + set(_visp_selected "") + foreach(_visp_arch IN LISTS VISP_ARCHS) + set(_visp_arch_src "${CMAKE_CURRENT_SOURCE_DIR}/arch/${_visp_arch}.cpp") + if(NOT EXISTS "${_visp_arch_src}") + message(FATAL_ERROR "VISP_ARCHS: arch/${_visp_arch}.cpp 가 없다") + endif() + list(APPEND _visp_selected "${_visp_arch_src}") + endforeach() + set(VISP_ARCH_SOURCES ${_visp_selected}) + message(STATUS "visp: arch 선택 빌드 — ${VISP_ARCHS}") +endif() + target_sources(visioncpp PRIVATE ${VISP_ARCH_SOURCES} arch_registry.cpp + runtime_graph.cpp + mmlab/config.cpp + mmlab/modules.cpp + mmlab/backbones.cpp + mmlab/heads.cpp + mmlab/transformers.cpp + mmlab/text_fusion.cpp + mmlab/model.cpp draw.cpp c-api.cpp image.cpp ml.cpp nn.cpp platform.cpp + mmdet_head.cpp + mmdet_run.cpp postproc.cpp tracker.cpp vision.cpp @@ -25,7 +51,7 @@ target_compile_options(visioncpp PRIVATE ${VISP_WARNINGS} ${VISP_COMP_OPTIONS}) target_include_directories(visioncpp PUBLIC $ $ - PRIVATE .. + PRIVATE .. ${PROJECT_SOURCE_DIR}/depend/llama/vendor ) target_link_options(visioncpp PRIVATE ${VISP_LINK_OPTIONS}) target_link_libraries(visioncpp diff --git a/src/visp/arch/FamAnn.cpp b/src/visp/arch/FamAnn.cpp new file mode 100644 index 0000000..15b9bfc --- /dev/null +++ b/src/visp/arch/FamAnn.cpp @@ -0,0 +1,266 @@ +// GENERATED BY SuperGate GTX Compiler (ggml/vision.cpp backend), DO NOT EDIT! +#include "visp/arch/FamAnn.h" +#include "visp/ml.h" +#include "visp/nn.h" +#include "visp/vision.h" +#include "util/string.h" +#include +#include // clamp 의 한쪽 경계가 없으면 ±INFINITY 를 낸다 + +#include + +namespace visp { + +tensor FamAnn_forward(model_ref m, tensor x, FamAnn_params const& p) { + (void)p; + // 입력 레이아웃 변환 (TODO: cwhn/whcn 자동 판별) + x = cwhn_to_contiguous_2d(m, x); + + tensor const1 = ggml_arange(m, 0.0625f, 0.5625f, 1.0f) /* const 0.0625 */; + tensor const2 = ggml_arange(m, 0.0625f, 0.5625f, 1.0f) /* const 0.0625 */; + tensor conv3 = conv_2d(m["backbone.stem.0"], x, 2, 1); + tensor act4 = ggml_relu(m, conv3); + tensor conv5 = conv_2d(m["backbone.stem.3"], act4, 1, 1); + tensor act6 = ggml_relu(m, conv5); + tensor conv7 = conv_2d(m["backbone.stem.6"], act6, 1, 1); + tensor act8 = ggml_relu(m, conv7); + tensor pool9 = ggml_pool_2d(m, act8, GGML_OP_POOL_MAX, 3, 3, 2, 2, 1, 1); + tensor conv10 = conv_2d(m["backbone.layer1.0.conv1"], pool9, 1, 0); + tensor act11 = ggml_relu(m, conv10); + tensor conv12 = conv_2d(m["backbone.layer1.0.conv2"], act11, 1, 1); + tensor act13 = ggml_relu(m, conv12); + tensor conv14 = conv_2d(m["backbone.layer1.0.conv3"], act13, 1, 0); + tensor conv15 = conv_2d(m["backbone.layer1.0.downsample.0"], pool9, 1, 0); + tensor add16 = ggml_add(m, conv14, conv15); + tensor act17 = ggml_relu(m, add16); + tensor conv18 = conv_2d(m["backbone.layer1.1.conv1"], act17, 1, 0); + tensor act19 = ggml_relu(m, conv18); + tensor conv20 = conv_2d(m["backbone.layer1.1.conv2"], act19, 1, 1); + tensor act21 = ggml_relu(m, conv20); + tensor conv22 = conv_2d(m["backbone.layer1.1.conv3"], act21, 1, 0); + tensor add23 = ggml_add(m, conv22, act17); + tensor act24 = ggml_relu(m, add23); + tensor conv25 = conv_2d(m["backbone.layer1.2.conv1"], act24, 1, 0); + tensor act26 = ggml_relu(m, conv25); + tensor conv27 = conv_2d(m["backbone.layer1.2.conv2"], act26, 1, 1); + tensor act28 = ggml_relu(m, conv27); + tensor conv29 = conv_2d(m["backbone.layer1.2.conv3"], act28, 1, 0); + tensor add30 = ggml_add(m, conv29, act24); + tensor act31 = ggml_relu(m, add30); + tensor conv32 = conv_2d(m["backbone.layer2.0.conv1"], act31, 1, 0); + tensor act33 = ggml_relu(m, conv32); + tensor conv34 = conv_2d(m["backbone.layer2.0.conv2"], act33, 2, 1); + tensor act35 = ggml_relu(m, conv34); + tensor conv36 = conv_2d(m["backbone.layer2.0.conv3"], act35, 1, 0); + tensor conv37 = conv_2d(m["backbone.layer2.0.downsample.0"], act31, 2, 0); + tensor add38 = ggml_add(m, conv36, conv37); + tensor act39 = ggml_relu(m, add38); + tensor conv40 = conv_2d(m["backbone.layer2.1.conv1"], act39, 1, 0); + tensor act41 = ggml_relu(m, conv40); + tensor conv42 = conv_2d(m["backbone.layer2.1.conv2"], act41, 1, 1); + tensor act43 = ggml_relu(m, conv42); + tensor conv44 = conv_2d(m["backbone.layer2.1.conv3"], act43, 1, 0); + tensor add45 = ggml_add(m, conv44, act39); + tensor act46 = ggml_relu(m, add45); + tensor conv47 = conv_2d(m["backbone.layer2.2.conv1"], act46, 1, 0); + tensor act48 = ggml_relu(m, conv47); + tensor conv49 = conv_2d(m["backbone.layer2.2.conv2"], act48, 1, 1); + tensor act50 = ggml_relu(m, conv49); + tensor conv51 = conv_2d(m["backbone.layer2.2.conv3"], act50, 1, 0); + tensor add52 = ggml_add(m, conv51, act46); + tensor act53 = ggml_relu(m, add52); + tensor conv54 = conv_2d(m["backbone.layer2.3.conv1"], act53, 1, 0); + tensor act55 = ggml_relu(m, conv54); + tensor conv56 = conv_2d(m["backbone.layer2.3.conv2"], act55, 1, 1); + tensor act57 = ggml_relu(m, conv56); + tensor conv58 = conv_2d(m["backbone.layer2.3.conv3"], act57, 1, 0); + tensor add59 = ggml_add(m, conv58, act53); + tensor act60 = ggml_relu(m, add59); + tensor conv61 = conv_2d(m["backbone.layer3.0.conv1"], act60, 1, 0); + tensor act62 = ggml_relu(m, conv61); + tensor conv63 = conv_2d(m["backbone.layer3.0.conv2"], act62, 1, 1); + tensor act64 = ggml_relu(m, conv63); + tensor conv65 = conv_2d(m["backbone.layer3.0.conv3"], act64, 1, 0); + tensor conv66 = conv_2d(m["backbone.layer3.0.downsample.0"], act60, 1, 0); + tensor add67 = ggml_add(m, conv65, conv66); + tensor act68 = ggml_relu(m, add67); + tensor conv69 = conv_2d(m["backbone.layer3.1.conv1"], act68, 1, 0); + tensor act70 = ggml_relu(m, conv69); + tensor conv71 = conv_2d(m["backbone.layer3.1.conv2"], act70, 1, 2, 2); + tensor act72 = ggml_relu(m, conv71); + tensor conv73 = conv_2d(m["backbone.layer3.1.conv3"], act72, 1, 0); + tensor add74 = ggml_add(m, conv73, act68); + tensor act75 = ggml_relu(m, add74); + tensor conv76 = conv_2d(m["backbone.layer3.2.conv1"], act75, 1, 0); + tensor act77 = ggml_relu(m, conv76); + tensor conv78 = conv_2d(m["backbone.layer3.2.conv2"], act77, 1, 2, 2); + tensor act79 = ggml_relu(m, conv78); + tensor conv80 = conv_2d(m["backbone.layer3.2.conv3"], act79, 1, 0); + tensor add81 = ggml_add(m, conv80, act75); + tensor act82 = ggml_relu(m, add81); + tensor conv83 = conv_2d(m["backbone.layer3.3.conv1"], act82, 1, 0); + tensor act84 = ggml_relu(m, conv83); + tensor conv85 = conv_2d(m["backbone.layer3.3.conv2"], act84, 1, 2, 2); + tensor act86 = ggml_relu(m, conv85); + tensor conv87 = conv_2d(m["backbone.layer3.3.conv3"], act86, 1, 0); + tensor add88 = ggml_add(m, conv87, act82); + tensor act89 = ggml_relu(m, add88); + tensor conv90 = conv_2d(m["backbone.layer3.4.conv1"], act89, 1, 0); + tensor act91 = ggml_relu(m, conv90); + tensor conv92 = conv_2d(m["backbone.layer3.4.conv2"], act91, 1, 2, 2); + tensor act93 = ggml_relu(m, conv92); + tensor conv94 = conv_2d(m["backbone.layer3.4.conv3"], act93, 1, 0); + tensor add95 = ggml_add(m, conv94, act89); + tensor act96 = ggml_relu(m, add95); + tensor conv97 = conv_2d(m["backbone.layer3.5.conv1"], act96, 1, 0); + tensor act98 = ggml_relu(m, conv97); + tensor conv99 = conv_2d(m["backbone.layer3.5.conv2"], act98, 1, 2, 2); + tensor act100 = ggml_relu(m, conv99); + tensor conv101 = conv_2d(m["backbone.layer3.5.conv3"], act100, 1, 0); + tensor add102 = ggml_add(m, conv101, act96); + tensor act103 = ggml_relu(m, add102); + tensor conv104 = conv_2d(m["backbone.layer4.0.conv1"], act103, 1, 0); + tensor act105 = ggml_relu(m, conv104); + tensor conv106 = conv_2d(m["backbone.layer4.0.conv2"], act105, 1, 2, 2); + tensor act107 = ggml_relu(m, conv106); + tensor conv108 = conv_2d(m["backbone.layer4.0.conv3"], act107, 1, 0); + tensor conv109 = conv_2d(m["backbone.layer4.0.downsample.0"], act103, 1, 0); + tensor add110 = ggml_add(m, conv108, conv109); + tensor act111 = ggml_relu(m, add110); + tensor conv112 = conv_2d(m["backbone.layer4.1.conv1"], act111, 1, 0); + tensor act113 = ggml_relu(m, conv112); + tensor conv114 = conv_2d(m["backbone.layer4.1.conv2"], act113, 1, 4, 4); + tensor act115 = ggml_relu(m, conv114); + tensor conv116 = conv_2d(m["backbone.layer4.1.conv3"], act115, 1, 0); + tensor add117 = ggml_add(m, conv116, act111); + tensor act118 = ggml_relu(m, add117); + tensor conv119 = conv_2d(m["backbone.layer4.2.conv1"], act118, 1, 0); + tensor act120 = ggml_relu(m, conv119); + tensor conv121 = conv_2d(m["backbone.layer4.2.conv2"], act120, 1, 4, 4); + tensor act122 = ggml_relu(m, conv121); + tensor conv123 = conv_2d(m["backbone.layer4.2.conv3"], act122, 1, 0); + tensor add124 = ggml_add(m, conv123, act118); + tensor act125 = ggml_relu(m, add124); + tensor conv126 = conv_2d(m["decode_head.fusion.stages.0.query_project.conv"], act125, 1, 0); + tensor act127 = ggml_relu(m, conv126); + tensor rs128 = ggml_reshape_3d(m, ggml_cont(m, act127), 8192, 256, 1); + tensor perm129 = ggml_cont(m, ggml_permute(m, rs128, 1, 0, 2, 3)); + tensor conv130 = conv_2d(m["decode_head.fusion.stages.0.key_project.conv"], act103, 1, 0); + tensor act131 = ggml_relu(m, conv130); + tensor conv132 = conv_2d(m["decode_head.fusion.stages.0.value_project"], act103, 1, 0); + tensor pool133 = ggml_pool_2d(m, act131, GGML_OP_POOL_AVG, 128, 64, 128, 64, 0, 0); + tensor rs134 = ggml_reshape_3d(m, ggml_cont(m, pool133), 1, 256, 1); + tensor aph135 = ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act131, 43, act131->ne[1], act131->ne[2], act131->ne[3], act131->nb[1], act131->nb[2], act131->nb[3], (size_t)0*act131->nb[0]))), 0.023255813953488372f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act131, 44, act131->ne[1], act131->ne[2], act131->ne[3], act131->nb[1], act131->nb[2], act131->nb[3], (size_t)42*act131->nb[0]))), 0.022727272727272728f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act131, 43, act131->ne[1], act131->ne[2], act131->ne[3], act131->nb[1], act131->nb[2], act131->nb[3], (size_t)85*act131->nb[0]))), 0.023255813953488372f), 0); + tensor apt136 = ggml_cont(m, ggml_permute(m, aph135, 1, 0, 2, 3)); + tensor apr137 = ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt136, 22, apt136->ne[1], apt136->ne[2], apt136->ne[3], apt136->nb[1], apt136->nb[2], apt136->nb[3], (size_t)0*apt136->nb[0]))), 0.045454545454545456f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt136, 22, apt136->ne[1], apt136->ne[2], apt136->ne[3], apt136->nb[1], apt136->nb[2], apt136->nb[3], (size_t)21*apt136->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt136, 22, apt136->ne[1], apt136->ne[2], apt136->ne[3], apt136->nb[1], apt136->nb[2], apt136->nb[3], (size_t)42*apt136->nb[0]))), 0.045454545454545456f), 0); + tensor pool138 = ggml_cont(m, ggml_permute(m, apr137, 1, 0, 2, 3)); + tensor rs139 = ggml_reshape_3d(m, ggml_cont(m, pool138), 9, 256, 1); + tensor aph140 = ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act131, 22, act131->ne[1], act131->ne[2], act131->ne[3], act131->nb[1], act131->nb[2], act131->nb[3], (size_t)0*act131->nb[0]))), 0.045454545454545456f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act131, 22, act131->ne[1], act131->ne[2], act131->ne[3], act131->nb[1], act131->nb[2], act131->nb[3], (size_t)21*act131->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act131, 22, act131->ne[1], act131->ne[2], act131->ne[3], act131->nb[1], act131->nb[2], act131->nb[3], (size_t)42*act131->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act131, 22, act131->ne[1], act131->ne[2], act131->ne[3], act131->nb[1], act131->nb[2], act131->nb[3], (size_t)64*act131->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act131, 22, act131->ne[1], act131->ne[2], act131->ne[3], act131->nb[1], act131->nb[2], act131->nb[3], (size_t)85*act131->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act131, 22, act131->ne[1], act131->ne[2], act131->ne[3], act131->nb[1], act131->nb[2], act131->nb[3], (size_t)106*act131->nb[0]))), 0.045454545454545456f), 0); + tensor apt141 = ggml_cont(m, ggml_permute(m, aph140, 1, 0, 2, 3)); + tensor apr142 = ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt141, 11, apt141->ne[1], apt141->ne[2], apt141->ne[3], apt141->nb[1], apt141->nb[2], apt141->nb[3], (size_t)0*apt141->nb[0]))), 0.09090909090909091f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt141, 12, apt141->ne[1], apt141->ne[2], apt141->ne[3], apt141->nb[1], apt141->nb[2], apt141->nb[3], (size_t)10*apt141->nb[0]))), 0.08333333333333333f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt141, 11, apt141->ne[1], apt141->ne[2], apt141->ne[3], apt141->nb[1], apt141->nb[2], apt141->nb[3], (size_t)21*apt141->nb[0]))), 0.09090909090909091f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt141, 11, apt141->ne[1], apt141->ne[2], apt141->ne[3], apt141->nb[1], apt141->nb[2], apt141->nb[3], (size_t)32*apt141->nb[0]))), 0.09090909090909091f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt141, 12, apt141->ne[1], apt141->ne[2], apt141->ne[3], apt141->nb[1], apt141->nb[2], apt141->nb[3], (size_t)42*apt141->nb[0]))), 0.08333333333333333f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt141, 11, apt141->ne[1], apt141->ne[2], apt141->ne[3], apt141->nb[1], apt141->nb[2], apt141->nb[3], (size_t)53*apt141->nb[0]))), 0.09090909090909091f), 0); + tensor pool143 = ggml_cont(m, ggml_permute(m, apr142, 1, 0, 2, 3)); + tensor rs144 = ggml_reshape_3d(m, ggml_cont(m, pool143), 36, 256, 1); + tensor pool145 = ggml_pool_2d(m, act131, GGML_OP_POOL_AVG, 16, 8, 16, 8, 0, 0); + tensor rs146 = ggml_reshape_3d(m, ggml_cont(m, pool145), 64, 256, 1); + tensor cat147 = ggml_concat(m, ggml_concat(m, ggml_concat(m, rs134, rs139, 0), rs144, 0), rs146, 0); + tensor pool148 = ggml_pool_2d(m, conv132, GGML_OP_POOL_AVG, 128, 64, 128, 64, 0, 0); + tensor rs149 = ggml_reshape_3d(m, ggml_cont(m, pool148), 1, 256, 1); + tensor aph150 = ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv132, 43, conv132->ne[1], conv132->ne[2], conv132->ne[3], conv132->nb[1], conv132->nb[2], conv132->nb[3], (size_t)0*conv132->nb[0]))), 0.023255813953488372f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv132, 44, conv132->ne[1], conv132->ne[2], conv132->ne[3], conv132->nb[1], conv132->nb[2], conv132->nb[3], (size_t)42*conv132->nb[0]))), 0.022727272727272728f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv132, 43, conv132->ne[1], conv132->ne[2], conv132->ne[3], conv132->nb[1], conv132->nb[2], conv132->nb[3], (size_t)85*conv132->nb[0]))), 0.023255813953488372f), 0); + tensor apt151 = ggml_cont(m, ggml_permute(m, aph150, 1, 0, 2, 3)); + tensor apr152 = ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt151, 22, apt151->ne[1], apt151->ne[2], apt151->ne[3], apt151->nb[1], apt151->nb[2], apt151->nb[3], (size_t)0*apt151->nb[0]))), 0.045454545454545456f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt151, 22, apt151->ne[1], apt151->ne[2], apt151->ne[3], apt151->nb[1], apt151->nb[2], apt151->nb[3], (size_t)21*apt151->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt151, 22, apt151->ne[1], apt151->ne[2], apt151->ne[3], apt151->nb[1], apt151->nb[2], apt151->nb[3], (size_t)42*apt151->nb[0]))), 0.045454545454545456f), 0); + tensor pool153 = ggml_cont(m, ggml_permute(m, apr152, 1, 0, 2, 3)); + tensor rs154 = ggml_reshape_3d(m, ggml_cont(m, pool153), 9, 256, 1); + tensor aph155 = ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv132, 22, conv132->ne[1], conv132->ne[2], conv132->ne[3], conv132->nb[1], conv132->nb[2], conv132->nb[3], (size_t)0*conv132->nb[0]))), 0.045454545454545456f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv132, 22, conv132->ne[1], conv132->ne[2], conv132->ne[3], conv132->nb[1], conv132->nb[2], conv132->nb[3], (size_t)21*conv132->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv132, 22, conv132->ne[1], conv132->ne[2], conv132->ne[3], conv132->nb[1], conv132->nb[2], conv132->nb[3], (size_t)42*conv132->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv132, 22, conv132->ne[1], conv132->ne[2], conv132->ne[3], conv132->nb[1], conv132->nb[2], conv132->nb[3], (size_t)64*conv132->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv132, 22, conv132->ne[1], conv132->ne[2], conv132->ne[3], conv132->nb[1], conv132->nb[2], conv132->nb[3], (size_t)85*conv132->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv132, 22, conv132->ne[1], conv132->ne[2], conv132->ne[3], conv132->nb[1], conv132->nb[2], conv132->nb[3], (size_t)106*conv132->nb[0]))), 0.045454545454545456f), 0); + tensor apt156 = ggml_cont(m, ggml_permute(m, aph155, 1, 0, 2, 3)); + tensor apr157 = ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt156, 11, apt156->ne[1], apt156->ne[2], apt156->ne[3], apt156->nb[1], apt156->nb[2], apt156->nb[3], (size_t)0*apt156->nb[0]))), 0.09090909090909091f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt156, 12, apt156->ne[1], apt156->ne[2], apt156->ne[3], apt156->nb[1], apt156->nb[2], apt156->nb[3], (size_t)10*apt156->nb[0]))), 0.08333333333333333f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt156, 11, apt156->ne[1], apt156->ne[2], apt156->ne[3], apt156->nb[1], apt156->nb[2], apt156->nb[3], (size_t)21*apt156->nb[0]))), 0.09090909090909091f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt156, 11, apt156->ne[1], apt156->ne[2], apt156->ne[3], apt156->nb[1], apt156->nb[2], apt156->nb[3], (size_t)32*apt156->nb[0]))), 0.09090909090909091f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt156, 12, apt156->ne[1], apt156->ne[2], apt156->ne[3], apt156->nb[1], apt156->nb[2], apt156->nb[3], (size_t)42*apt156->nb[0]))), 0.08333333333333333f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt156, 11, apt156->ne[1], apt156->ne[2], apt156->ne[3], apt156->nb[1], apt156->nb[2], apt156->nb[3], (size_t)53*apt156->nb[0]))), 0.09090909090909091f), 0); + tensor pool158 = ggml_cont(m, ggml_permute(m, apr157, 1, 0, 2, 3)); + tensor rs159 = ggml_reshape_3d(m, ggml_cont(m, pool158), 36, 256, 1); + tensor pool160 = ggml_pool_2d(m, conv132, GGML_OP_POOL_AVG, 16, 8, 16, 8, 0, 0); + tensor rs161 = ggml_reshape_3d(m, ggml_cont(m, pool160), 64, 256, 1); + tensor cat162 = ggml_concat(m, ggml_concat(m, ggml_concat(m, rs149, rs154, 0), rs159, 0), rs161, 0); + tensor rs163 = ggml_reshape_3d(m, ggml_cont(m, cat147), 110, 256, 1); + tensor rs164 = ggml_reshape_3d(m, ggml_cont(m, cat162), 110, 256, 1); + tensor perm165 = ggml_cont(m, ggml_permute(m, rs164, 1, 0, 2, 3)); + tensor mm166 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, rs163, 1, 0, 2, 3)), perm129); + tensor t167 = ggml_mul(m, as_f32(m, mm166), as_f32(m, const2)); + tensor sm168 = ggml_soft_max(m, t167); + tensor mm169 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, perm165, 1, 0, 2, 3)), sm168); + tensor perm170 = ggml_cont(m, ggml_permute(m, mm169, 1, 0, 2, 3)); + tensor rs171 = ggml_reshape_4d(m, ggml_cont(m, perm170), 128, 64, 256, 1); + tensor conv172 = conv_2d(m["decode_head.fusion.stages.0.out_project"], rs171, 1, 0); + tensor stk173 = ggml_cont(m, conv172) /* TODO(ggml): stack [1, 1, 2048, 64, 128] 를 4D 로 못 낮춘다 */; + tensor sum174 = ggml_reshape_4d(m, ggml_cont(m, stk173), 128, 64, 2048, 1); + tensor cat175 = ggml_concat(m, sum174, act125, 2); + tensor conv176 = conv_2d(m["decode_head.fusion.bottleneck.conv"], cat175, 1, 0); + tensor conv177 = conv_2d(m["decode_head.bottleneck.conv"], conv176, 1, 1); + tensor act178 = ggml_relu(m, conv177); + tensor conv179 = conv_2d(m["decode_head.context.stages.0.key_project.conv"], act178, 1, 0); + tensor act180 = ggml_relu(m, conv179); + tensor rs181 = ggml_reshape_3d(m, ggml_cont(m, act180), 8192, 256, 1); + tensor perm182 = ggml_cont(m, ggml_permute(m, rs181, 1, 0, 2, 3)); + tensor conv183 = conv_2d(m["decode_head.context.stages.0.key_project.conv"], act178, 1, 0); + tensor act184 = ggml_relu(m, conv183); + tensor conv185 = conv_2d(m["decode_head.context.stages.0.value_project"], act178, 1, 0); + tensor pool186 = ggml_pool_2d(m, act184, GGML_OP_POOL_AVG, 128, 64, 128, 64, 0, 0); + tensor rs187 = ggml_reshape_3d(m, ggml_cont(m, pool186), 1, 256, 1); + tensor aph188 = ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act184, 43, act184->ne[1], act184->ne[2], act184->ne[3], act184->nb[1], act184->nb[2], act184->nb[3], (size_t)0*act184->nb[0]))), 0.023255813953488372f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act184, 44, act184->ne[1], act184->ne[2], act184->ne[3], act184->nb[1], act184->nb[2], act184->nb[3], (size_t)42*act184->nb[0]))), 0.022727272727272728f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act184, 43, act184->ne[1], act184->ne[2], act184->ne[3], act184->nb[1], act184->nb[2], act184->nb[3], (size_t)85*act184->nb[0]))), 0.023255813953488372f), 0); + tensor apt189 = ggml_cont(m, ggml_permute(m, aph188, 1, 0, 2, 3)); + tensor apr190 = ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt189, 22, apt189->ne[1], apt189->ne[2], apt189->ne[3], apt189->nb[1], apt189->nb[2], apt189->nb[3], (size_t)0*apt189->nb[0]))), 0.045454545454545456f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt189, 22, apt189->ne[1], apt189->ne[2], apt189->ne[3], apt189->nb[1], apt189->nb[2], apt189->nb[3], (size_t)21*apt189->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt189, 22, apt189->ne[1], apt189->ne[2], apt189->ne[3], apt189->nb[1], apt189->nb[2], apt189->nb[3], (size_t)42*apt189->nb[0]))), 0.045454545454545456f), 0); + tensor pool191 = ggml_cont(m, ggml_permute(m, apr190, 1, 0, 2, 3)); + tensor rs192 = ggml_reshape_3d(m, ggml_cont(m, pool191), 9, 256, 1); + tensor aph193 = ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act184, 22, act184->ne[1], act184->ne[2], act184->ne[3], act184->nb[1], act184->nb[2], act184->nb[3], (size_t)0*act184->nb[0]))), 0.045454545454545456f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act184, 22, act184->ne[1], act184->ne[2], act184->ne[3], act184->nb[1], act184->nb[2], act184->nb[3], (size_t)21*act184->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act184, 22, act184->ne[1], act184->ne[2], act184->ne[3], act184->nb[1], act184->nb[2], act184->nb[3], (size_t)42*act184->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act184, 22, act184->ne[1], act184->ne[2], act184->ne[3], act184->nb[1], act184->nb[2], act184->nb[3], (size_t)64*act184->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act184, 22, act184->ne[1], act184->ne[2], act184->ne[3], act184->nb[1], act184->nb[2], act184->nb[3], (size_t)85*act184->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act184, 22, act184->ne[1], act184->ne[2], act184->ne[3], act184->nb[1], act184->nb[2], act184->nb[3], (size_t)106*act184->nb[0]))), 0.045454545454545456f), 0); + tensor apt194 = ggml_cont(m, ggml_permute(m, aph193, 1, 0, 2, 3)); + tensor apr195 = ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt194, 11, apt194->ne[1], apt194->ne[2], apt194->ne[3], apt194->nb[1], apt194->nb[2], apt194->nb[3], (size_t)0*apt194->nb[0]))), 0.09090909090909091f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt194, 12, apt194->ne[1], apt194->ne[2], apt194->ne[3], apt194->nb[1], apt194->nb[2], apt194->nb[3], (size_t)10*apt194->nb[0]))), 0.08333333333333333f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt194, 11, apt194->ne[1], apt194->ne[2], apt194->ne[3], apt194->nb[1], apt194->nb[2], apt194->nb[3], (size_t)21*apt194->nb[0]))), 0.09090909090909091f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt194, 11, apt194->ne[1], apt194->ne[2], apt194->ne[3], apt194->nb[1], apt194->nb[2], apt194->nb[3], (size_t)32*apt194->nb[0]))), 0.09090909090909091f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt194, 12, apt194->ne[1], apt194->ne[2], apt194->ne[3], apt194->nb[1], apt194->nb[2], apt194->nb[3], (size_t)42*apt194->nb[0]))), 0.08333333333333333f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt194, 11, apt194->ne[1], apt194->ne[2], apt194->ne[3], apt194->nb[1], apt194->nb[2], apt194->nb[3], (size_t)53*apt194->nb[0]))), 0.09090909090909091f), 0); + tensor pool196 = ggml_cont(m, ggml_permute(m, apr195, 1, 0, 2, 3)); + tensor rs197 = ggml_reshape_3d(m, ggml_cont(m, pool196), 36, 256, 1); + tensor pool198 = ggml_pool_2d(m, act184, GGML_OP_POOL_AVG, 16, 8, 16, 8, 0, 0); + tensor rs199 = ggml_reshape_3d(m, ggml_cont(m, pool198), 64, 256, 1); + tensor cat200 = ggml_concat(m, ggml_concat(m, ggml_concat(m, rs187, rs192, 0), rs197, 0), rs199, 0); + tensor pool201 = ggml_pool_2d(m, conv185, GGML_OP_POOL_AVG, 128, 64, 128, 64, 0, 0); + tensor rs202 = ggml_reshape_3d(m, ggml_cont(m, pool201), 1, 256, 1); + tensor aph203 = ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv185, 43, conv185->ne[1], conv185->ne[2], conv185->ne[3], conv185->nb[1], conv185->nb[2], conv185->nb[3], (size_t)0*conv185->nb[0]))), 0.023255813953488372f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv185, 44, conv185->ne[1], conv185->ne[2], conv185->ne[3], conv185->nb[1], conv185->nb[2], conv185->nb[3], (size_t)42*conv185->nb[0]))), 0.022727272727272728f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv185, 43, conv185->ne[1], conv185->ne[2], conv185->ne[3], conv185->nb[1], conv185->nb[2], conv185->nb[3], (size_t)85*conv185->nb[0]))), 0.023255813953488372f), 0); + tensor apt204 = ggml_cont(m, ggml_permute(m, aph203, 1, 0, 2, 3)); + tensor apr205 = ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt204, 22, apt204->ne[1], apt204->ne[2], apt204->ne[3], apt204->nb[1], apt204->nb[2], apt204->nb[3], (size_t)0*apt204->nb[0]))), 0.045454545454545456f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt204, 22, apt204->ne[1], apt204->ne[2], apt204->ne[3], apt204->nb[1], apt204->nb[2], apt204->nb[3], (size_t)21*apt204->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt204, 22, apt204->ne[1], apt204->ne[2], apt204->ne[3], apt204->nb[1], apt204->nb[2], apt204->nb[3], (size_t)42*apt204->nb[0]))), 0.045454545454545456f), 0); + tensor pool206 = ggml_cont(m, ggml_permute(m, apr205, 1, 0, 2, 3)); + tensor rs207 = ggml_reshape_3d(m, ggml_cont(m, pool206), 9, 256, 1); + tensor aph208 = ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv185, 22, conv185->ne[1], conv185->ne[2], conv185->ne[3], conv185->nb[1], conv185->nb[2], conv185->nb[3], (size_t)0*conv185->nb[0]))), 0.045454545454545456f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv185, 22, conv185->ne[1], conv185->ne[2], conv185->ne[3], conv185->nb[1], conv185->nb[2], conv185->nb[3], (size_t)21*conv185->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv185, 22, conv185->ne[1], conv185->ne[2], conv185->ne[3], conv185->nb[1], conv185->nb[2], conv185->nb[3], (size_t)42*conv185->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv185, 22, conv185->ne[1], conv185->ne[2], conv185->ne[3], conv185->nb[1], conv185->nb[2], conv185->nb[3], (size_t)64*conv185->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv185, 22, conv185->ne[1], conv185->ne[2], conv185->ne[3], conv185->nb[1], conv185->nb[2], conv185->nb[3], (size_t)85*conv185->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, conv185, 22, conv185->ne[1], conv185->ne[2], conv185->ne[3], conv185->nb[1], conv185->nb[2], conv185->nb[3], (size_t)106*conv185->nb[0]))), 0.045454545454545456f), 0); + tensor apt209 = ggml_cont(m, ggml_permute(m, aph208, 1, 0, 2, 3)); + tensor apr210 = ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt209, 11, apt209->ne[1], apt209->ne[2], apt209->ne[3], apt209->nb[1], apt209->nb[2], apt209->nb[3], (size_t)0*apt209->nb[0]))), 0.09090909090909091f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt209, 12, apt209->ne[1], apt209->ne[2], apt209->ne[3], apt209->nb[1], apt209->nb[2], apt209->nb[3], (size_t)10*apt209->nb[0]))), 0.08333333333333333f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt209, 11, apt209->ne[1], apt209->ne[2], apt209->ne[3], apt209->nb[1], apt209->nb[2], apt209->nb[3], (size_t)21*apt209->nb[0]))), 0.09090909090909091f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt209, 11, apt209->ne[1], apt209->ne[2], apt209->ne[3], apt209->nb[1], apt209->nb[2], apt209->nb[3], (size_t)32*apt209->nb[0]))), 0.09090909090909091f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt209, 12, apt209->ne[1], apt209->ne[2], apt209->ne[3], apt209->nb[1], apt209->nb[2], apt209->nb[3], (size_t)42*apt209->nb[0]))), 0.08333333333333333f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt209, 11, apt209->ne[1], apt209->ne[2], apt209->ne[3], apt209->nb[1], apt209->nb[2], apt209->nb[3], (size_t)53*apt209->nb[0]))), 0.09090909090909091f), 0); + tensor pool211 = ggml_cont(m, ggml_permute(m, apr210, 1, 0, 2, 3)); + tensor rs212 = ggml_reshape_3d(m, ggml_cont(m, pool211), 36, 256, 1); + tensor pool213 = ggml_pool_2d(m, conv185, GGML_OP_POOL_AVG, 16, 8, 16, 8, 0, 0); + tensor rs214 = ggml_reshape_3d(m, ggml_cont(m, pool213), 64, 256, 1); + tensor cat215 = ggml_concat(m, ggml_concat(m, ggml_concat(m, rs202, rs207, 0), rs212, 0), rs214, 0); + tensor rs216 = ggml_reshape_3d(m, ggml_cont(m, cat200), 110, 256, 1); + tensor rs217 = ggml_reshape_3d(m, ggml_cont(m, cat215), 110, 256, 1); + tensor perm218 = ggml_cont(m, ggml_permute(m, rs217, 1, 0, 2, 3)); + tensor mm219 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, rs216, 1, 0, 2, 3)), perm182); + tensor t220 = ggml_mul(m, as_f32(m, mm219), as_f32(m, const1)); + tensor sm221 = ggml_soft_max(m, t220); + tensor mm222 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, perm218, 1, 0, 2, 3)), sm221); + tensor perm223 = ggml_cont(m, ggml_permute(m, mm222, 1, 0, 2, 3)); + tensor rs224 = ggml_reshape_4d(m, ggml_cont(m, perm223), 128, 64, 256, 1); + tensor conv225 = conv_2d(m["decode_head.context.stages.0.out_project"], rs224, 1, 0); + tensor stk226 = ggml_cont(m, conv225) /* TODO(ggml): stack [1, 1, 512, 64, 128] 를 4D 로 못 낮춘다 */; + tensor sum227 = ggml_reshape_4d(m, ggml_cont(m, stk226), 128, 64, 512, 1); + tensor cat228 = ggml_concat(m, sum227, act178, 2); + tensor conv229 = conv_2d(m["decode_head.context.bottleneck.conv"], cat228, 1, 0); + tensor act230 = ggml_relu(m, conv229); + tensor conv231 = conv_2d(m["decode_head.conv_seg"], act230, 1, 0); + + tensor out_0 = compute_graph_output(m, contiguous_2d_to_cwhn(m, conv231), "out_0"); + return out_0; +} + +FamAnn_params FamAnn_detect_params(model_file const& f) { + FamAnn_params p{}; + // GGUF general.architecture 검증 (모델은 정적 unroll 이라 추가 하이퍼파라미터 없음). + if (std::string_view arch = f.arch(); arch != "famann") { + throw except( + "Architecture expected to be 'famann', but was '{}' ({})", + arch, f.path); + } + return p; +} + +} // namespace visp diff --git a/src/visp/arch/FamAnn.h b/src/visp/arch/FamAnn.h new file mode 100644 index 0000000..f0e263a --- /dev/null +++ b/src/visp/arch/FamAnn.h @@ -0,0 +1,14 @@ +// GENERATED BY SuperGate GTX Compiler (ggml/vision.cpp backend), DO NOT EDIT! +#pragma once +#include "visp/ml.h" + +namespace visp { + +struct FamAnn_params { + // TODO(ggml): 모델 하이퍼파라미터 (block_count 등) +}; + +tensor FamAnn_forward(model_ref m, tensor x, FamAnn_params const& p); +FamAnn_params FamAnn_detect_params(model_file const& f); + +} // namespace visp diff --git a/src/visp/arch/FamApcnet.cpp b/src/visp/arch/FamApcnet.cpp new file mode 100644 index 0000000..f06d84c --- /dev/null +++ b/src/visp/arch/FamApcnet.cpp @@ -0,0 +1,264 @@ +// GENERATED BY SuperGate GTX Compiler (ggml/vision.cpp backend), DO NOT EDIT! +#include "visp/arch/FamApcnet.h" +#include "visp/ml.h" +#include "visp/nn.h" +#include "visp/vision.h" +#include "util/string.h" +#include +#include // clamp 의 한쪽 경계가 없으면 ±INFINITY 를 낸다 + +#include + +namespace visp { + +tensor FamApcnet_forward(model_ref m, tensor x, FamApcnet_params const& p) { + (void)p; + // 입력 레이아웃 변환 (TODO: cwhn/whcn 자동 판별) + x = cwhn_to_contiguous_2d(m, x); + + tensor conv1 = conv_2d(m["backbone.stem.0"], x, 2, 1); + tensor act2 = ggml_relu(m, conv1); + tensor conv3 = conv_2d(m["backbone.stem.3"], act2, 1, 1); + tensor act4 = ggml_relu(m, conv3); + tensor conv5 = conv_2d(m["backbone.stem.6"], act4, 1, 1); + tensor act6 = ggml_relu(m, conv5); + tensor pool7 = ggml_pool_2d(m, act6, GGML_OP_POOL_MAX, 3, 3, 2, 2, 1, 1); + tensor conv8 = conv_2d(m["backbone.layer1.0.conv1"], pool7, 1, 0); + tensor act9 = ggml_relu(m, conv8); + tensor conv10 = conv_2d(m["backbone.layer1.0.conv2"], act9, 1, 1); + tensor act11 = ggml_relu(m, conv10); + tensor conv12 = conv_2d(m["backbone.layer1.0.conv3"], act11, 1, 0); + tensor conv13 = conv_2d(m["backbone.layer1.0.downsample.0"], pool7, 1, 0); + tensor add14 = ggml_add(m, conv12, conv13); + tensor act15 = ggml_relu(m, add14); + tensor conv16 = conv_2d(m["backbone.layer1.1.conv1"], act15, 1, 0); + tensor act17 = ggml_relu(m, conv16); + tensor conv18 = conv_2d(m["backbone.layer1.1.conv2"], act17, 1, 1); + tensor act19 = ggml_relu(m, conv18); + tensor conv20 = conv_2d(m["backbone.layer1.1.conv3"], act19, 1, 0); + tensor add21 = ggml_add(m, conv20, act15); + tensor act22 = ggml_relu(m, add21); + tensor conv23 = conv_2d(m["backbone.layer1.2.conv1"], act22, 1, 0); + tensor act24 = ggml_relu(m, conv23); + tensor conv25 = conv_2d(m["backbone.layer1.2.conv2"], act24, 1, 1); + tensor act26 = ggml_relu(m, conv25); + tensor conv27 = conv_2d(m["backbone.layer1.2.conv3"], act26, 1, 0); + tensor add28 = ggml_add(m, conv27, act22); + tensor act29 = ggml_relu(m, add28); + tensor conv30 = conv_2d(m["backbone.layer2.0.conv1"], act29, 1, 0); + tensor act31 = ggml_relu(m, conv30); + tensor conv32 = conv_2d(m["backbone.layer2.0.conv2"], act31, 2, 1); + tensor act33 = ggml_relu(m, conv32); + tensor conv34 = conv_2d(m["backbone.layer2.0.conv3"], act33, 1, 0); + tensor conv35 = conv_2d(m["backbone.layer2.0.downsample.0"], act29, 2, 0); + tensor add36 = ggml_add(m, conv34, conv35); + tensor act37 = ggml_relu(m, add36); + tensor conv38 = conv_2d(m["backbone.layer2.1.conv1"], act37, 1, 0); + tensor act39 = ggml_relu(m, conv38); + tensor conv40 = conv_2d(m["backbone.layer2.1.conv2"], act39, 1, 1); + tensor act41 = ggml_relu(m, conv40); + tensor conv42 = conv_2d(m["backbone.layer2.1.conv3"], act41, 1, 0); + tensor add43 = ggml_add(m, conv42, act37); + tensor act44 = ggml_relu(m, add43); + tensor conv45 = conv_2d(m["backbone.layer2.2.conv1"], act44, 1, 0); + tensor act46 = ggml_relu(m, conv45); + tensor conv47 = conv_2d(m["backbone.layer2.2.conv2"], act46, 1, 1); + tensor act48 = ggml_relu(m, conv47); + tensor conv49 = conv_2d(m["backbone.layer2.2.conv3"], act48, 1, 0); + tensor add50 = ggml_add(m, conv49, act44); + tensor act51 = ggml_relu(m, add50); + tensor conv52 = conv_2d(m["backbone.layer2.3.conv1"], act51, 1, 0); + tensor act53 = ggml_relu(m, conv52); + tensor conv54 = conv_2d(m["backbone.layer2.3.conv2"], act53, 1, 1); + tensor act55 = ggml_relu(m, conv54); + tensor conv56 = conv_2d(m["backbone.layer2.3.conv3"], act55, 1, 0); + tensor add57 = ggml_add(m, conv56, act51); + tensor act58 = ggml_relu(m, add57); + tensor conv59 = conv_2d(m["backbone.layer3.0.conv1"], act58, 1, 0); + tensor act60 = ggml_relu(m, conv59); + tensor conv61 = conv_2d(m["backbone.layer3.0.conv2"], act60, 1, 1); + tensor act62 = ggml_relu(m, conv61); + tensor conv63 = conv_2d(m["backbone.layer3.0.conv3"], act62, 1, 0); + tensor conv64 = conv_2d(m["backbone.layer3.0.downsample.0"], act58, 1, 0); + tensor add65 = ggml_add(m, conv63, conv64); + tensor act66 = ggml_relu(m, add65); + tensor conv67 = conv_2d(m["backbone.layer3.1.conv1"], act66, 1, 0); + tensor act68 = ggml_relu(m, conv67); + tensor conv69 = conv_2d(m["backbone.layer3.1.conv2"], act68, 1, 2, 2); + tensor act70 = ggml_relu(m, conv69); + tensor conv71 = conv_2d(m["backbone.layer3.1.conv3"], act70, 1, 0); + tensor add72 = ggml_add(m, conv71, act66); + tensor act73 = ggml_relu(m, add72); + tensor conv74 = conv_2d(m["backbone.layer3.2.conv1"], act73, 1, 0); + tensor act75 = ggml_relu(m, conv74); + tensor conv76 = conv_2d(m["backbone.layer3.2.conv2"], act75, 1, 2, 2); + tensor act77 = ggml_relu(m, conv76); + tensor conv78 = conv_2d(m["backbone.layer3.2.conv3"], act77, 1, 0); + tensor add79 = ggml_add(m, conv78, act73); + tensor act80 = ggml_relu(m, add79); + tensor conv81 = conv_2d(m["backbone.layer3.3.conv1"], act80, 1, 0); + tensor act82 = ggml_relu(m, conv81); + tensor conv83 = conv_2d(m["backbone.layer3.3.conv2"], act82, 1, 2, 2); + tensor act84 = ggml_relu(m, conv83); + tensor conv85 = conv_2d(m["backbone.layer3.3.conv3"], act84, 1, 0); + tensor add86 = ggml_add(m, conv85, act80); + tensor act87 = ggml_relu(m, add86); + tensor conv88 = conv_2d(m["backbone.layer3.4.conv1"], act87, 1, 0); + tensor act89 = ggml_relu(m, conv88); + tensor conv90 = conv_2d(m["backbone.layer3.4.conv2"], act89, 1, 2, 2); + tensor act91 = ggml_relu(m, conv90); + tensor conv92 = conv_2d(m["backbone.layer3.4.conv3"], act91, 1, 0); + tensor add93 = ggml_add(m, conv92, act87); + tensor act94 = ggml_relu(m, add93); + tensor conv95 = conv_2d(m["backbone.layer3.5.conv1"], act94, 1, 0); + tensor act96 = ggml_relu(m, conv95); + tensor conv97 = conv_2d(m["backbone.layer3.5.conv2"], act96, 1, 2, 2); + tensor act98 = ggml_relu(m, conv97); + tensor conv99 = conv_2d(m["backbone.layer3.5.conv3"], act98, 1, 0); + tensor add100 = ggml_add(m, conv99, act94); + tensor act101 = ggml_relu(m, add100); + tensor conv102 = conv_2d(m["backbone.layer4.0.conv1"], act101, 1, 0); + tensor act103 = ggml_relu(m, conv102); + tensor conv104 = conv_2d(m["backbone.layer4.0.conv2"], act103, 1, 2, 2); + tensor act105 = ggml_relu(m, conv104); + tensor conv106 = conv_2d(m["backbone.layer4.0.conv3"], act105, 1, 0); + tensor conv107 = conv_2d(m["backbone.layer4.0.downsample.0"], act101, 1, 0); + tensor add108 = ggml_add(m, conv106, conv107); + tensor act109 = ggml_relu(m, add108); + tensor conv110 = conv_2d(m["backbone.layer4.1.conv1"], act109, 1, 0); + tensor act111 = ggml_relu(m, conv110); + tensor conv112 = conv_2d(m["backbone.layer4.1.conv2"], act111, 1, 4, 4); + tensor act113 = ggml_relu(m, conv112); + tensor conv114 = conv_2d(m["backbone.layer4.1.conv3"], act113, 1, 0); + tensor add115 = ggml_add(m, conv114, act109); + tensor act116 = ggml_relu(m, add115); + tensor conv117 = conv_2d(m["backbone.layer4.2.conv1"], act116, 1, 0); + tensor act118 = ggml_relu(m, conv117); + tensor conv119 = conv_2d(m["backbone.layer4.2.conv2"], act118, 1, 4, 4); + tensor act120 = ggml_relu(m, conv119); + tensor conv121 = conv_2d(m["backbone.layer4.2.conv3"], act120, 1, 0); + tensor add122 = ggml_add(m, conv121, act116); + tensor act123 = ggml_relu(m, add122); + tensor pool124 = ggml_pool_2d(m, act123, GGML_OP_POOL_AVG, 128, 64, 128, 64, 0, 0); + tensor conv125 = conv_2d(m["decode_head.acm_modules.0.input_redu_conv.conv"], act123, 1, 0); + tensor act126 = ggml_relu(m, conv125); + tensor conv127 = conv_2d(m["decode_head.acm_modules.0.pooled_redu_conv.conv"], pool124, 1, 0); + tensor act128 = ggml_relu(m, conv127); + tensor rs129 = ggml_reshape_3d(m, ggml_cont(m, act128), 1, 512, 1); + tensor perm130 = ggml_cont(m, ggml_permute(m, rs129, 1, 0, 2, 3)); + tensor pool131 = ggml_pool_2d(m, act126, GGML_OP_POOL_AVG, 128, 64, 128, 64, 0, 0); + tensor conv132 = conv_2d(m["decode_head.acm_modules.0.global_info.conv"], pool131, 1, 0); + tensor act133 = ggml_relu(m, conv132); + tensor up134 = ggml_interpolate(m, act133, 128, 64, 512, 1, GGML_SCALE_MODE_NEAREST) /* interpolate -> target ne */; + tensor add135 = ggml_add(m, act126, up134); + tensor conv136 = conv_2d(m["decode_head.acm_modules.0.gla"], add135, 1, 0); + tensor perm137 = ggml_cont(m, ggml_permute(m, conv136, 1, 2, 0, 3)); + tensor t139 = ggml_sigmoid(m, ggml_cont(m, ggml_reshape_3d(m, ggml_cont(m, perm137), 1, 8192, 1))); + tensor mm140 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, perm130, 1, 0, 2, 3)), t139); + tensor perm141 = ggml_cont(m, ggml_permute(m, mm140, 1, 0, 2, 3)); + tensor rs142 = ggml_reshape_4d(m, ggml_cont(m, perm141), 128, 64, 512, 1); + tensor conv143 = conv_2d(m["decode_head.acm_modules.0.residual_conv.conv"], rs142, 1, 0); + tensor act144 = ggml_relu(m, conv143); + tensor add145 = ggml_add(m, act144, act126); + tensor act146 = ggml_relu(m, add145); + tensor conv147 = conv_2d(m["decode_head.acm_modules.0.fusion_conv.conv"], act146, 1, 0); + tensor act148 = ggml_relu(m, conv147); + tensor pool149 = ggml_pool_2d(m, act123, GGML_OP_POOL_AVG, 64, 32, 64, 32, 0, 0); + tensor conv150 = conv_2d(m["decode_head.acm_modules.1.input_redu_conv.conv"], act123, 1, 0); + tensor act151 = ggml_relu(m, conv150); + tensor conv152 = conv_2d(m["decode_head.acm_modules.1.pooled_redu_conv.conv"], pool149, 1, 0); + tensor act153 = ggml_relu(m, conv152); + tensor rs154 = ggml_reshape_3d(m, ggml_cont(m, act153), 4, 512, 1); + tensor perm155 = ggml_cont(m, ggml_permute(m, rs154, 1, 0, 2, 3)); + tensor pool156 = ggml_pool_2d(m, act151, GGML_OP_POOL_AVG, 128, 64, 128, 64, 0, 0); + tensor conv157 = conv_2d(m["decode_head.acm_modules.1.global_info.conv"], pool156, 1, 0); + tensor act158 = ggml_relu(m, conv157); + tensor up159 = ggml_interpolate(m, act158, 128, 64, 512, 1, GGML_SCALE_MODE_NEAREST) /* interpolate -> target ne */; + tensor add160 = ggml_add(m, act151, up159); + tensor conv161 = conv_2d(m["decode_head.acm_modules.1.gla"], add160, 1, 0); + tensor perm162 = ggml_cont(m, ggml_permute(m, conv161, 1, 2, 0, 3)); + tensor t164 = ggml_sigmoid(m, ggml_cont(m, ggml_reshape_3d(m, ggml_cont(m, perm162), 4, 8192, 1))); + tensor mm165 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, perm155, 1, 0, 2, 3)), t164); + tensor perm166 = ggml_cont(m, ggml_permute(m, mm165, 1, 0, 2, 3)); + tensor rs167 = ggml_reshape_4d(m, ggml_cont(m, perm166), 128, 64, 512, 1); + tensor conv168 = conv_2d(m["decode_head.acm_modules.1.residual_conv.conv"], rs167, 1, 0); + tensor act169 = ggml_relu(m, conv168); + tensor add170 = ggml_add(m, act169, act151); + tensor act171 = ggml_relu(m, add170); + tensor conv172 = conv_2d(m["decode_head.acm_modules.1.fusion_conv.conv"], act171, 1, 0); + tensor act173 = ggml_relu(m, conv172); + tensor aph174 = ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act123, 43, act123->ne[1], act123->ne[2], act123->ne[3], act123->nb[1], act123->nb[2], act123->nb[3], (size_t)0*act123->nb[0]))), 0.023255813953488372f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act123, 44, act123->ne[1], act123->ne[2], act123->ne[3], act123->nb[1], act123->nb[2], act123->nb[3], (size_t)42*act123->nb[0]))), 0.022727272727272728f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act123, 43, act123->ne[1], act123->ne[2], act123->ne[3], act123->nb[1], act123->nb[2], act123->nb[3], (size_t)85*act123->nb[0]))), 0.023255813953488372f), 0); + tensor apt175 = ggml_cont(m, ggml_permute(m, aph174, 1, 0, 2, 3)); + tensor apr176 = ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt175, 22, apt175->ne[1], apt175->ne[2], apt175->ne[3], apt175->nb[1], apt175->nb[2], apt175->nb[3], (size_t)0*apt175->nb[0]))), 0.045454545454545456f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt175, 22, apt175->ne[1], apt175->ne[2], apt175->ne[3], apt175->nb[1], apt175->nb[2], apt175->nb[3], (size_t)21*apt175->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt175, 22, apt175->ne[1], apt175->ne[2], apt175->ne[3], apt175->nb[1], apt175->nb[2], apt175->nb[3], (size_t)42*apt175->nb[0]))), 0.045454545454545456f), 0); + tensor pool177 = ggml_cont(m, ggml_permute(m, apr176, 1, 0, 2, 3)); + tensor conv178 = conv_2d(m["decode_head.acm_modules.2.input_redu_conv.conv"], act123, 1, 0); + tensor act179 = ggml_relu(m, conv178); + tensor conv180 = conv_2d(m["decode_head.acm_modules.2.pooled_redu_conv.conv"], pool177, 1, 0); + tensor act181 = ggml_relu(m, conv180); + tensor rs182 = ggml_reshape_3d(m, ggml_cont(m, act181), 9, 512, 1); + tensor perm183 = ggml_cont(m, ggml_permute(m, rs182, 1, 0, 2, 3)); + tensor pool184 = ggml_pool_2d(m, act179, GGML_OP_POOL_AVG, 128, 64, 128, 64, 0, 0); + tensor conv185 = conv_2d(m["decode_head.acm_modules.2.global_info.conv"], pool184, 1, 0); + tensor act186 = ggml_relu(m, conv185); + tensor up187 = ggml_interpolate(m, act186, 128, 64, 512, 1, GGML_SCALE_MODE_NEAREST) /* interpolate -> target ne */; + tensor add188 = ggml_add(m, act179, up187); + tensor conv189 = conv_2d(m["decode_head.acm_modules.2.gla"], add188, 1, 0); + tensor perm190 = ggml_cont(m, ggml_permute(m, conv189, 1, 2, 0, 3)); + tensor t192 = ggml_sigmoid(m, ggml_cont(m, ggml_reshape_3d(m, ggml_cont(m, perm190), 9, 8192, 1))); + tensor mm193 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, perm183, 1, 0, 2, 3)), t192); + tensor perm194 = ggml_cont(m, ggml_permute(m, mm193, 1, 0, 2, 3)); + tensor rs195 = ggml_reshape_4d(m, ggml_cont(m, perm194), 128, 64, 512, 1); + tensor conv196 = conv_2d(m["decode_head.acm_modules.2.residual_conv.conv"], rs195, 1, 0); + tensor act197 = ggml_relu(m, conv196); + tensor add198 = ggml_add(m, act197, act179); + tensor act199 = ggml_relu(m, add198); + tensor conv200 = conv_2d(m["decode_head.acm_modules.2.fusion_conv.conv"], act199, 1, 0); + tensor act201 = ggml_relu(m, conv200); + tensor aph202 = ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act123, 22, act123->ne[1], act123->ne[2], act123->ne[3], act123->nb[1], act123->nb[2], act123->nb[3], (size_t)0*act123->nb[0]))), 0.045454545454545456f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act123, 22, act123->ne[1], act123->ne[2], act123->ne[3], act123->nb[1], act123->nb[2], act123->nb[3], (size_t)21*act123->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act123, 22, act123->ne[1], act123->ne[2], act123->ne[3], act123->nb[1], act123->nb[2], act123->nb[3], (size_t)42*act123->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act123, 22, act123->ne[1], act123->ne[2], act123->ne[3], act123->nb[1], act123->nb[2], act123->nb[3], (size_t)64*act123->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act123, 22, act123->ne[1], act123->ne[2], act123->ne[3], act123->nb[1], act123->nb[2], act123->nb[3], (size_t)85*act123->nb[0]))), 0.045454545454545456f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, act123, 22, act123->ne[1], act123->ne[2], act123->ne[3], act123->nb[1], act123->nb[2], act123->nb[3], (size_t)106*act123->nb[0]))), 0.045454545454545456f), 0); + tensor apt203 = ggml_cont(m, ggml_permute(m, aph202, 1, 0, 2, 3)); + tensor apr204 = ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt203, 11, apt203->ne[1], apt203->ne[2], apt203->ne[3], apt203->nb[1], apt203->nb[2], apt203->nb[3], (size_t)0*apt203->nb[0]))), 0.09090909090909091f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt203, 12, apt203->ne[1], apt203->ne[2], apt203->ne[3], apt203->nb[1], apt203->nb[2], apt203->nb[3], (size_t)10*apt203->nb[0]))), 0.08333333333333333f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt203, 11, apt203->ne[1], apt203->ne[2], apt203->ne[3], apt203->nb[1], apt203->nb[2], apt203->nb[3], (size_t)21*apt203->nb[0]))), 0.09090909090909091f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt203, 11, apt203->ne[1], apt203->ne[2], apt203->ne[3], apt203->nb[1], apt203->nb[2], apt203->nb[3], (size_t)32*apt203->nb[0]))), 0.09090909090909091f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt203, 12, apt203->ne[1], apt203->ne[2], apt203->ne[3], apt203->nb[1], apt203->nb[2], apt203->nb[3], (size_t)42*apt203->nb[0]))), 0.08333333333333333f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt203, 11, apt203->ne[1], apt203->ne[2], apt203->ne[3], apt203->nb[1], apt203->nb[2], apt203->nb[3], (size_t)53*apt203->nb[0]))), 0.09090909090909091f), 0); + tensor pool205 = ggml_cont(m, ggml_permute(m, apr204, 1, 0, 2, 3)); + tensor conv206 = conv_2d(m["decode_head.acm_modules.3.input_redu_conv.conv"], act123, 1, 0); + tensor act207 = ggml_relu(m, conv206); + tensor conv208 = conv_2d(m["decode_head.acm_modules.3.pooled_redu_conv.conv"], pool205, 1, 0); + tensor act209 = ggml_relu(m, conv208); + tensor rs210 = ggml_reshape_3d(m, ggml_cont(m, act209), 36, 512, 1); + tensor perm211 = ggml_cont(m, ggml_permute(m, rs210, 1, 0, 2, 3)); + tensor pool212 = ggml_pool_2d(m, act207, GGML_OP_POOL_AVG, 128, 64, 128, 64, 0, 0); + tensor conv213 = conv_2d(m["decode_head.acm_modules.3.global_info.conv"], pool212, 1, 0); + tensor act214 = ggml_relu(m, conv213); + tensor up215 = ggml_interpolate(m, act214, 128, 64, 512, 1, GGML_SCALE_MODE_NEAREST) /* interpolate -> target ne */; + tensor add216 = ggml_add(m, act207, up215); + tensor conv217 = conv_2d(m["decode_head.acm_modules.3.gla"], add216, 1, 0); + tensor perm218 = ggml_cont(m, ggml_permute(m, conv217, 1, 2, 0, 3)); + tensor t220 = ggml_sigmoid(m, ggml_cont(m, ggml_reshape_3d(m, ggml_cont(m, perm218), 36, 8192, 1))); + tensor mm221 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, perm211, 1, 0, 2, 3)), t220); + tensor perm222 = ggml_cont(m, ggml_permute(m, mm221, 1, 0, 2, 3)); + tensor rs223 = ggml_reshape_4d(m, ggml_cont(m, perm222), 128, 64, 512, 1); + tensor conv224 = conv_2d(m["decode_head.acm_modules.3.residual_conv.conv"], rs223, 1, 0); + tensor act225 = ggml_relu(m, conv224); + tensor add226 = ggml_add(m, act225, act207); + tensor act227 = ggml_relu(m, add226); + tensor conv228 = conv_2d(m["decode_head.acm_modules.3.fusion_conv.conv"], act227, 1, 0); + tensor act229 = ggml_relu(m, conv228); + tensor cat230 = ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, act123, act148, 2), act173, 2), act201, 2), act229, 2); + tensor conv231 = conv_2d(m["decode_head.bottleneck.conv"], cat230, 1, 1); + tensor act232 = ggml_relu(m, conv231); + tensor conv233 = conv_2d(m["decode_head.conv_seg"], act232, 1, 0); + + tensor out_0 = compute_graph_output(m, contiguous_2d_to_cwhn(m, conv233), "out_0"); + return out_0; +} + +FamApcnet_params FamApcnet_detect_params(model_file const& f) { + FamApcnet_params p{}; + // GGUF general.architecture 검증 (모델은 정적 unroll 이라 추가 하이퍼파라미터 없음). + if (std::string_view arch = f.arch(); arch != "famapcnet") { + throw except( + "Architecture expected to be 'famapcnet', but was '{}' ({})", + arch, f.path); + } + return p; +} + +} // namespace visp diff --git a/src/visp/arch/FamApcnet.h b/src/visp/arch/FamApcnet.h new file mode 100644 index 0000000..50046d0 --- /dev/null +++ b/src/visp/arch/FamApcnet.h @@ -0,0 +1,14 @@ +// GENERATED BY SuperGate GTX Compiler (ggml/vision.cpp backend), DO NOT EDIT! +#pragma once +#include "visp/ml.h" + +namespace visp { + +struct FamApcnet_params { + // TODO(ggml): 모델 하이퍼파라미터 (block_count 등) +}; + +tensor FamApcnet_forward(model_ref m, tensor x, FamApcnet_params const& p); +FamApcnet_params FamApcnet_detect_params(model_file const& f); + +} // namespace visp diff --git a/src/visp/arch/FamBeit.cpp b/src/visp/arch/FamBeit.cpp new file mode 100644 index 0000000..2638f25 --- /dev/null +++ b/src/visp/arch/FamBeit.cpp @@ -0,0 +1,491 @@ +// GENERATED BY SuperGate GTX Compiler (ggml/vision.cpp backend), DO NOT EDIT! +#include "visp/arch/FamBeit.h" +#include "visp/ml.h" +#include "visp/nn.h" +#include "visp/vision.h" +#include "util/string.h" +#include +#include // clamp 의 한쪽 경계가 없으면 ±INFINITY 를 낸다 + +#include + +namespace visp { + +tensor FamBeit_forward(model_ref m, tensor x, FamBeit_params const& p) { + (void)p; + // 입력 레이아웃 변환 (TODO: cwhn/whcn 자동 판별) + x = cwhn_to_contiguous_2d(m, x); + + tensor const1 = ggml_arange(m, 0.125f, 0.625f, 1.0f) /* const 0.125 */; + tensor const2 = ggml_arange(m, 0.125f, 0.625f, 1.0f) /* const 0.125 */; + tensor const3 = ggml_arange(m, 0.125f, 0.625f, 1.0f) /* const 0.125 */; + tensor const4 = ggml_arange(m, 0.125f, 0.625f, 1.0f) /* const 0.125 */; + tensor const5 = ggml_arange(m, 0.125f, 0.625f, 1.0f) /* const 0.125 */; + tensor const6 = ggml_arange(m, 0.125f, 0.625f, 1.0f) /* const 0.125 */; + tensor const7 = ggml_arange(m, 0.125f, 0.625f, 1.0f) /* const 0.125 */; + tensor const8 = ggml_arange(m, 0.125f, 0.625f, 1.0f) /* const 0.125 */; + tensor const9 = ggml_arange(m, 0.125f, 0.625f, 1.0f) /* const 0.125 */; + tensor const10 = ggml_arange(m, 0.125f, 0.625f, 1.0f) /* const 0.125 */; + tensor const11 = ggml_arange(m, 0.125f, 0.625f, 1.0f) /* const 0.125 */; + tensor const12 = ggml_arange(m, 0.125f, 0.625f, 1.0f) /* const 0.125 */; + tensor conv13 = conv_2d(m["backbone.patch_embed.projection"], x, 16, 0); + tensor flat14 = ggml_reshape_3d(m, ggml_cont(m, conv13), 1600, 768, 1); + tensor t15 = ggml_cont(m, ggml_permute(m, flat14, 1, 0, 2, 3)); + tensor exp16 = ggml_repeat(m, m.find("backbone.cls_token"), ggml_new_tensor_3d(m, GGML_TYPE_F32, 768, 1, 1)); + tensor cat17 = ggml_concat(m, exp16, t15, 1); + tensor ln18 = layer_norm(m["backbone.layers.0.ln1"], cat17); + tensor like19 = ggml_fill(m, ggml_new_tensor_1d(m, GGML_TYPE_F32, 768), 0.0f); + tensor cat20 = ggml_concat(m, ggml_concat(m, m.find("backbone.layers.0.attn.q_bias"), like19, 0), m.find("backbone.layers.0.attn.v_bias"), 0); + tensor fcd21 = ggml_add(m, ggml_mul_mat(m, m.find("backbone.layers.0.attn.qkv.weight"), ln18), cat20); + tensor rs22 = ggml_reshape_4d(m, ggml_cont(m, fcd21), 64, 12, 3, 1601); + tensor perm23 = ggml_cont(m, ggml_permute(m, rs22, 0, 2, 3, 1)); + tensor sel24 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm23), 1229568, (size_t)0*perm23->nb[0])), 64, 1601, 12, 1); + tensor sel25 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm23), 1229568, (size_t)1229568*perm23->nb[0])), 64, 1601, 12, 1); + tensor sel26 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm23), 1229568, (size_t)2459136*perm23->nb[0])), 64, 1601, 12, 1); + tensor t27 = ggml_mul(m, as_f32(m, sel24), as_f32(m, const1)); + tensor t28 = ggml_cont(m, ggml_permute(m, sel25, 1, 0, 2, 3)); + tensor mm29 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, t28, 1, 0, 2, 3)), t27); + tensor rs30 = ggml_reshape_1d(m, ggml_cont(m, m.find("backbone.layers.0.attn.relative_position_index")), 2563201); + tensor gat31 = ggml_get_rows(m, m.find("backbone.layers.0.attn.relative_position_bias_table"), ggml_cast(m, ggml_cont(m, rs30), GGML_TYPE_I32)) /* index: dim 미상 → torch dim 0 (get_rows 행 선택) 가정 */; + tensor rs32 = ggml_reshape_3d(m, ggml_cont(m, gat31), 12, 1601, 1601); + tensor perm33 = ggml_cont(m, ggml_permute(m, rs32, 2, 0, 1, 3)); + tensor uns34 = ggml_reshape_4d(m, ggml_cont(m, perm33), 1601, 1601, 12, 1); + tensor add35 = ggml_add(m, mm29, uns34); + tensor sm36 = ggml_soft_max(m, add35); + tensor mm37 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, sel26, 1, 0, 2, 3)), sm36); + tensor t38 = ggml_cont(m, ggml_permute(m, mm37, 0, 2, 1, 3)); + tensor rs39 = ggml_reshape_3d(m, ggml_cont(m, t38), 768, 1601, 1); + tensor fc40 = linear(m["backbone.layers.0.attn.proj"], rs39); + tensor t41 = ggml_mul(m, as_f32(m, fc40), as_f32(m, m.find("backbone.layers.0.gamma_1"))); + tensor add42 = ggml_add(m, cat17, t41); + tensor ln43 = layer_norm(m["backbone.layers.0.ln2"], add42); + tensor fc44 = linear(m["backbone.layers.0.ffn.layers.0.0"], ln43); + tensor act45 = ggml_gelu(m, fc44); + tensor fc46 = linear(m["backbone.layers.0.ffn.layers.1"], act45); + tensor t47 = ggml_mul(m, as_f32(m, fc46), as_f32(m, m.find("backbone.layers.0.gamma_2"))); + tensor add48 = ggml_add(m, add42, t47); + tensor ln49 = layer_norm(m["backbone.layers.1.ln1"], add48); + tensor like50 = ggml_fill(m, ggml_new_tensor_1d(m, GGML_TYPE_F32, 768), 0.0f); + tensor cat51 = ggml_concat(m, ggml_concat(m, m.find("backbone.layers.1.attn.q_bias"), like50, 0), m.find("backbone.layers.1.attn.v_bias"), 0); + tensor fcd52 = ggml_add(m, ggml_mul_mat(m, m.find("backbone.layers.1.attn.qkv.weight"), ln49), cat51); + tensor rs53 = ggml_reshape_4d(m, ggml_cont(m, fcd52), 64, 12, 3, 1601); + tensor perm54 = ggml_cont(m, ggml_permute(m, rs53, 0, 2, 3, 1)); + tensor sel55 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm54), 1229568, (size_t)0*perm54->nb[0])), 64, 1601, 12, 1); + tensor sel56 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm54), 1229568, (size_t)1229568*perm54->nb[0])), 64, 1601, 12, 1); + tensor sel57 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm54), 1229568, (size_t)2459136*perm54->nb[0])), 64, 1601, 12, 1); + tensor t58 = ggml_mul(m, as_f32(m, sel55), as_f32(m, const2)); + tensor t59 = ggml_cont(m, ggml_permute(m, sel56, 1, 0, 2, 3)); + tensor mm60 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, t59, 1, 0, 2, 3)), t58); + tensor rs61 = ggml_reshape_1d(m, ggml_cont(m, m.find("backbone.layers.1.attn.relative_position_index")), 2563201); + tensor gat62 = ggml_get_rows(m, m.find("backbone.layers.1.attn.relative_position_bias_table"), ggml_cast(m, ggml_cont(m, rs61), GGML_TYPE_I32)) /* index: dim 미상 → torch dim 0 (get_rows 행 선택) 가정 */; + tensor rs63 = ggml_reshape_3d(m, ggml_cont(m, gat62), 12, 1601, 1601); + tensor perm64 = ggml_cont(m, ggml_permute(m, rs63, 2, 0, 1, 3)); + tensor uns65 = ggml_reshape_4d(m, ggml_cont(m, perm64), 1601, 1601, 12, 1); + tensor add66 = ggml_add(m, mm60, uns65); + tensor sm67 = ggml_soft_max(m, add66); + tensor mm68 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, sel57, 1, 0, 2, 3)), sm67); + tensor t69 = ggml_cont(m, ggml_permute(m, mm68, 0, 2, 1, 3)); + tensor rs70 = ggml_reshape_3d(m, ggml_cont(m, t69), 768, 1601, 1); + tensor fc71 = linear(m["backbone.layers.1.attn.proj"], rs70); + tensor t72 = ggml_mul(m, as_f32(m, fc71), as_f32(m, m.find("backbone.layers.1.gamma_1"))); + tensor add73 = ggml_add(m, add48, t72); + tensor ln74 = layer_norm(m["backbone.layers.1.ln2"], add73); + tensor fc75 = linear(m["backbone.layers.1.ffn.layers.0.0"], ln74); + tensor act76 = ggml_gelu(m, fc75); + tensor fc77 = linear(m["backbone.layers.1.ffn.layers.1"], act76); + tensor t78 = ggml_mul(m, as_f32(m, fc77), as_f32(m, m.find("backbone.layers.1.gamma_2"))); + tensor add79 = ggml_add(m, add73, t78); + tensor ln80 = layer_norm(m["backbone.layers.2.ln1"], add79); + tensor like81 = ggml_fill(m, ggml_new_tensor_1d(m, GGML_TYPE_F32, 768), 0.0f); + tensor cat82 = ggml_concat(m, ggml_concat(m, m.find("backbone.layers.2.attn.q_bias"), like81, 0), m.find("backbone.layers.2.attn.v_bias"), 0); + tensor fcd83 = ggml_add(m, ggml_mul_mat(m, m.find("backbone.layers.2.attn.qkv.weight"), ln80), cat82); + tensor rs84 = ggml_reshape_4d(m, ggml_cont(m, fcd83), 64, 12, 3, 1601); + tensor perm85 = ggml_cont(m, ggml_permute(m, rs84, 0, 2, 3, 1)); + tensor sel86 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm85), 1229568, (size_t)0*perm85->nb[0])), 64, 1601, 12, 1); + tensor sel87 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm85), 1229568, (size_t)1229568*perm85->nb[0])), 64, 1601, 12, 1); + tensor sel88 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm85), 1229568, (size_t)2459136*perm85->nb[0])), 64, 1601, 12, 1); + tensor t89 = ggml_mul(m, as_f32(m, sel86), as_f32(m, const3)); + tensor t90 = ggml_cont(m, ggml_permute(m, sel87, 1, 0, 2, 3)); + tensor mm91 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, t90, 1, 0, 2, 3)), t89); + tensor rs92 = ggml_reshape_1d(m, ggml_cont(m, m.find("backbone.layers.2.attn.relative_position_index")), 2563201); + tensor gat93 = ggml_get_rows(m, m.find("backbone.layers.2.attn.relative_position_bias_table"), ggml_cast(m, ggml_cont(m, rs92), GGML_TYPE_I32)) /* index: dim 미상 → torch dim 0 (get_rows 행 선택) 가정 */; + tensor rs94 = ggml_reshape_3d(m, ggml_cont(m, gat93), 12, 1601, 1601); + tensor perm95 = ggml_cont(m, ggml_permute(m, rs94, 2, 0, 1, 3)); + tensor uns96 = ggml_reshape_4d(m, ggml_cont(m, perm95), 1601, 1601, 12, 1); + tensor add97 = ggml_add(m, mm91, uns96); + tensor sm98 = ggml_soft_max(m, add97); + tensor mm99 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, sel88, 1, 0, 2, 3)), sm98); + tensor t100 = ggml_cont(m, ggml_permute(m, mm99, 0, 2, 1, 3)); + tensor rs101 = ggml_reshape_3d(m, ggml_cont(m, t100), 768, 1601, 1); + tensor fc102 = linear(m["backbone.layers.2.attn.proj"], rs101); + tensor t103 = ggml_mul(m, as_f32(m, fc102), as_f32(m, m.find("backbone.layers.2.gamma_1"))); + tensor add104 = ggml_add(m, add79, t103); + tensor ln105 = layer_norm(m["backbone.layers.2.ln2"], add104); + tensor fc106 = linear(m["backbone.layers.2.ffn.layers.0.0"], ln105); + tensor act107 = ggml_gelu(m, fc106); + tensor fc108 = linear(m["backbone.layers.2.ffn.layers.1"], act107); + tensor t109 = ggml_mul(m, as_f32(m, fc108), as_f32(m, m.find("backbone.layers.2.gamma_2"))); + tensor add110 = ggml_add(m, add104, t109); + tensor ln111 = layer_norm(m["backbone.layers.3.ln1"], add110); + tensor like112 = ggml_fill(m, ggml_new_tensor_1d(m, GGML_TYPE_F32, 768), 0.0f); + tensor cat113 = ggml_concat(m, ggml_concat(m, m.find("backbone.layers.3.attn.q_bias"), like112, 0), m.find("backbone.layers.3.attn.v_bias"), 0); + tensor fcd114 = ggml_add(m, ggml_mul_mat(m, m.find("backbone.layers.3.attn.qkv.weight"), ln111), cat113); + tensor rs115 = ggml_reshape_4d(m, ggml_cont(m, fcd114), 64, 12, 3, 1601); + tensor perm116 = ggml_cont(m, ggml_permute(m, rs115, 0, 2, 3, 1)); + tensor sel117 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm116), 1229568, (size_t)0*perm116->nb[0])), 64, 1601, 12, 1); + tensor sel118 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm116), 1229568, (size_t)1229568*perm116->nb[0])), 64, 1601, 12, 1); + tensor sel119 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm116), 1229568, (size_t)2459136*perm116->nb[0])), 64, 1601, 12, 1); + tensor t120 = ggml_mul(m, as_f32(m, sel117), as_f32(m, const4)); + tensor t121 = ggml_cont(m, ggml_permute(m, sel118, 1, 0, 2, 3)); + tensor mm122 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, t121, 1, 0, 2, 3)), t120); + tensor rs123 = ggml_reshape_1d(m, ggml_cont(m, m.find("backbone.layers.3.attn.relative_position_index")), 2563201); + tensor gat124 = ggml_get_rows(m, m.find("backbone.layers.3.attn.relative_position_bias_table"), ggml_cast(m, ggml_cont(m, rs123), GGML_TYPE_I32)) /* index: dim 미상 → torch dim 0 (get_rows 행 선택) 가정 */; + tensor rs125 = ggml_reshape_3d(m, ggml_cont(m, gat124), 12, 1601, 1601); + tensor perm126 = ggml_cont(m, ggml_permute(m, rs125, 2, 0, 1, 3)); + tensor uns127 = ggml_reshape_4d(m, ggml_cont(m, perm126), 1601, 1601, 12, 1); + tensor add128 = ggml_add(m, mm122, uns127); + tensor sm129 = ggml_soft_max(m, add128); + tensor mm130 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, sel119, 1, 0, 2, 3)), sm129); + tensor t131 = ggml_cont(m, ggml_permute(m, mm130, 0, 2, 1, 3)); + tensor rs132 = ggml_reshape_3d(m, ggml_cont(m, t131), 768, 1601, 1); + tensor fc133 = linear(m["backbone.layers.3.attn.proj"], rs132); + tensor t134 = ggml_mul(m, as_f32(m, fc133), as_f32(m, m.find("backbone.layers.3.gamma_1"))); + tensor add135 = ggml_add(m, add110, t134); + tensor ln136 = layer_norm(m["backbone.layers.3.ln2"], add135); + tensor fc137 = linear(m["backbone.layers.3.ffn.layers.0.0"], ln136); + tensor act138 = ggml_gelu(m, fc137); + tensor fc139 = linear(m["backbone.layers.3.ffn.layers.1"], act138); + tensor t140 = ggml_mul(m, as_f32(m, fc139), as_f32(m, m.find("backbone.layers.3.gamma_2"))); + tensor add141 = ggml_add(m, add135, t140); + tensor sl142 = ggml_view_3d(m, add141, 768, 1600, 1, add141->nb[1], add141->nb[2], 1*add141->nb[1]); + tensor rs143 = ggml_reshape_4d(m, ggml_cont(m, sl142), 768, 40, 40, 1); + tensor perm144 = ggml_cont(m, ggml_permute(m, rs143, 2, 0, 1, 3)); + tensor ln145 = layer_norm(m["backbone.layers.4.ln1"], add141); + tensor like146 = ggml_fill(m, ggml_new_tensor_1d(m, GGML_TYPE_F32, 768), 0.0f); + tensor cat147 = ggml_concat(m, ggml_concat(m, m.find("backbone.layers.4.attn.q_bias"), like146, 0), m.find("backbone.layers.4.attn.v_bias"), 0); + tensor fcd148 = ggml_add(m, ggml_mul_mat(m, m.find("backbone.layers.4.attn.qkv.weight"), ln145), cat147); + tensor rs149 = ggml_reshape_4d(m, ggml_cont(m, fcd148), 64, 12, 3, 1601); + tensor perm150 = ggml_cont(m, ggml_permute(m, rs149, 0, 2, 3, 1)); + tensor sel151 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm150), 1229568, (size_t)0*perm150->nb[0])), 64, 1601, 12, 1); + tensor sel152 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm150), 1229568, (size_t)1229568*perm150->nb[0])), 64, 1601, 12, 1); + tensor sel153 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm150), 1229568, (size_t)2459136*perm150->nb[0])), 64, 1601, 12, 1); + tensor t154 = ggml_mul(m, as_f32(m, sel151), as_f32(m, const5)); + tensor t155 = ggml_cont(m, ggml_permute(m, sel152, 1, 0, 2, 3)); + tensor mm156 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, t155, 1, 0, 2, 3)), t154); + tensor rs157 = ggml_reshape_1d(m, ggml_cont(m, m.find("backbone.layers.4.attn.relative_position_index")), 2563201); + tensor gat158 = ggml_get_rows(m, m.find("backbone.layers.4.attn.relative_position_bias_table"), ggml_cast(m, ggml_cont(m, rs157), GGML_TYPE_I32)) /* index: dim 미상 → torch dim 0 (get_rows 행 선택) 가정 */; + tensor rs159 = ggml_reshape_3d(m, ggml_cont(m, gat158), 12, 1601, 1601); + tensor perm160 = ggml_cont(m, ggml_permute(m, rs159, 2, 0, 1, 3)); + tensor uns161 = ggml_reshape_4d(m, ggml_cont(m, perm160), 1601, 1601, 12, 1); + tensor add162 = ggml_add(m, mm156, uns161); + tensor sm163 = ggml_soft_max(m, add162); + tensor mm164 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, sel153, 1, 0, 2, 3)), sm163); + tensor t165 = ggml_cont(m, ggml_permute(m, mm164, 0, 2, 1, 3)); + tensor rs166 = ggml_reshape_3d(m, ggml_cont(m, t165), 768, 1601, 1); + tensor fc167 = linear(m["backbone.layers.4.attn.proj"], rs166); + tensor t168 = ggml_mul(m, as_f32(m, fc167), as_f32(m, m.find("backbone.layers.4.gamma_1"))); + tensor add169 = ggml_add(m, add141, t168); + tensor ln170 = layer_norm(m["backbone.layers.4.ln2"], add169); + tensor fc171 = linear(m["backbone.layers.4.ffn.layers.0.0"], ln170); + tensor act172 = ggml_gelu(m, fc171); + tensor fc173 = linear(m["backbone.layers.4.ffn.layers.1"], act172); + tensor t174 = ggml_mul(m, as_f32(m, fc173), as_f32(m, m.find("backbone.layers.4.gamma_2"))); + tensor add175 = ggml_add(m, add169, t174); + tensor ln176 = layer_norm(m["backbone.layers.5.ln1"], add175); + tensor like177 = ggml_fill(m, ggml_new_tensor_1d(m, GGML_TYPE_F32, 768), 0.0f); + tensor cat178 = ggml_concat(m, ggml_concat(m, m.find("backbone.layers.5.attn.q_bias"), like177, 0), m.find("backbone.layers.5.attn.v_bias"), 0); + tensor fcd179 = ggml_add(m, ggml_mul_mat(m, m.find("backbone.layers.5.attn.qkv.weight"), ln176), cat178); + tensor rs180 = ggml_reshape_4d(m, ggml_cont(m, fcd179), 64, 12, 3, 1601); + tensor perm181 = ggml_cont(m, ggml_permute(m, rs180, 0, 2, 3, 1)); + tensor sel182 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm181), 1229568, (size_t)0*perm181->nb[0])), 64, 1601, 12, 1); + tensor sel183 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm181), 1229568, (size_t)1229568*perm181->nb[0])), 64, 1601, 12, 1); + tensor sel184 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm181), 1229568, (size_t)2459136*perm181->nb[0])), 64, 1601, 12, 1); + tensor t185 = ggml_mul(m, as_f32(m, sel182), as_f32(m, const6)); + tensor t186 = ggml_cont(m, ggml_permute(m, sel183, 1, 0, 2, 3)); + tensor mm187 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, t186, 1, 0, 2, 3)), t185); + tensor rs188 = ggml_reshape_1d(m, ggml_cont(m, m.find("backbone.layers.5.attn.relative_position_index")), 2563201); + tensor gat189 = ggml_get_rows(m, m.find("backbone.layers.5.attn.relative_position_bias_table"), ggml_cast(m, ggml_cont(m, rs188), GGML_TYPE_I32)) /* index: dim 미상 → torch dim 0 (get_rows 행 선택) 가정 */; + tensor rs190 = ggml_reshape_3d(m, ggml_cont(m, gat189), 12, 1601, 1601); + tensor perm191 = ggml_cont(m, ggml_permute(m, rs190, 2, 0, 1, 3)); + tensor uns192 = ggml_reshape_4d(m, ggml_cont(m, perm191), 1601, 1601, 12, 1); + tensor add193 = ggml_add(m, mm187, uns192); + tensor sm194 = ggml_soft_max(m, add193); + tensor mm195 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, sel184, 1, 0, 2, 3)), sm194); + tensor t196 = ggml_cont(m, ggml_permute(m, mm195, 0, 2, 1, 3)); + tensor rs197 = ggml_reshape_3d(m, ggml_cont(m, t196), 768, 1601, 1); + tensor fc198 = linear(m["backbone.layers.5.attn.proj"], rs197); + tensor t199 = ggml_mul(m, as_f32(m, fc198), as_f32(m, m.find("backbone.layers.5.gamma_1"))); + tensor add200 = ggml_add(m, add175, t199); + tensor ln201 = layer_norm(m["backbone.layers.5.ln2"], add200); + tensor fc202 = linear(m["backbone.layers.5.ffn.layers.0.0"], ln201); + tensor act203 = ggml_gelu(m, fc202); + tensor fc204 = linear(m["backbone.layers.5.ffn.layers.1"], act203); + tensor t205 = ggml_mul(m, as_f32(m, fc204), as_f32(m, m.find("backbone.layers.5.gamma_2"))); + tensor add206 = ggml_add(m, add200, t205); + tensor sl207 = ggml_view_3d(m, add206, 768, 1600, 1, add206->nb[1], add206->nb[2], 1*add206->nb[1]); + tensor rs208 = ggml_reshape_4d(m, ggml_cont(m, sl207), 768, 40, 40, 1); + tensor perm209 = ggml_cont(m, ggml_permute(m, rs208, 2, 0, 1, 3)); + tensor ln210 = layer_norm(m["backbone.layers.6.ln1"], add206); + tensor like211 = ggml_fill(m, ggml_new_tensor_1d(m, GGML_TYPE_F32, 768), 0.0f); + tensor cat212 = ggml_concat(m, ggml_concat(m, m.find("backbone.layers.6.attn.q_bias"), like211, 0), m.find("backbone.layers.6.attn.v_bias"), 0); + tensor fcd213 = ggml_add(m, ggml_mul_mat(m, m.find("backbone.layers.6.attn.qkv.weight"), ln210), cat212); + tensor rs214 = ggml_reshape_4d(m, ggml_cont(m, fcd213), 64, 12, 3, 1601); + tensor perm215 = ggml_cont(m, ggml_permute(m, rs214, 0, 2, 3, 1)); + tensor sel216 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm215), 1229568, (size_t)0*perm215->nb[0])), 64, 1601, 12, 1); + tensor sel217 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm215), 1229568, (size_t)1229568*perm215->nb[0])), 64, 1601, 12, 1); + tensor sel218 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm215), 1229568, (size_t)2459136*perm215->nb[0])), 64, 1601, 12, 1); + tensor t219 = ggml_mul(m, as_f32(m, sel216), as_f32(m, const7)); + tensor t220 = ggml_cont(m, ggml_permute(m, sel217, 1, 0, 2, 3)); + tensor mm221 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, t220, 1, 0, 2, 3)), t219); + tensor rs222 = ggml_reshape_1d(m, ggml_cont(m, m.find("backbone.layers.6.attn.relative_position_index")), 2563201); + tensor gat223 = ggml_get_rows(m, m.find("backbone.layers.6.attn.relative_position_bias_table"), ggml_cast(m, ggml_cont(m, rs222), GGML_TYPE_I32)) /* index: dim 미상 → torch dim 0 (get_rows 행 선택) 가정 */; + tensor rs224 = ggml_reshape_3d(m, ggml_cont(m, gat223), 12, 1601, 1601); + tensor perm225 = ggml_cont(m, ggml_permute(m, rs224, 2, 0, 1, 3)); + tensor uns226 = ggml_reshape_4d(m, ggml_cont(m, perm225), 1601, 1601, 12, 1); + tensor add227 = ggml_add(m, mm221, uns226); + tensor sm228 = ggml_soft_max(m, add227); + tensor mm229 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, sel218, 1, 0, 2, 3)), sm228); + tensor t230 = ggml_cont(m, ggml_permute(m, mm229, 0, 2, 1, 3)); + tensor rs231 = ggml_reshape_3d(m, ggml_cont(m, t230), 768, 1601, 1); + tensor fc232 = linear(m["backbone.layers.6.attn.proj"], rs231); + tensor t233 = ggml_mul(m, as_f32(m, fc232), as_f32(m, m.find("backbone.layers.6.gamma_1"))); + tensor add234 = ggml_add(m, add206, t233); + tensor ln235 = layer_norm(m["backbone.layers.6.ln2"], add234); + tensor fc236 = linear(m["backbone.layers.6.ffn.layers.0.0"], ln235); + tensor act237 = ggml_gelu(m, fc236); + tensor fc238 = linear(m["backbone.layers.6.ffn.layers.1"], act237); + tensor t239 = ggml_mul(m, as_f32(m, fc238), as_f32(m, m.find("backbone.layers.6.gamma_2"))); + tensor add240 = ggml_add(m, add234, t239); + tensor ln241 = layer_norm(m["backbone.layers.7.ln1"], add240); + tensor like242 = ggml_fill(m, ggml_new_tensor_1d(m, GGML_TYPE_F32, 768), 0.0f); + tensor cat243 = ggml_concat(m, ggml_concat(m, m.find("backbone.layers.7.attn.q_bias"), like242, 0), m.find("backbone.layers.7.attn.v_bias"), 0); + tensor fcd244 = ggml_add(m, ggml_mul_mat(m, m.find("backbone.layers.7.attn.qkv.weight"), ln241), cat243); + tensor rs245 = ggml_reshape_4d(m, ggml_cont(m, fcd244), 64, 12, 3, 1601); + tensor perm246 = ggml_cont(m, ggml_permute(m, rs245, 0, 2, 3, 1)); + tensor sel247 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm246), 1229568, (size_t)0*perm246->nb[0])), 64, 1601, 12, 1); + tensor sel248 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm246), 1229568, (size_t)1229568*perm246->nb[0])), 64, 1601, 12, 1); + tensor sel249 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm246), 1229568, (size_t)2459136*perm246->nb[0])), 64, 1601, 12, 1); + tensor t250 = ggml_mul(m, as_f32(m, sel247), as_f32(m, const8)); + tensor t251 = ggml_cont(m, ggml_permute(m, sel248, 1, 0, 2, 3)); + tensor mm252 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, t251, 1, 0, 2, 3)), t250); + tensor rs253 = ggml_reshape_1d(m, ggml_cont(m, m.find("backbone.layers.7.attn.relative_position_index")), 2563201); + tensor gat254 = ggml_get_rows(m, m.find("backbone.layers.7.attn.relative_position_bias_table"), ggml_cast(m, ggml_cont(m, rs253), GGML_TYPE_I32)) /* index: dim 미상 → torch dim 0 (get_rows 행 선택) 가정 */; + tensor rs255 = ggml_reshape_3d(m, ggml_cont(m, gat254), 12, 1601, 1601); + tensor perm256 = ggml_cont(m, ggml_permute(m, rs255, 2, 0, 1, 3)); + tensor uns257 = ggml_reshape_4d(m, ggml_cont(m, perm256), 1601, 1601, 12, 1); + tensor add258 = ggml_add(m, mm252, uns257); + tensor sm259 = ggml_soft_max(m, add258); + tensor mm260 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, sel249, 1, 0, 2, 3)), sm259); + tensor t261 = ggml_cont(m, ggml_permute(m, mm260, 0, 2, 1, 3)); + tensor rs262 = ggml_reshape_3d(m, ggml_cont(m, t261), 768, 1601, 1); + tensor fc263 = linear(m["backbone.layers.7.attn.proj"], rs262); + tensor t264 = ggml_mul(m, as_f32(m, fc263), as_f32(m, m.find("backbone.layers.7.gamma_1"))); + tensor add265 = ggml_add(m, add240, t264); + tensor ln266 = layer_norm(m["backbone.layers.7.ln2"], add265); + tensor fc267 = linear(m["backbone.layers.7.ffn.layers.0.0"], ln266); + tensor act268 = ggml_gelu(m, fc267); + tensor fc269 = linear(m["backbone.layers.7.ffn.layers.1"], act268); + tensor t270 = ggml_mul(m, as_f32(m, fc269), as_f32(m, m.find("backbone.layers.7.gamma_2"))); + tensor add271 = ggml_add(m, add265, t270); + tensor sl272 = ggml_view_3d(m, add271, 768, 1600, 1, add271->nb[1], add271->nb[2], 1*add271->nb[1]); + tensor rs273 = ggml_reshape_4d(m, ggml_cont(m, sl272), 768, 40, 40, 1); + tensor perm274 = ggml_cont(m, ggml_permute(m, rs273, 2, 0, 1, 3)); + tensor ln275 = layer_norm(m["backbone.layers.8.ln1"], add271); + tensor like276 = ggml_fill(m, ggml_new_tensor_1d(m, GGML_TYPE_F32, 768), 0.0f); + tensor cat277 = ggml_concat(m, ggml_concat(m, m.find("backbone.layers.8.attn.q_bias"), like276, 0), m.find("backbone.layers.8.attn.v_bias"), 0); + tensor fcd278 = ggml_add(m, ggml_mul_mat(m, m.find("backbone.layers.8.attn.qkv.weight"), ln275), cat277); + tensor rs279 = ggml_reshape_4d(m, ggml_cont(m, fcd278), 64, 12, 3, 1601); + tensor perm280 = ggml_cont(m, ggml_permute(m, rs279, 0, 2, 3, 1)); + tensor sel281 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm280), 1229568, (size_t)0*perm280->nb[0])), 64, 1601, 12, 1); + tensor sel282 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm280), 1229568, (size_t)1229568*perm280->nb[0])), 64, 1601, 12, 1); + tensor sel283 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm280), 1229568, (size_t)2459136*perm280->nb[0])), 64, 1601, 12, 1); + tensor t284 = ggml_mul(m, as_f32(m, sel281), as_f32(m, const9)); + tensor t285 = ggml_cont(m, ggml_permute(m, sel282, 1, 0, 2, 3)); + tensor mm286 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, t285, 1, 0, 2, 3)), t284); + tensor rs287 = ggml_reshape_1d(m, ggml_cont(m, m.find("backbone.layers.8.attn.relative_position_index")), 2563201); + tensor gat288 = ggml_get_rows(m, m.find("backbone.layers.8.attn.relative_position_bias_table"), ggml_cast(m, ggml_cont(m, rs287), GGML_TYPE_I32)) /* index: dim 미상 → torch dim 0 (get_rows 행 선택) 가정 */; + tensor rs289 = ggml_reshape_3d(m, ggml_cont(m, gat288), 12, 1601, 1601); + tensor perm290 = ggml_cont(m, ggml_permute(m, rs289, 2, 0, 1, 3)); + tensor uns291 = ggml_reshape_4d(m, ggml_cont(m, perm290), 1601, 1601, 12, 1); + tensor add292 = ggml_add(m, mm286, uns291); + tensor sm293 = ggml_soft_max(m, add292); + tensor mm294 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, sel283, 1, 0, 2, 3)), sm293); + tensor t295 = ggml_cont(m, ggml_permute(m, mm294, 0, 2, 1, 3)); + tensor rs296 = ggml_reshape_3d(m, ggml_cont(m, t295), 768, 1601, 1); + tensor fc297 = linear(m["backbone.layers.8.attn.proj"], rs296); + tensor t298 = ggml_mul(m, as_f32(m, fc297), as_f32(m, m.find("backbone.layers.8.gamma_1"))); + tensor add299 = ggml_add(m, add271, t298); + tensor ln300 = layer_norm(m["backbone.layers.8.ln2"], add299); + tensor fc301 = linear(m["backbone.layers.8.ffn.layers.0.0"], ln300); + tensor act302 = ggml_gelu(m, fc301); + tensor fc303 = linear(m["backbone.layers.8.ffn.layers.1"], act302); + tensor t304 = ggml_mul(m, as_f32(m, fc303), as_f32(m, m.find("backbone.layers.8.gamma_2"))); + tensor add305 = ggml_add(m, add299, t304); + tensor ln306 = layer_norm(m["backbone.layers.9.ln1"], add305); + tensor like307 = ggml_fill(m, ggml_new_tensor_1d(m, GGML_TYPE_F32, 768), 0.0f); + tensor cat308 = ggml_concat(m, ggml_concat(m, m.find("backbone.layers.9.attn.q_bias"), like307, 0), m.find("backbone.layers.9.attn.v_bias"), 0); + tensor fcd309 = ggml_add(m, ggml_mul_mat(m, m.find("backbone.layers.9.attn.qkv.weight"), ln306), cat308); + tensor rs310 = ggml_reshape_4d(m, ggml_cont(m, fcd309), 64, 12, 3, 1601); + tensor perm311 = ggml_cont(m, ggml_permute(m, rs310, 0, 2, 3, 1)); + tensor sel312 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm311), 1229568, (size_t)0*perm311->nb[0])), 64, 1601, 12, 1); + tensor sel313 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm311), 1229568, (size_t)1229568*perm311->nb[0])), 64, 1601, 12, 1); + tensor sel314 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm311), 1229568, (size_t)2459136*perm311->nb[0])), 64, 1601, 12, 1); + tensor t315 = ggml_mul(m, as_f32(m, sel312), as_f32(m, const10)); + tensor t316 = ggml_cont(m, ggml_permute(m, sel313, 1, 0, 2, 3)); + tensor mm317 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, t316, 1, 0, 2, 3)), t315); + tensor rs318 = ggml_reshape_1d(m, ggml_cont(m, m.find("backbone.layers.9.attn.relative_position_index")), 2563201); + tensor gat319 = ggml_get_rows(m, m.find("backbone.layers.9.attn.relative_position_bias_table"), ggml_cast(m, ggml_cont(m, rs318), GGML_TYPE_I32)) /* index: dim 미상 → torch dim 0 (get_rows 행 선택) 가정 */; + tensor rs320 = ggml_reshape_3d(m, ggml_cont(m, gat319), 12, 1601, 1601); + tensor perm321 = ggml_cont(m, ggml_permute(m, rs320, 2, 0, 1, 3)); + tensor uns322 = ggml_reshape_4d(m, ggml_cont(m, perm321), 1601, 1601, 12, 1); + tensor add323 = ggml_add(m, mm317, uns322); + tensor sm324 = ggml_soft_max(m, add323); + tensor mm325 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, sel314, 1, 0, 2, 3)), sm324); + tensor t326 = ggml_cont(m, ggml_permute(m, mm325, 0, 2, 1, 3)); + tensor rs327 = ggml_reshape_3d(m, ggml_cont(m, t326), 768, 1601, 1); + tensor fc328 = linear(m["backbone.layers.9.attn.proj"], rs327); + tensor t329 = ggml_mul(m, as_f32(m, fc328), as_f32(m, m.find("backbone.layers.9.gamma_1"))); + tensor add330 = ggml_add(m, add305, t329); + tensor ln331 = layer_norm(m["backbone.layers.9.ln2"], add330); + tensor fc332 = linear(m["backbone.layers.9.ffn.layers.0.0"], ln331); + tensor act333 = ggml_gelu(m, fc332); + tensor fc334 = linear(m["backbone.layers.9.ffn.layers.1"], act333); + tensor t335 = ggml_mul(m, as_f32(m, fc334), as_f32(m, m.find("backbone.layers.9.gamma_2"))); + tensor add336 = ggml_add(m, add330, t335); + tensor ln337 = layer_norm(m["backbone.layers.10.ln1"], add336); + tensor like338 = ggml_fill(m, ggml_new_tensor_1d(m, GGML_TYPE_F32, 768), 0.0f); + tensor cat339 = ggml_concat(m, ggml_concat(m, m.find("backbone.layers.10.attn.q_bias"), like338, 0), m.find("backbone.layers.10.attn.v_bias"), 0); + tensor fcd340 = ggml_add(m, ggml_mul_mat(m, m.find("backbone.layers.10.attn.qkv.weight"), ln337), cat339); + tensor rs341 = ggml_reshape_4d(m, ggml_cont(m, fcd340), 64, 12, 3, 1601); + tensor perm342 = ggml_cont(m, ggml_permute(m, rs341, 0, 2, 3, 1)); + tensor sel343 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm342), 1229568, (size_t)0*perm342->nb[0])), 64, 1601, 12, 1); + tensor sel344 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm342), 1229568, (size_t)1229568*perm342->nb[0])), 64, 1601, 12, 1); + tensor sel345 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm342), 1229568, (size_t)2459136*perm342->nb[0])), 64, 1601, 12, 1); + tensor t346 = ggml_mul(m, as_f32(m, sel343), as_f32(m, const11)); + tensor t347 = ggml_cont(m, ggml_permute(m, sel344, 1, 0, 2, 3)); + tensor mm348 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, t347, 1, 0, 2, 3)), t346); + tensor rs349 = ggml_reshape_1d(m, ggml_cont(m, m.find("backbone.layers.10.attn.relative_position_index")), 2563201); + tensor gat350 = ggml_get_rows(m, m.find("backbone.layers.10.attn.relative_position_bias_table"), ggml_cast(m, ggml_cont(m, rs349), GGML_TYPE_I32)) /* index: dim 미상 → torch dim 0 (get_rows 행 선택) 가정 */; + tensor rs351 = ggml_reshape_3d(m, ggml_cont(m, gat350), 12, 1601, 1601); + tensor perm352 = ggml_cont(m, ggml_permute(m, rs351, 2, 0, 1, 3)); + tensor uns353 = ggml_reshape_4d(m, ggml_cont(m, perm352), 1601, 1601, 12, 1); + tensor add354 = ggml_add(m, mm348, uns353); + tensor sm355 = ggml_soft_max(m, add354); + tensor mm356 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, sel345, 1, 0, 2, 3)), sm355); + tensor t357 = ggml_cont(m, ggml_permute(m, mm356, 0, 2, 1, 3)); + tensor rs358 = ggml_reshape_3d(m, ggml_cont(m, t357), 768, 1601, 1); + tensor fc359 = linear(m["backbone.layers.10.attn.proj"], rs358); + tensor t360 = ggml_mul(m, as_f32(m, fc359), as_f32(m, m.find("backbone.layers.10.gamma_1"))); + tensor add361 = ggml_add(m, add336, t360); + tensor ln362 = layer_norm(m["backbone.layers.10.ln2"], add361); + tensor fc363 = linear(m["backbone.layers.10.ffn.layers.0.0"], ln362); + tensor act364 = ggml_gelu(m, fc363); + tensor fc365 = linear(m["backbone.layers.10.ffn.layers.1"], act364); + tensor t366 = ggml_mul(m, as_f32(m, fc365), as_f32(m, m.find("backbone.layers.10.gamma_2"))); + tensor add367 = ggml_add(m, add361, t366); + tensor ln368 = layer_norm(m["backbone.layers.11.ln1"], add367); + tensor like369 = ggml_fill(m, ggml_new_tensor_1d(m, GGML_TYPE_F32, 768), 0.0f); + tensor cat370 = ggml_concat(m, ggml_concat(m, m.find("backbone.layers.11.attn.q_bias"), like369, 0), m.find("backbone.layers.11.attn.v_bias"), 0); + tensor fcd371 = ggml_add(m, ggml_mul_mat(m, m.find("backbone.layers.11.attn.qkv.weight"), ln368), cat370); + tensor rs372 = ggml_reshape_4d(m, ggml_cont(m, fcd371), 64, 12, 3, 1601); + tensor perm373 = ggml_cont(m, ggml_permute(m, rs372, 0, 2, 3, 1)); + tensor sel374 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm373), 1229568, (size_t)0*perm373->nb[0])), 64, 1601, 12, 1); + tensor sel375 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm373), 1229568, (size_t)1229568*perm373->nb[0])), 64, 1601, 12, 1); + tensor sel376 = ggml_reshape_4d(m, ggml_cont(m, ggml_view_1d(m, ggml_cont(m, perm373), 1229568, (size_t)2459136*perm373->nb[0])), 64, 1601, 12, 1); + tensor t377 = ggml_mul(m, as_f32(m, sel374), as_f32(m, const12)); + tensor t378 = ggml_cont(m, ggml_permute(m, sel375, 1, 0, 2, 3)); + tensor mm379 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, t378, 1, 0, 2, 3)), t377); + tensor rs380 = ggml_reshape_1d(m, ggml_cont(m, m.find("backbone.layers.11.attn.relative_position_index")), 2563201); + tensor gat381 = ggml_get_rows(m, m.find("backbone.layers.11.attn.relative_position_bias_table"), ggml_cast(m, ggml_cont(m, rs380), GGML_TYPE_I32)) /* index: dim 미상 → torch dim 0 (get_rows 행 선택) 가정 */; + tensor rs382 = ggml_reshape_3d(m, ggml_cont(m, gat381), 12, 1601, 1601); + tensor perm383 = ggml_cont(m, ggml_permute(m, rs382, 2, 0, 1, 3)); + tensor uns384 = ggml_reshape_4d(m, ggml_cont(m, perm383), 1601, 1601, 12, 1); + tensor add385 = ggml_add(m, mm379, uns384); + tensor sm386 = ggml_soft_max(m, add385); + tensor mm387 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, sel376, 1, 0, 2, 3)), sm386); + tensor t388 = ggml_cont(m, ggml_permute(m, mm387, 0, 2, 1, 3)); + tensor rs389 = ggml_reshape_3d(m, ggml_cont(m, t388), 768, 1601, 1); + tensor fc390 = linear(m["backbone.layers.11.attn.proj"], rs389); + tensor t391 = ggml_mul(m, as_f32(m, fc390), as_f32(m, m.find("backbone.layers.11.gamma_1"))); + tensor add392 = ggml_add(m, add367, t391); + tensor ln393 = layer_norm(m["backbone.layers.11.ln2"], add392); + tensor fc394 = linear(m["backbone.layers.11.ffn.layers.0.0"], ln393); + tensor act395 = ggml_gelu(m, fc394); + tensor fc396 = linear(m["backbone.layers.11.ffn.layers.1"], act395); + tensor t397 = ggml_mul(m, as_f32(m, fc396), as_f32(m, m.find("backbone.layers.11.gamma_2"))); + tensor add398 = ggml_add(m, add392, t397); + tensor sl399 = ggml_view_3d(m, add398, 768, 1600, 1, add398->nb[1], add398->nb[2], 1*add398->nb[1]); + tensor rs400 = ggml_reshape_4d(m, ggml_cont(m, sl399), 768, 40, 40, 1); + tensor perm401 = ggml_cont(m, ggml_permute(m, rs400, 2, 0, 1, 3)); + tensor convT402 = conv_transpose_2d(m["neck.upsample_4x.0"], perm144, 2, 0, 1, 0); + tensor gelu403 = ggml_gelu(m, convT402); + tensor convT404 = conv_transpose_2d(m["neck.upsample_4x.3"], gelu403, 2, 0, 1, 0); + tensor convT405 = conv_transpose_2d(m["neck.upsample_2x.0"], perm209, 2, 0, 1, 0); + tensor pool406 = ggml_pool_2d(m, perm401, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0); + tensor conv407 = conv_2d(m["decode_head.lateral_convs.0.conv"], convT404, 1, 0); + tensor act408 = ggml_relu(m, conv407); + tensor conv409 = conv_2d(m["decode_head.lateral_convs.1.conv"], convT405, 1, 0); + tensor act410 = ggml_relu(m, conv409); + tensor conv411 = conv_2d(m["decode_head.lateral_convs.2.conv"], perm274, 1, 0); + tensor act412 = ggml_relu(m, conv411); + tensor pool413 = ggml_pool_2d(m, pool406, GGML_OP_POOL_AVG, 20, 20, 20, 20, 0, 0); + tensor conv414 = conv_2d(m["decode_head.psp_modules.0.1.conv"], pool413, 1, 0); + tensor act415 = ggml_relu(m, conv414); + tensor up416 = ggml_interpolate(m, act415, 20, 20, 768, 1, GGML_SCALE_MODE_BILINEAR) /* interpolate -> target ne, mode=bilinear */; + tensor pool417 = ggml_pool_2d(m, pool406, GGML_OP_POOL_AVG, 10, 10, 10, 10, 0, 0); + tensor conv418 = conv_2d(m["decode_head.psp_modules.1.1.conv"], pool417, 1, 0); + tensor act419 = ggml_relu(m, conv418); + tensor up420 = ggml_interpolate(m, act419, 20, 20, 768, 1, GGML_SCALE_MODE_BILINEAR) /* interpolate -> target ne, mode=bilinear */; + tensor aph421 = ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, pool406, 7, pool406->ne[1], pool406->ne[2], pool406->ne[3], pool406->nb[1], pool406->nb[2], pool406->nb[3], (size_t)0*pool406->nb[0]))), 0.14285714285714285f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, pool406, 8, pool406->ne[1], pool406->ne[2], pool406->ne[3], pool406->nb[1], pool406->nb[2], pool406->nb[3], (size_t)6*pool406->nb[0]))), 0.125f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, pool406, 7, pool406->ne[1], pool406->ne[2], pool406->ne[3], pool406->nb[1], pool406->nb[2], pool406->nb[3], (size_t)13*pool406->nb[0]))), 0.14285714285714285f), 0); + tensor apt422 = ggml_cont(m, ggml_permute(m, aph421, 1, 0, 2, 3)); + tensor apr423 = ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt422, 7, apt422->ne[1], apt422->ne[2], apt422->ne[3], apt422->nb[1], apt422->nb[2], apt422->nb[3], (size_t)0*apt422->nb[0]))), 0.14285714285714285f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt422, 8, apt422->ne[1], apt422->ne[2], apt422->ne[3], apt422->nb[1], apt422->nb[2], apt422->nb[3], (size_t)6*apt422->nb[0]))), 0.125f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt422, 7, apt422->ne[1], apt422->ne[2], apt422->ne[3], apt422->nb[1], apt422->nb[2], apt422->nb[3], (size_t)13*apt422->nb[0]))), 0.14285714285714285f), 0); + tensor pool424 = ggml_cont(m, ggml_permute(m, apr423, 1, 0, 2, 3)); + tensor conv425 = conv_2d(m["decode_head.psp_modules.2.1.conv"], pool424, 1, 0); + tensor act426 = ggml_relu(m, conv425); + tensor up427 = ggml_interpolate(m, act426, 20, 20, 768, 1, GGML_SCALE_MODE_BILINEAR) /* interpolate -> target ne, mode=bilinear */; + tensor aph428 = ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, pool406, 4, pool406->ne[1], pool406->ne[2], pool406->ne[3], pool406->nb[1], pool406->nb[2], pool406->nb[3], (size_t)0*pool406->nb[0]))), 0.25f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, pool406, 4, pool406->ne[1], pool406->ne[2], pool406->ne[3], pool406->nb[1], pool406->nb[2], pool406->nb[3], (size_t)3*pool406->nb[0]))), 0.25f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, pool406, 4, pool406->ne[1], pool406->ne[2], pool406->ne[3], pool406->nb[1], pool406->nb[2], pool406->nb[3], (size_t)6*pool406->nb[0]))), 0.25f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, pool406, 4, pool406->ne[1], pool406->ne[2], pool406->ne[3], pool406->nb[1], pool406->nb[2], pool406->nb[3], (size_t)10*pool406->nb[0]))), 0.25f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, pool406, 4, pool406->ne[1], pool406->ne[2], pool406->ne[3], pool406->nb[1], pool406->nb[2], pool406->nb[3], (size_t)13*pool406->nb[0]))), 0.25f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, pool406, 4, pool406->ne[1], pool406->ne[2], pool406->ne[3], pool406->nb[1], pool406->nb[2], pool406->nb[3], (size_t)16*pool406->nb[0]))), 0.25f), 0); + tensor apt429 = ggml_cont(m, ggml_permute(m, aph428, 1, 0, 2, 3)); + tensor apr430 = ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt429, 4, apt429->ne[1], apt429->ne[2], apt429->ne[3], apt429->nb[1], apt429->nb[2], apt429->nb[3], (size_t)0*apt429->nb[0]))), 0.25f), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt429, 4, apt429->ne[1], apt429->ne[2], apt429->ne[3], apt429->nb[1], apt429->nb[2], apt429->nb[3], (size_t)3*apt429->nb[0]))), 0.25f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt429, 4, apt429->ne[1], apt429->ne[2], apt429->ne[3], apt429->nb[1], apt429->nb[2], apt429->nb[3], (size_t)6*apt429->nb[0]))), 0.25f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt429, 4, apt429->ne[1], apt429->ne[2], apt429->ne[3], apt429->nb[1], apt429->nb[2], apt429->nb[3], (size_t)10*apt429->nb[0]))), 0.25f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt429, 4, apt429->ne[1], apt429->ne[2], apt429->ne[3], apt429->nb[1], apt429->nb[2], apt429->nb[3], (size_t)13*apt429->nb[0]))), 0.25f), 0), ggml_scale(m, ggml_sum_rows(m, ggml_cont(m, ggml_view_4d(m, apt429, 4, apt429->ne[1], apt429->ne[2], apt429->ne[3], apt429->nb[1], apt429->nb[2], apt429->nb[3], (size_t)16*apt429->nb[0]))), 0.25f), 0); + tensor pool431 = ggml_cont(m, ggml_permute(m, apr430, 1, 0, 2, 3)); + tensor conv432 = conv_2d(m["decode_head.psp_modules.3.1.conv"], pool431, 1, 0); + tensor act433 = ggml_relu(m, conv432); + tensor up434 = ggml_interpolate(m, act433, 20, 20, 768, 1, GGML_SCALE_MODE_BILINEAR) /* interpolate -> target ne, mode=bilinear */; + tensor cat435 = ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, pool406, up416, 2), up420, 2), up427, 2), up434, 2); + tensor conv436 = conv_2d(m["decode_head.bottleneck.conv"], cat435, 1, 1); + tensor act437 = ggml_relu(m, conv436); + tensor up438 = ggml_interpolate(m, act437, 40, 40, 768, 1, GGML_SCALE_MODE_BILINEAR) /* interpolate -> target ne, mode=bilinear */; + tensor add439 = ggml_add(m, act412, up438); + tensor up440 = ggml_interpolate(m, add439, 80, 80, 768, 1, GGML_SCALE_MODE_BILINEAR) /* interpolate -> target ne, mode=bilinear */; + tensor add441 = ggml_add(m, act410, up440); + tensor up442 = ggml_interpolate(m, add441, 160, 160, 768, 1, GGML_SCALE_MODE_BILINEAR) /* interpolate -> target ne, mode=bilinear */; + tensor add443 = ggml_add(m, act408, up442); + tensor conv444 = conv_2d(m["decode_head.fpn_convs.0.conv"], add443, 1, 1); + tensor act445 = ggml_relu(m, conv444); + tensor conv446 = conv_2d(m["decode_head.fpn_convs.1.conv"], add441, 1, 1); + tensor act447 = ggml_relu(m, conv446); + tensor conv448 = conv_2d(m["decode_head.fpn_convs.2.conv"], add439, 1, 1); + tensor act449 = ggml_relu(m, conv448); + tensor up450 = ggml_interpolate(m, act437, 160, 160, 768, 1, GGML_SCALE_MODE_BILINEAR) /* interpolate -> target ne, mode=bilinear */; + tensor up451 = ggml_interpolate(m, act449, 160, 160, 768, 1, GGML_SCALE_MODE_BILINEAR) /* interpolate -> target ne, mode=bilinear */; + tensor up452 = ggml_interpolate(m, act447, 160, 160, 768, 1, GGML_SCALE_MODE_BILINEAR) /* interpolate -> target ne, mode=bilinear */; + tensor cat453 = ggml_concat(m, ggml_concat(m, ggml_concat(m, act445, up452, 2), up451, 2), up450, 2); + tensor conv454 = conv_2d(m["decode_head.fpn_bottleneck.conv"], cat453, 1, 1); + tensor act455 = ggml_relu(m, conv454); + tensor conv456 = conv_2d(m["decode_head.conv_seg"], act455, 1, 0); + + tensor out_0 = compute_graph_output(m, contiguous_2d_to_cwhn(m, conv456), "out_0"); + return out_0; +} + +FamBeit_params FamBeit_detect_params(model_file const& f) { + FamBeit_params p{}; + // GGUF general.architecture 검증 (모델은 정적 unroll 이라 추가 하이퍼파라미터 없음). + if (std::string_view arch = f.arch(); arch != "fambeit") { + throw except( + "Architecture expected to be 'fambeit', but was '{}' ({})", + arch, f.path); + } + return p; +} + +} // namespace visp diff --git a/src/visp/arch/Yolo26m.h b/src/visp/arch/FamBeit.h similarity index 61% rename from src/visp/arch/Yolo26m.h rename to src/visp/arch/FamBeit.h index edcf069..2e8aaac 100644 --- a/src/visp/arch/Yolo26m.h +++ b/src/visp/arch/FamBeit.h @@ -4,11 +4,11 @@ namespace visp { -struct Yolo26m_params { +struct FamBeit_params { // TODO(ggml): 모델 하이퍼파라미터 (block_count 등) }; -tensor Yolo26m_forward(model_ref m, tensor x, Yolo26m_params const& p); -Yolo26m_params Yolo26m_detect_params(model_file const& f); +tensor FamBeit_forward(model_ref m, tensor x, FamBeit_params const& p); +FamBeit_params FamBeit_detect_params(model_file const& f); } // namespace visp diff --git a/src/visp/arch/FamBisenetv1.cpp b/src/visp/arch/FamBisenetv1.cpp new file mode 100644 index 0000000..693b467 --- /dev/null +++ b/src/visp/arch/FamBisenetv1.cpp @@ -0,0 +1,124 @@ +// GENERATED BY SuperGate GTX Compiler (ggml/vision.cpp backend), DO NOT EDIT! +#include "visp/arch/FamBisenetv1.h" +#include "visp/ml.h" +#include "visp/nn.h" +#include "visp/vision.h" +#include "util/string.h" +#include +#include // clamp 의 한쪽 경계가 없으면 ±INFINITY 를 낸다 + +#include + +namespace visp { + +tensor FamBisenetv1_forward(model_ref m, tensor x, FamBisenetv1_params const& p) { + (void)p; + // 입력 레이아웃 변환 (TODO: cwhn/whcn 자동 판별) + x = cwhn_to_contiguous_2d(m, x); + + tensor conv1 = conv_2d(m["backbone.context_path.backbone.conv1"], x, 2, 3); + tensor act2 = ggml_relu(m, conv1); + tensor pool3 = ggml_pool_2d(m, act2, GGML_OP_POOL_MAX, 3, 3, 2, 2, 1, 1); + tensor conv4 = conv_2d(m["backbone.context_path.backbone.layer1.0.conv1"], pool3, 1, 1); + tensor act5 = ggml_relu(m, conv4); + tensor conv6 = conv_2d(m["backbone.context_path.backbone.layer1.0.conv2"], act5, 1, 1); + tensor add7 = ggml_add(m, conv6, pool3); + tensor act8 = ggml_relu(m, add7); + tensor conv9 = conv_2d(m["backbone.context_path.backbone.layer1.1.conv1"], act8, 1, 1); + tensor act10 = ggml_relu(m, conv9); + tensor conv11 = conv_2d(m["backbone.context_path.backbone.layer1.1.conv2"], act10, 1, 1); + tensor add12 = ggml_add(m, conv11, act8); + tensor act13 = ggml_relu(m, add12); + tensor conv14 = conv_2d(m["backbone.context_path.backbone.layer2.0.conv1"], act13, 2, 1); + tensor act15 = ggml_relu(m, conv14); + tensor conv16 = conv_2d(m["backbone.context_path.backbone.layer2.0.conv2"], act15, 1, 1); + tensor conv17 = conv_2d(m["backbone.cp.backbone.layer2.0.downsample.0"], act13, 2, 0); + tensor add18 = ggml_add(m, conv16, conv17); + tensor act19 = ggml_relu(m, add18); + tensor conv20 = conv_2d(m["backbone.context_path.backbone.layer2.1.conv1"], act19, 1, 1); + tensor act21 = ggml_relu(m, conv20); + tensor conv22 = conv_2d(m["backbone.context_path.backbone.layer2.1.conv2"], act21, 1, 1); + tensor add23 = ggml_add(m, conv22, act19); + tensor act24 = ggml_relu(m, add23); + tensor conv25 = conv_2d(m["backbone.context_path.backbone.layer3.0.conv1"], act24, 2, 1); + tensor act26 = ggml_relu(m, conv25); + tensor conv27 = conv_2d(m["backbone.context_path.backbone.layer3.0.conv2"], act26, 1, 1); + tensor conv28 = conv_2d(m["backbone.cp.backbone.layer3.0.downsample.0"], act24, 2, 0); + tensor add29 = ggml_add(m, conv27, conv28); + tensor act30 = ggml_relu(m, add29); + tensor conv31 = conv_2d(m["backbone.context_path.backbone.layer3.1.conv1"], act30, 1, 1); + tensor act32 = ggml_relu(m, conv31); + tensor conv33 = conv_2d(m["backbone.context_path.backbone.layer3.1.conv2"], act32, 1, 1); + tensor add34 = ggml_add(m, conv33, act30); + tensor act35 = ggml_relu(m, add34); + tensor conv36 = conv_2d(m["backbone.context_path.backbone.layer4.0.conv1"], act35, 2, 1); + tensor act37 = ggml_relu(m, conv36); + tensor conv38 = conv_2d(m["backbone.context_path.backbone.layer4.0.conv2"], act37, 1, 1); + tensor conv39 = conv_2d(m["backbone.cp.backbone.layer4.0.downsample.0"], act35, 2, 0); + tensor add40 = ggml_add(m, conv38, conv39); + tensor act41 = ggml_relu(m, add40); + tensor conv42 = conv_2d(m["backbone.context_path.backbone.layer4.1.conv1"], act41, 1, 1); + tensor act43 = ggml_relu(m, conv42); + tensor conv44 = conv_2d(m["backbone.context_path.backbone.layer4.1.conv2"], act43, 1, 1); + tensor add45 = ggml_add(m, conv44, act41); + tensor act46 = ggml_relu(m, add45); + tensor pool47 = ggml_pool_2d(m, act46, GGML_OP_POOL_AVG, 32, 32, 32, 32, 0, 0); + tensor conv48 = conv_2d(m["backbone.context_path.gap_conv.1.conv"], pool47, 1, 0); + tensor act49 = ggml_relu(m, conv48); + tensor conv50 = conv_2d(m["backbone.context_path.arm32.conv_layer.conv"], act46, 1, 1); + tensor act51 = ggml_relu(m, conv50); + tensor pool52 = ggml_pool_2d(m, act51, GGML_OP_POOL_AVG, 32, 32, 32, 32, 0, 0); + tensor conv53 = conv_2d(m["backbone.cp.arm32.acl.1.conv"], pool52, 1, 0); + tensor act54 = ggml_sigmoid(m, conv53); + tensor t55 = ggml_mul(m, as_f32(m, act51), as_f32(m, act54)); + tensor add56 = ggml_add(m, t55, act49); + tensor up57 = ggml_interpolate(m, add56, 64, 64, 128, 1, GGML_SCALE_MODE_NEAREST) /* interpolate -> target ne */; + tensor conv58 = conv_2d(m["backbone.context_path.conv_head32.conv"], up57, 1, 1); + tensor act59 = ggml_relu(m, conv58); + tensor conv60 = conv_2d(m["backbone.context_path.arm16.conv_layer.conv"], act35, 1, 1); + tensor act61 = ggml_relu(m, conv60); + tensor pool62 = ggml_pool_2d(m, act61, GGML_OP_POOL_AVG, 64, 64, 64, 64, 0, 0); + tensor conv63 = conv_2d(m["backbone.cp.arm16.acl.1.conv"], pool62, 1, 0); + tensor act64 = ggml_sigmoid(m, conv63); + tensor t65 = ggml_mul(m, as_f32(m, act61), as_f32(m, act64)); + tensor add66 = ggml_add(m, t65, act59); + tensor up67 = ggml_interpolate(m, add66, 128, 128, 128, 1, GGML_SCALE_MODE_NEAREST) /* interpolate -> target ne */; + tensor conv68 = conv_2d(m["backbone.context_path.conv_head16.conv"], up67, 1, 1); + tensor act69 = ggml_relu(m, conv68); + tensor conv70 = conv_2d(m["backbone.spatial_path.layer1.conv"], x, 2, 3); + tensor act71 = ggml_relu(m, conv70); + tensor conv72 = conv_2d(m["backbone.spatial_path.layer2.conv"], act71, 2, 1); + tensor act73 = ggml_relu(m, conv72); + tensor conv74 = conv_2d(m["backbone.spatial_path.layer3.conv"], act73, 2, 1); + tensor act75 = ggml_relu(m, conv74); + tensor conv76 = conv_2d(m["backbone.spatial_path.layer4.conv"], act75, 1, 0); + tensor act77 = ggml_relu(m, conv76); + tensor cat78 = ggml_concat(m, act77, act69, 2); + tensor conv79 = conv_2d(m["backbone.ffm.conv1.conv"], cat78, 1, 0); + tensor act80 = ggml_relu(m, conv79); + tensor pool81 = ggml_pool_2d(m, act80, GGML_OP_POOL_AVG, 128, 128, 128, 128, 0, 0); + tensor conv82 = conv_2d(m["backbone.ffm.conv_atten.0.conv"], pool81, 1, 0); + tensor act83 = ggml_relu(m, conv82); + tensor t84 = ggml_sigmoid(m, act83); + tensor t85 = ggml_mul(m, as_f32(m, act80), as_f32(m, t84)); + tensor add86 = ggml_add(m, t85, act80); + tensor conv87 = conv_2d(m["decode_head.convs.0.conv"], add86, 1, 1); + tensor act88 = ggml_relu(m, conv87); + tensor conv89 = conv_2d(m["decode_head.conv_seg"], act88, 1, 0); + + tensor out_0 = compute_graph_output(m, contiguous_2d_to_cwhn(m, conv89), "out_0"); + return out_0; +} + +FamBisenetv1_params FamBisenetv1_detect_params(model_file const& f) { + FamBisenetv1_params p{}; + // GGUF general.architecture 검증 (모델은 정적 unroll 이라 추가 하이퍼파라미터 없음). + if (std::string_view arch = f.arch(); arch != "fambisenetv1") { + throw except( + "Architecture expected to be 'fambisenetv1', but was '{}' ({})", + arch, f.path); + } + return p; +} + +} // namespace visp diff --git a/src/visp/arch/FamBisenetv1.h b/src/visp/arch/FamBisenetv1.h new file mode 100644 index 0000000..8c3cdbc --- /dev/null +++ b/src/visp/arch/FamBisenetv1.h @@ -0,0 +1,14 @@ +// GENERATED BY SuperGate GTX Compiler (ggml/vision.cpp backend), DO NOT EDIT! +#pragma once +#include "visp/ml.h" + +namespace visp { + +struct FamBisenetv1_params { + // TODO(ggml): 모델 하이퍼파라미터 (block_count 등) +}; + +tensor FamBisenetv1_forward(model_ref m, tensor x, FamBisenetv1_params const& p); +FamBisenetv1_params FamBisenetv1_detect_params(model_file const& f); + +} // namespace visp diff --git a/src/visp/arch/FamBisenetv2.cpp b/src/visp/arch/FamBisenetv2.cpp new file mode 100644 index 0000000..f58bec1 --- /dev/null +++ b/src/visp/arch/FamBisenetv2.cpp @@ -0,0 +1,151 @@ +// GENERATED BY SuperGate GTX Compiler (ggml/vision.cpp backend), DO NOT EDIT! +#include "visp/arch/FamBisenetv2.h" +#include "visp/ml.h" +#include "visp/nn.h" +#include "visp/vision.h" +#include "util/string.h" +#include +#include // clamp 의 한쪽 경계가 없으면 ±INFINITY 를 낸다 + +#include + +namespace visp { + +tensor FamBisenetv2_forward(model_ref m, tensor x, FamBisenetv2_params const& p) { + (void)p; + // 입력 레이아웃 변환 (TODO: cwhn/whcn 자동 판별) + x = cwhn_to_contiguous_2d(m, x); + + tensor conv1 = conv_2d(m["backbone.detail.detail_branch.0.0.conv"], x, 2, 1); + tensor act2 = ggml_relu(m, conv1); + tensor conv3 = conv_2d(m["backbone.detail.detail_branch.0.1.conv"], act2, 1, 1); + tensor act4 = ggml_relu(m, conv3); + tensor conv5 = conv_2d(m["backbone.detail.detail_branch.1.0.conv"], act4, 2, 1); + tensor act6 = ggml_relu(m, conv5); + tensor conv7 = conv_2d(m["backbone.detail.detail_branch.1.1.conv"], act6, 1, 1); + tensor act8 = ggml_relu(m, conv7); + tensor conv9 = conv_2d(m["backbone.detail.detail_branch.1.2.conv"], act8, 1, 1); + tensor act10 = ggml_relu(m, conv9); + tensor conv11 = conv_2d(m["backbone.detail.detail_branch.2.0.conv"], act10, 2, 1); + tensor act12 = ggml_relu(m, conv11); + tensor conv13 = conv_2d(m["backbone.detail.detail_branch.2.1.conv"], act12, 1, 1); + tensor act14 = ggml_relu(m, conv13); + tensor conv15 = conv_2d(m["backbone.detail.detail_branch.2.2.conv"], act14, 1, 1); + tensor act16 = ggml_relu(m, conv15); + tensor conv17 = conv_2d(m["backbone.semantic.stage1.conv_first.conv"], x, 2, 1); + tensor act18 = ggml_relu(m, conv17); + tensor conv19 = conv_2d(m["backbone.semantic.stage1.convs.0.conv"], act18, 1, 0); + tensor act20 = ggml_relu(m, conv19); + tensor conv21 = conv_2d(m["backbone.semantic.stage1.convs.1.conv"], act20, 2, 1); + tensor act22 = ggml_relu(m, conv21); + tensor pool23 = ggml_pool_2d(m, act18, GGML_OP_POOL_MAX, 3, 3, 2, 2, 1, 1); + tensor cat24 = ggml_concat(m, act22, pool23, 2); + tensor conv25 = conv_2d(m["backbone.semantic.stage1.fuse_last.conv"], cat24, 1, 1); + tensor act26 = ggml_relu(m, conv25); + tensor conv27 = conv_2d(m["backbone.semantic.stage2.0.conv1.conv"], act26, 1, 1); + tensor act28 = ggml_relu(m, conv27); + tensor dwconv29 = conv_2d_grouped(m["backbone.semantic.stage2.0.dwconv.0.conv"], act28, 2, 1, 1, 16); + tensor dwconv30 = conv_2d_depthwise(m["backbone.semantic.stage2.0.dwconv.1.conv"], dwconv29, 1, 1); + tensor act31 = ggml_relu(m, dwconv30); + tensor conv32 = conv_2d(m["backbone.semantic.stage2.0.conv2.0.conv"], act31, 1, 0); + tensor dwconv33 = conv_2d_depthwise(m["backbone.semantic.stage2.0.shortcut.0.dc.conv"], act26, 2, 1); + tensor conv34 = conv_2d(m["backbone.semantic.stage2.0.shortcut.0.pc.conv"], dwconv33, 1, 0); + tensor add35 = ggml_add(m, conv32, conv34); + tensor act36 = ggml_relu(m, add35); + tensor conv37 = conv_2d(m["backbone.semantic.stage2.1.conv1.conv"], act36, 1, 1); + tensor act38 = ggml_relu(m, conv37); + tensor dwconv39 = conv_2d_grouped(m["backbone.semantic.stage2.1.dwconv.0.conv"], act38, 1, 1, 1, 32); + tensor act40 = ggml_relu(m, dwconv39); + tensor conv41 = conv_2d(m["backbone.semantic.stage2.1.conv2.0.conv"], act40, 1, 0); + tensor add42 = ggml_add(m, conv41, act36); + tensor act43 = ggml_relu(m, add42); + tensor conv44 = conv_2d(m["backbone.semantic.stage3.0.conv1.conv"], act43, 1, 1); + tensor act45 = ggml_relu(m, conv44); + tensor dwconv46 = conv_2d_grouped(m["backbone.semantic.stage3.0.dwconv.0.conv"], act45, 2, 1, 1, 32); + tensor dwconv47 = conv_2d_depthwise(m["backbone.semantic.stage3.0.dwconv.1.conv"], dwconv46, 1, 1); + tensor act48 = ggml_relu(m, dwconv47); + tensor conv49 = conv_2d(m["backbone.semantic.stage3.0.conv2.0.conv"], act48, 1, 0); + tensor dwconv50 = conv_2d_depthwise(m["backbone.semantic.stage3.0.shortcut.0.dc.conv"], act43, 2, 1); + tensor conv51 = conv_2d(m["backbone.semantic.stage3.0.shortcut.0.pc.conv"], dwconv50, 1, 0); + tensor add52 = ggml_add(m, conv49, conv51); + tensor act53 = ggml_relu(m, add52); + tensor conv54 = conv_2d(m["backbone.semantic.stage3.1.conv1.conv"], act53, 1, 1); + tensor act55 = ggml_relu(m, conv54); + tensor dwconv56 = conv_2d_grouped(m["backbone.semantic.stage3.1.dwconv.0.conv"], act55, 1, 1, 1, 64); + tensor act57 = ggml_relu(m, dwconv56); + tensor conv58 = conv_2d(m["backbone.semantic.stage3.1.conv2.0.conv"], act57, 1, 0); + tensor add59 = ggml_add(m, conv58, act53); + tensor act60 = ggml_relu(m, add59); + tensor conv61 = conv_2d(m["backbone.semantic.stage4.0.conv1.conv"], act60, 1, 1); + tensor act62 = ggml_relu(m, conv61); + tensor dwconv63 = conv_2d_grouped(m["backbone.semantic.stage4.0.dwconv.0.conv"], act62, 2, 1, 1, 64); + tensor dwconv64 = conv_2d_depthwise(m["backbone.semantic.stage4.0.dwconv.1.conv"], dwconv63, 1, 1); + tensor act65 = ggml_relu(m, dwconv64); + tensor conv66 = conv_2d(m["backbone.semantic.stage4.0.conv2.0.conv"], act65, 1, 0); + tensor dwconv67 = conv_2d_depthwise(m["backbone.semantic.stage4.0.shortcut.0.dc.conv"], act60, 2, 1); + tensor conv68 = conv_2d(m["backbone.semantic.stage4.0.shortcut.0.pc.conv"], dwconv67, 1, 0); + tensor add69 = ggml_add(m, conv66, conv68); + tensor act70 = ggml_relu(m, add69); + tensor conv71 = conv_2d(m["backbone.semantic.stage4.1.conv1.conv"], act70, 1, 1); + tensor act72 = ggml_relu(m, conv71); + tensor dwconv73 = conv_2d_grouped(m["backbone.semantic.stage4.1.dwconv.0.conv"], act72, 1, 1, 1, 128); + tensor act74 = ggml_relu(m, dwconv73); + tensor conv75 = conv_2d(m["backbone.semantic.stage4.1.conv2.0.conv"], act74, 1, 0); + tensor add76 = ggml_add(m, conv75, act70); + tensor act77 = ggml_relu(m, add76); + tensor conv78 = conv_2d(m["backbone.semantic.stage4.2.conv1.conv"], act77, 1, 1); + tensor act79 = ggml_relu(m, conv78); + tensor dwconv80 = conv_2d_grouped(m["backbone.semantic.stage4.2.dwconv.0.conv"], act79, 1, 1, 1, 128); + tensor act81 = ggml_relu(m, dwconv80); + tensor conv82 = conv_2d(m["backbone.semantic.stage4.2.conv2.0.conv"], act81, 1, 0); + tensor add83 = ggml_add(m, conv82, act77); + tensor act84 = ggml_relu(m, add83); + tensor conv85 = conv_2d(m["backbone.semantic.stage4.3.conv1.conv"], act84, 1, 1); + tensor act86 = ggml_relu(m, conv85); + tensor dwconv87 = conv_2d_grouped(m["backbone.semantic.stage4.3.dwconv.0.conv"], act86, 1, 1, 1, 128); + tensor act88 = ggml_relu(m, dwconv87); + tensor conv89 = conv_2d(m["backbone.semantic.stage4.3.conv2.0.conv"], act88, 1, 0); + tensor add90 = ggml_add(m, conv89, act84); + tensor act91 = ggml_relu(m, add90); + tensor pool92 = ggml_pool_2d(m, act91, GGML_OP_POOL_AVG, 32, 32, 32, 32, 0, 0); + tensor bn93 = batch_norm_2d(m["backbone.semantic.stage4_CEBlock.gap.1"], pool92); + tensor conv94 = conv_2d(m["backbone.semantic.stage4_CEBlock.conv_gap.conv"], bn93, 1, 0); + tensor act95 = ggml_relu(m, conv94); + tensor add96 = ggml_add(m, act91, act95); + tensor conv97 = conv_2d(m["backbone.semantic.stage4_CEBlock.conv_last.conv"], add96, 1, 1); + tensor act98 = ggml_relu(m, conv97); + tensor dwconv99 = conv_2d_depthwise(m["backbone.bga.detail_dwconv.0.depthwise_conv.conv"], act16, 1, 1); + tensor conv100 = conv_2d(m["backbone.bga.detail_dwconv.0.pointwise_conv.conv"], dwconv99, 1, 0); + tensor conv101 = conv_2d(m["backbone.bga.detail_down.0.conv"], act16, 2, 1); + tensor pool102 = ggml_pool_2d(m, conv101, GGML_OP_POOL_AVG, 3, 3, 2, 2, 1, 1); + tensor conv103 = conv_2d(m["backbone.bga.semantic_conv.0.conv"], act98, 1, 1); + tensor dwconv104 = conv_2d_depthwise(m["backbone.bga.semantic_dwconv.0.depthwise_conv.conv"], act98, 1, 1); + tensor conv105 = conv_2d(m["backbone.bga.semantic_dwconv.0.pointwise_conv.conv"], dwconv104, 1, 0); + tensor act106 = ggml_sigmoid(m, conv105); + tensor t108 = ggml_sigmoid(m, ggml_interpolate(m, conv103, 128, 128, 128, 1, GGML_SCALE_MODE_BILINEAR) /* interpolate -> target ne, mode=bilinear */); + tensor t109 = ggml_mul(m, as_f32(m, conv100), as_f32(m, t108)); + tensor t110 = ggml_mul(m, as_f32(m, pool102), as_f32(m, act106)); + tensor up111 = ggml_interpolate(m, t110, 128, 128, 128, 1, GGML_SCALE_MODE_BILINEAR) /* interpolate -> target ne, mode=bilinear */; + tensor add112 = ggml_add(m, t109, up111); + tensor conv113 = conv_2d(m["backbone.bga.conv.conv"], add112, 1, 1); + tensor act114 = ggml_relu(m, conv113); + tensor conv115 = conv_2d(m["decode_head.convs.0.conv"], act114, 1, 1); + tensor act116 = ggml_relu(m, conv115); + tensor conv117 = conv_2d(m["decode_head.conv_seg"], act116, 1, 0); + + tensor out_0 = compute_graph_output(m, contiguous_2d_to_cwhn(m, conv117), "out_0"); + return out_0; +} + +FamBisenetv2_params FamBisenetv2_detect_params(model_file const& f) { + FamBisenetv2_params p{}; + // GGUF general.architecture 검증 (모델은 정적 unroll 이라 추가 하이퍼파라미터 없음). + if (std::string_view arch = f.arch(); arch != "fambisenetv2") { + throw except( + "Architecture expected to be 'fambisenetv2', but was '{}' ({})", + arch, f.path); + } + return p; +} + +} // namespace visp diff --git a/src/visp/arch/FamBisenetv2.h b/src/visp/arch/FamBisenetv2.h new file mode 100644 index 0000000..6c4140d --- /dev/null +++ b/src/visp/arch/FamBisenetv2.h @@ -0,0 +1,14 @@ +// GENERATED BY SuperGate GTX Compiler (ggml/vision.cpp backend), DO NOT EDIT! +#pragma once +#include "visp/ml.h" + +namespace visp { + +struct FamBisenetv2_params { + // TODO(ggml): 모델 하이퍼파라미터 (block_count 등) +}; + +tensor FamBisenetv2_forward(model_ref m, tensor x, FamBisenetv2_params const& p); +FamBisenetv2_params FamBisenetv2_detect_params(model_file const& f); + +} // namespace visp diff --git a/src/visp/arch/FamCgnet.cpp b/src/visp/arch/FamCgnet.cpp new file mode 100644 index 0000000..262e69b --- /dev/null +++ b/src/visp/arch/FamCgnet.cpp @@ -0,0 +1,438 @@ +// GENERATED BY SuperGate GTX Compiler (ggml/vision.cpp backend), DO NOT EDIT! +#include "visp/arch/FamCgnet.h" +#include "visp/ml.h" +#include "visp/nn.h" +#include "visp/vision.h" +#include "util/string.h" +#include +#include // clamp 의 한쪽 경계가 없으면 ±INFINITY 를 낸다 + +#include + +namespace visp { + +tensor FamCgnet_forward(model_ref m, tensor x, FamCgnet_params const& p) { + (void)p; + // 입력 레이아웃 변환 (TODO: cwhn/whcn 자동 판별) + x = cwhn_to_contiguous_2d(m, x); + + tensor pool1 = ggml_pool_2d(m, x, GGML_OP_POOL_AVG, 3, 3, 2, 2, 1, 1); + tensor pool2 = ggml_pool_2d(m, x, GGML_OP_POOL_AVG, 3, 3, 2, 2, 1, 1); + tensor pool3 = ggml_pool_2d(m, pool2, GGML_OP_POOL_AVG, 3, 3, 2, 2, 1, 1); + tensor conv4 = conv_2d(m["backbone.stem.0.conv"], x, 2, 1); + tensor prelu5 = prelu(m["backbone.stem.0.activate"], conv4); + tensor conv6 = conv_2d(m["backbone.stem.1.conv"], prelu5, 1, 1); + tensor prelu7 = prelu(m["backbone.stem.1.activate"], conv6); + tensor conv8 = conv_2d(m["backbone.stem.2.conv"], prelu7, 1, 1); + tensor prelu9 = prelu(m["backbone.stem.2.activate"], conv8); + tensor cat10 = ggml_concat(m, prelu9, pool1, 2); + tensor bn11 = batch_norm_2d(m["backbone.norm_prelu_0.0"], cat10); + tensor prelu12 = prelu(m["backbone.norm_prelu_0.1"], bn11); + tensor conv13 = conv_2d(m["backbone.level1.0.conv1x1.conv"], prelu12, 2, 1); + tensor prelu14 = prelu(m["backbone.level1.0.conv1x1.activate"], conv13); + tensor dwconv15 = conv_2d_depthwise(m["backbone.level1.0.f_loc"], prelu14, 1, 1); + tensor dwconv16 = conv_2d_depthwise(m["backbone.level1.0.f_sur"], prelu14, 1, 2, 2); + tensor cat17 = ggml_concat(m, dwconv15, dwconv16, 2); + tensor bn18 = batch_norm_2d(m["backbone.level1.0.bn"], cat17); + tensor prelu19 = prelu(m["backbone.level1.0.activate"], bn18); + tensor conv20 = conv_2d(m["backbone.level1.0.bottleneck"], prelu19, 1, 0); + tensor pool21 = ggml_pool_2d(m, conv20, GGML_OP_POOL_AVG, 170, 170, 170, 170, 0, 0); + tensor rs22 = ggml_reshape_2d(m, ggml_cont(m, pool21), 64, 1); + tensor fc23 = linear(m["backbone.level1.0.f_glo.fc.0"], rs22); + tensor act24 = ggml_relu(m, fc23); + tensor fc25 = linear(m["backbone.level1.0.f_glo.fc.2"], act24); + tensor act26 = ggml_sigmoid(m, fc25); + tensor rs27 = ggml_reshape_4d(m, ggml_cont(m, act26), 1, 1, 64, 1); + tensor t28 = ggml_mul(m, as_f32(m, conv20), as_f32(m, rs27)); + tensor conv29 = conv_2d(m["backbone.level1.1.conv1x1.conv"], t28, 1, 0); + tensor prelu30 = prelu(m["backbone.level1.1.conv1x1.activate"], conv29); + tensor dwconv31 = conv_2d_depthwise(m["backbone.level1.1.f_loc"], prelu30, 1, 1); + tensor dwconv32 = conv_2d_depthwise(m["backbone.level1.1.f_sur"], prelu30, 1, 2, 2); + tensor cat33 = ggml_concat(m, dwconv31, dwconv32, 2); + tensor bn34 = batch_norm_2d(m["backbone.level1.1.bn"], cat33); + tensor prelu35 = prelu(m["backbone.level1.1.activate"], bn34); + tensor pool36 = ggml_pool_2d(m, prelu35, GGML_OP_POOL_AVG, 170, 170, 170, 170, 0, 0); + tensor rs37 = ggml_reshape_2d(m, ggml_cont(m, pool36), 64, 1); + tensor fc38 = linear(m["backbone.level1.1.f_glo.fc.0"], rs37); + tensor act39 = ggml_relu(m, fc38); + tensor fc40 = linear(m["backbone.level1.1.f_glo.fc.2"], act39); + tensor act41 = ggml_sigmoid(m, fc40); + tensor rs42 = ggml_reshape_4d(m, ggml_cont(m, act41), 1, 1, 64, 1); + tensor t43 = ggml_mul(m, as_f32(m, prelu35), as_f32(m, rs42)); + tensor add44 = ggml_add(m, t28, t43); + tensor conv45 = conv_2d(m["backbone.level1.2.conv1x1.conv"], add44, 1, 0); + tensor prelu46 = prelu(m["backbone.level1.2.conv1x1.activate"], conv45); + tensor dwconv47 = conv_2d_depthwise(m["backbone.level1.2.f_loc"], prelu46, 1, 1); + tensor dwconv48 = conv_2d_depthwise(m["backbone.level1.2.f_sur"], prelu46, 1, 2, 2); + tensor cat49 = ggml_concat(m, dwconv47, dwconv48, 2); + tensor bn50 = batch_norm_2d(m["backbone.level1.2.bn"], cat49); + tensor prelu51 = prelu(m["backbone.level1.2.activate"], bn50); + tensor pool52 = ggml_pool_2d(m, prelu51, GGML_OP_POOL_AVG, 170, 170, 170, 170, 0, 0); + tensor rs53 = ggml_reshape_2d(m, ggml_cont(m, pool52), 64, 1); + tensor fc54 = linear(m["backbone.level1.2.f_glo.fc.0"], rs53); + tensor act55 = ggml_relu(m, fc54); + tensor fc56 = linear(m["backbone.level1.2.f_glo.fc.2"], act55); + tensor act57 = ggml_sigmoid(m, fc56); + tensor rs58 = ggml_reshape_4d(m, ggml_cont(m, act57), 1, 1, 64, 1); + tensor t59 = ggml_mul(m, as_f32(m, prelu51), as_f32(m, rs58)); + tensor add60 = ggml_add(m, add44, t59); + tensor cat61 = ggml_concat(m, ggml_concat(m, add60, t28, 2), pool3, 2); + tensor bn62 = batch_norm_2d(m["backbone.norm_prelu_1.0"], cat61); + tensor prelu63 = prelu(m["backbone.norm_prelu_1.1"], bn62); + tensor conv64 = conv_2d(m["backbone.level2.0.conv1x1.conv"], prelu63, 2, 1); + tensor prelu65 = prelu(m["backbone.level2.0.conv1x1.activate"], conv64); + tensor dwconv66 = conv_2d_depthwise(m["backbone.level2.0.f_loc"], prelu65, 1, 1); + tensor dwconv67 = conv_2d_depthwise(m["backbone.level2.0.f_sur"], prelu65, 1, 4, 4); + tensor cat68 = ggml_concat(m, dwconv66, dwconv67, 2); + tensor bn69 = batch_norm_2d(m["backbone.level2.0.bn"], cat68); + tensor prelu70 = prelu(m["backbone.level2.0.activate"], bn69); + tensor conv71 = conv_2d(m["backbone.level2.0.bottleneck"], prelu70, 1, 0); + tensor pool72 = ggml_pool_2d(m, conv71, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs73 = ggml_reshape_2d(m, ggml_cont(m, pool72), 128, 1); + tensor fc74 = linear(m["backbone.level2.0.f_glo.fc.0"], rs73); + tensor act75 = ggml_relu(m, fc74); + tensor fc76 = linear(m["backbone.level2.0.f_glo.fc.2"], act75); + tensor act77 = ggml_sigmoid(m, fc76); + tensor rs78 = ggml_reshape_4d(m, ggml_cont(m, act77), 1, 1, 128, 1); + tensor t79 = ggml_mul(m, as_f32(m, conv71), as_f32(m, rs78)); + tensor conv80 = conv_2d(m["backbone.level2.1.conv1x1.conv"], t79, 1, 0); + tensor prelu81 = prelu(m["backbone.level2.1.conv1x1.activate"], conv80); + tensor dwconv82 = conv_2d_depthwise(m["backbone.level2.1.f_loc"], prelu81, 1, 1); + tensor dwconv83 = conv_2d_depthwise(m["backbone.level2.1.f_sur"], prelu81, 1, 4, 4); + tensor cat84 = ggml_concat(m, dwconv82, dwconv83, 2); + tensor bn85 = batch_norm_2d(m["backbone.level2.1.bn"], cat84); + tensor prelu86 = prelu(m["backbone.level2.1.activate"], bn85); + tensor pool87 = ggml_pool_2d(m, prelu86, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs88 = ggml_reshape_2d(m, ggml_cont(m, pool87), 128, 1); + tensor fc89 = linear(m["backbone.level2.1.f_glo.fc.0"], rs88); + tensor act90 = ggml_relu(m, fc89); + tensor fc91 = linear(m["backbone.level2.1.f_glo.fc.2"], act90); + tensor act92 = ggml_sigmoid(m, fc91); + tensor rs93 = ggml_reshape_4d(m, ggml_cont(m, act92), 1, 1, 128, 1); + tensor t94 = ggml_mul(m, as_f32(m, prelu86), as_f32(m, rs93)); + tensor add95 = ggml_add(m, t79, t94); + tensor conv96 = conv_2d(m["backbone.level2.2.conv1x1.conv"], add95, 1, 0); + tensor prelu97 = prelu(m["backbone.level2.2.conv1x1.activate"], conv96); + tensor dwconv98 = conv_2d_depthwise(m["backbone.level2.2.f_loc"], prelu97, 1, 1); + tensor dwconv99 = conv_2d_depthwise(m["backbone.level2.2.f_sur"], prelu97, 1, 4, 4); + tensor cat100 = ggml_concat(m, dwconv98, dwconv99, 2); + tensor bn101 = batch_norm_2d(m["backbone.level2.2.bn"], cat100); + tensor prelu102 = prelu(m["backbone.level2.2.activate"], bn101); + tensor pool103 = ggml_pool_2d(m, prelu102, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs104 = ggml_reshape_2d(m, ggml_cont(m, pool103), 128, 1); + tensor fc105 = linear(m["backbone.level2.2.f_glo.fc.0"], rs104); + tensor act106 = ggml_relu(m, fc105); + tensor fc107 = linear(m["backbone.level2.2.f_glo.fc.2"], act106); + tensor act108 = ggml_sigmoid(m, fc107); + tensor rs109 = ggml_reshape_4d(m, ggml_cont(m, act108), 1, 1, 128, 1); + tensor t110 = ggml_mul(m, as_f32(m, prelu102), as_f32(m, rs109)); + tensor add111 = ggml_add(m, add95, t110); + tensor conv112 = conv_2d(m["backbone.level2.3.conv1x1.conv"], add111, 1, 0); + tensor prelu113 = prelu(m["backbone.level2.3.conv1x1.activate"], conv112); + tensor dwconv114 = conv_2d_depthwise(m["backbone.level2.3.f_loc"], prelu113, 1, 1); + tensor dwconv115 = conv_2d_depthwise(m["backbone.level2.3.f_sur"], prelu113, 1, 4, 4); + tensor cat116 = ggml_concat(m, dwconv114, dwconv115, 2); + tensor bn117 = batch_norm_2d(m["backbone.level2.3.bn"], cat116); + tensor prelu118 = prelu(m["backbone.level2.3.activate"], bn117); + tensor pool119 = ggml_pool_2d(m, prelu118, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs120 = ggml_reshape_2d(m, ggml_cont(m, pool119), 128, 1); + tensor fc121 = linear(m["backbone.level2.3.f_glo.fc.0"], rs120); + tensor act122 = ggml_relu(m, fc121); + tensor fc123 = linear(m["backbone.level2.3.f_glo.fc.2"], act122); + tensor act124 = ggml_sigmoid(m, fc123); + tensor rs125 = ggml_reshape_4d(m, ggml_cont(m, act124), 1, 1, 128, 1); + tensor t126 = ggml_mul(m, as_f32(m, prelu118), as_f32(m, rs125)); + tensor add127 = ggml_add(m, add111, t126); + tensor conv128 = conv_2d(m["backbone.level2.4.conv1x1.conv"], add127, 1, 0); + tensor prelu129 = prelu(m["backbone.level2.4.conv1x1.activate"], conv128); + tensor dwconv130 = conv_2d_depthwise(m["backbone.level2.4.f_loc"], prelu129, 1, 1); + tensor dwconv131 = conv_2d_depthwise(m["backbone.level2.4.f_sur"], prelu129, 1, 4, 4); + tensor cat132 = ggml_concat(m, dwconv130, dwconv131, 2); + tensor bn133 = batch_norm_2d(m["backbone.level2.4.bn"], cat132); + tensor prelu134 = prelu(m["backbone.level2.4.activate"], bn133); + tensor pool135 = ggml_pool_2d(m, prelu134, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs136 = ggml_reshape_2d(m, ggml_cont(m, pool135), 128, 1); + tensor fc137 = linear(m["backbone.level2.4.f_glo.fc.0"], rs136); + tensor act138 = ggml_relu(m, fc137); + tensor fc139 = linear(m["backbone.level2.4.f_glo.fc.2"], act138); + tensor act140 = ggml_sigmoid(m, fc139); + tensor rs141 = ggml_reshape_4d(m, ggml_cont(m, act140), 1, 1, 128, 1); + tensor t142 = ggml_mul(m, as_f32(m, prelu134), as_f32(m, rs141)); + tensor add143 = ggml_add(m, add127, t142); + tensor conv144 = conv_2d(m["backbone.level2.5.conv1x1.conv"], add143, 1, 0); + tensor prelu145 = prelu(m["backbone.level2.5.conv1x1.activate"], conv144); + tensor dwconv146 = conv_2d_depthwise(m["backbone.level2.5.f_loc"], prelu145, 1, 1); + tensor dwconv147 = conv_2d_depthwise(m["backbone.level2.5.f_sur"], prelu145, 1, 4, 4); + tensor cat148 = ggml_concat(m, dwconv146, dwconv147, 2); + tensor bn149 = batch_norm_2d(m["backbone.level2.5.bn"], cat148); + tensor prelu150 = prelu(m["backbone.level2.5.activate"], bn149); + tensor pool151 = ggml_pool_2d(m, prelu150, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs152 = ggml_reshape_2d(m, ggml_cont(m, pool151), 128, 1); + tensor fc153 = linear(m["backbone.level2.5.f_glo.fc.0"], rs152); + tensor act154 = ggml_relu(m, fc153); + tensor fc155 = linear(m["backbone.level2.5.f_glo.fc.2"], act154); + tensor act156 = ggml_sigmoid(m, fc155); + tensor rs157 = ggml_reshape_4d(m, ggml_cont(m, act156), 1, 1, 128, 1); + tensor t158 = ggml_mul(m, as_f32(m, prelu150), as_f32(m, rs157)); + tensor add159 = ggml_add(m, add143, t158); + tensor conv160 = conv_2d(m["backbone.level2.6.conv1x1.conv"], add159, 1, 0); + tensor prelu161 = prelu(m["backbone.level2.6.conv1x1.activate"], conv160); + tensor dwconv162 = conv_2d_depthwise(m["backbone.level2.6.f_loc"], prelu161, 1, 1); + tensor dwconv163 = conv_2d_depthwise(m["backbone.level2.6.f_sur"], prelu161, 1, 4, 4); + tensor cat164 = ggml_concat(m, dwconv162, dwconv163, 2); + tensor bn165 = batch_norm_2d(m["backbone.level2.6.bn"], cat164); + tensor prelu166 = prelu(m["backbone.level2.6.activate"], bn165); + tensor pool167 = ggml_pool_2d(m, prelu166, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs168 = ggml_reshape_2d(m, ggml_cont(m, pool167), 128, 1); + tensor fc169 = linear(m["backbone.level2.6.f_glo.fc.0"], rs168); + tensor act170 = ggml_relu(m, fc169); + tensor fc171 = linear(m["backbone.level2.6.f_glo.fc.2"], act170); + tensor act172 = ggml_sigmoid(m, fc171); + tensor rs173 = ggml_reshape_4d(m, ggml_cont(m, act172), 1, 1, 128, 1); + tensor t174 = ggml_mul(m, as_f32(m, prelu166), as_f32(m, rs173)); + tensor add175 = ggml_add(m, add159, t174); + tensor conv176 = conv_2d(m["backbone.level2.7.conv1x1.conv"], add175, 1, 0); + tensor prelu177 = prelu(m["backbone.level2.7.conv1x1.activate"], conv176); + tensor dwconv178 = conv_2d_depthwise(m["backbone.level2.7.f_loc"], prelu177, 1, 1); + tensor dwconv179 = conv_2d_depthwise(m["backbone.level2.7.f_sur"], prelu177, 1, 4, 4); + tensor cat180 = ggml_concat(m, dwconv178, dwconv179, 2); + tensor bn181 = batch_norm_2d(m["backbone.level2.7.bn"], cat180); + tensor prelu182 = prelu(m["backbone.level2.7.activate"], bn181); + tensor pool183 = ggml_pool_2d(m, prelu182, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs184 = ggml_reshape_2d(m, ggml_cont(m, pool183), 128, 1); + tensor fc185 = linear(m["backbone.level2.7.f_glo.fc.0"], rs184); + tensor act186 = ggml_relu(m, fc185); + tensor fc187 = linear(m["backbone.level2.7.f_glo.fc.2"], act186); + tensor act188 = ggml_sigmoid(m, fc187); + tensor rs189 = ggml_reshape_4d(m, ggml_cont(m, act188), 1, 1, 128, 1); + tensor t190 = ggml_mul(m, as_f32(m, prelu182), as_f32(m, rs189)); + tensor add191 = ggml_add(m, add175, t190); + tensor conv192 = conv_2d(m["backbone.level2.8.conv1x1.conv"], add191, 1, 0); + tensor prelu193 = prelu(m["backbone.level2.8.conv1x1.activate"], conv192); + tensor dwconv194 = conv_2d_depthwise(m["backbone.level2.8.f_loc"], prelu193, 1, 1); + tensor dwconv195 = conv_2d_depthwise(m["backbone.level2.8.f_sur"], prelu193, 1, 4, 4); + tensor cat196 = ggml_concat(m, dwconv194, dwconv195, 2); + tensor bn197 = batch_norm_2d(m["backbone.level2.8.bn"], cat196); + tensor prelu198 = prelu(m["backbone.level2.8.activate"], bn197); + tensor pool199 = ggml_pool_2d(m, prelu198, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs200 = ggml_reshape_2d(m, ggml_cont(m, pool199), 128, 1); + tensor fc201 = linear(m["backbone.level2.8.f_glo.fc.0"], rs200); + tensor act202 = ggml_relu(m, fc201); + tensor fc203 = linear(m["backbone.level2.8.f_glo.fc.2"], act202); + tensor act204 = ggml_sigmoid(m, fc203); + tensor rs205 = ggml_reshape_4d(m, ggml_cont(m, act204), 1, 1, 128, 1); + tensor t206 = ggml_mul(m, as_f32(m, prelu198), as_f32(m, rs205)); + tensor add207 = ggml_add(m, add191, t206); + tensor conv208 = conv_2d(m["backbone.level2.9.conv1x1.conv"], add207, 1, 0); + tensor prelu209 = prelu(m["backbone.level2.9.conv1x1.activate"], conv208); + tensor dwconv210 = conv_2d_depthwise(m["backbone.level2.9.f_loc"], prelu209, 1, 1); + tensor dwconv211 = conv_2d_depthwise(m["backbone.level2.9.f_sur"], prelu209, 1, 4, 4); + tensor cat212 = ggml_concat(m, dwconv210, dwconv211, 2); + tensor bn213 = batch_norm_2d(m["backbone.level2.9.bn"], cat212); + tensor prelu214 = prelu(m["backbone.level2.9.activate"], bn213); + tensor pool215 = ggml_pool_2d(m, prelu214, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs216 = ggml_reshape_2d(m, ggml_cont(m, pool215), 128, 1); + tensor fc217 = linear(m["backbone.level2.9.f_glo.fc.0"], rs216); + tensor act218 = ggml_relu(m, fc217); + tensor fc219 = linear(m["backbone.level2.9.f_glo.fc.2"], act218); + tensor act220 = ggml_sigmoid(m, fc219); + tensor rs221 = ggml_reshape_4d(m, ggml_cont(m, act220), 1, 1, 128, 1); + tensor t222 = ggml_mul(m, as_f32(m, prelu214), as_f32(m, rs221)); + tensor add223 = ggml_add(m, add207, t222); + tensor conv224 = conv_2d(m["backbone.level2.10.conv1x1.conv"], add223, 1, 0); + tensor prelu225 = prelu(m["backbone.level2.10.conv1x1.activate"], conv224); + tensor dwconv226 = conv_2d_depthwise(m["backbone.level2.10.f_loc"], prelu225, 1, 1); + tensor dwconv227 = conv_2d_depthwise(m["backbone.level2.10.f_sur"], prelu225, 1, 4, 4); + tensor cat228 = ggml_concat(m, dwconv226, dwconv227, 2); + tensor bn229 = batch_norm_2d(m["backbone.level2.10.bn"], cat228); + tensor prelu230 = prelu(m["backbone.level2.10.activate"], bn229); + tensor pool231 = ggml_pool_2d(m, prelu230, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs232 = ggml_reshape_2d(m, ggml_cont(m, pool231), 128, 1); + tensor fc233 = linear(m["backbone.level2.10.f_glo.fc.0"], rs232); + tensor act234 = ggml_relu(m, fc233); + tensor fc235 = linear(m["backbone.level2.10.f_glo.fc.2"], act234); + tensor act236 = ggml_sigmoid(m, fc235); + tensor rs237 = ggml_reshape_4d(m, ggml_cont(m, act236), 1, 1, 128, 1); + tensor t238 = ggml_mul(m, as_f32(m, prelu230), as_f32(m, rs237)); + tensor add239 = ggml_add(m, add223, t238); + tensor conv240 = conv_2d(m["backbone.level2.11.conv1x1.conv"], add239, 1, 0); + tensor prelu241 = prelu(m["backbone.level2.11.conv1x1.activate"], conv240); + tensor dwconv242 = conv_2d_depthwise(m["backbone.level2.11.f_loc"], prelu241, 1, 1); + tensor dwconv243 = conv_2d_depthwise(m["backbone.level2.11.f_sur"], prelu241, 1, 4, 4); + tensor cat244 = ggml_concat(m, dwconv242, dwconv243, 2); + tensor bn245 = batch_norm_2d(m["backbone.level2.11.bn"], cat244); + tensor prelu246 = prelu(m["backbone.level2.11.activate"], bn245); + tensor pool247 = ggml_pool_2d(m, prelu246, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs248 = ggml_reshape_2d(m, ggml_cont(m, pool247), 128, 1); + tensor fc249 = linear(m["backbone.level2.11.f_glo.fc.0"], rs248); + tensor act250 = ggml_relu(m, fc249); + tensor fc251 = linear(m["backbone.level2.11.f_glo.fc.2"], act250); + tensor act252 = ggml_sigmoid(m, fc251); + tensor rs253 = ggml_reshape_4d(m, ggml_cont(m, act252), 1, 1, 128, 1); + tensor t254 = ggml_mul(m, as_f32(m, prelu246), as_f32(m, rs253)); + tensor add255 = ggml_add(m, add239, t254); + tensor conv256 = conv_2d(m["backbone.level2.12.conv1x1.conv"], add255, 1, 0); + tensor prelu257 = prelu(m["backbone.level2.12.conv1x1.activate"], conv256); + tensor dwconv258 = conv_2d_depthwise(m["backbone.level2.12.f_loc"], prelu257, 1, 1); + tensor dwconv259 = conv_2d_depthwise(m["backbone.level2.12.f_sur"], prelu257, 1, 4, 4); + tensor cat260 = ggml_concat(m, dwconv258, dwconv259, 2); + tensor bn261 = batch_norm_2d(m["backbone.level2.12.bn"], cat260); + tensor prelu262 = prelu(m["backbone.level2.12.activate"], bn261); + tensor pool263 = ggml_pool_2d(m, prelu262, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs264 = ggml_reshape_2d(m, ggml_cont(m, pool263), 128, 1); + tensor fc265 = linear(m["backbone.level2.12.f_glo.fc.0"], rs264); + tensor act266 = ggml_relu(m, fc265); + tensor fc267 = linear(m["backbone.level2.12.f_glo.fc.2"], act266); + tensor act268 = ggml_sigmoid(m, fc267); + tensor rs269 = ggml_reshape_4d(m, ggml_cont(m, act268), 1, 1, 128, 1); + tensor t270 = ggml_mul(m, as_f32(m, prelu262), as_f32(m, rs269)); + tensor add271 = ggml_add(m, add255, t270); + tensor conv272 = conv_2d(m["backbone.level2.13.conv1x1.conv"], add271, 1, 0); + tensor prelu273 = prelu(m["backbone.level2.13.conv1x1.activate"], conv272); + tensor dwconv274 = conv_2d_depthwise(m["backbone.level2.13.f_loc"], prelu273, 1, 1); + tensor dwconv275 = conv_2d_depthwise(m["backbone.level2.13.f_sur"], prelu273, 1, 4, 4); + tensor cat276 = ggml_concat(m, dwconv274, dwconv275, 2); + tensor bn277 = batch_norm_2d(m["backbone.level2.13.bn"], cat276); + tensor prelu278 = prelu(m["backbone.level2.13.activate"], bn277); + tensor pool279 = ggml_pool_2d(m, prelu278, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs280 = ggml_reshape_2d(m, ggml_cont(m, pool279), 128, 1); + tensor fc281 = linear(m["backbone.level2.13.f_glo.fc.0"], rs280); + tensor act282 = ggml_relu(m, fc281); + tensor fc283 = linear(m["backbone.level2.13.f_glo.fc.2"], act282); + tensor act284 = ggml_sigmoid(m, fc283); + tensor rs285 = ggml_reshape_4d(m, ggml_cont(m, act284), 1, 1, 128, 1); + tensor t286 = ggml_mul(m, as_f32(m, prelu278), as_f32(m, rs285)); + tensor add287 = ggml_add(m, add271, t286); + tensor conv288 = conv_2d(m["backbone.level2.14.conv1x1.conv"], add287, 1, 0); + tensor prelu289 = prelu(m["backbone.level2.14.conv1x1.activate"], conv288); + tensor dwconv290 = conv_2d_depthwise(m["backbone.level2.14.f_loc"], prelu289, 1, 1); + tensor dwconv291 = conv_2d_depthwise(m["backbone.level2.14.f_sur"], prelu289, 1, 4, 4); + tensor cat292 = ggml_concat(m, dwconv290, dwconv291, 2); + tensor bn293 = batch_norm_2d(m["backbone.level2.14.bn"], cat292); + tensor prelu294 = prelu(m["backbone.level2.14.activate"], bn293); + tensor pool295 = ggml_pool_2d(m, prelu294, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs296 = ggml_reshape_2d(m, ggml_cont(m, pool295), 128, 1); + tensor fc297 = linear(m["backbone.level2.14.f_glo.fc.0"], rs296); + tensor act298 = ggml_relu(m, fc297); + tensor fc299 = linear(m["backbone.level2.14.f_glo.fc.2"], act298); + tensor act300 = ggml_sigmoid(m, fc299); + tensor rs301 = ggml_reshape_4d(m, ggml_cont(m, act300), 1, 1, 128, 1); + tensor t302 = ggml_mul(m, as_f32(m, prelu294), as_f32(m, rs301)); + tensor add303 = ggml_add(m, add287, t302); + tensor conv304 = conv_2d(m["backbone.level2.15.conv1x1.conv"], add303, 1, 0); + tensor prelu305 = prelu(m["backbone.level2.15.conv1x1.activate"], conv304); + tensor dwconv306 = conv_2d_depthwise(m["backbone.level2.15.f_loc"], prelu305, 1, 1); + tensor dwconv307 = conv_2d_depthwise(m["backbone.level2.15.f_sur"], prelu305, 1, 4, 4); + tensor cat308 = ggml_concat(m, dwconv306, dwconv307, 2); + tensor bn309 = batch_norm_2d(m["backbone.level2.15.bn"], cat308); + tensor prelu310 = prelu(m["backbone.level2.15.activate"], bn309); + tensor pool311 = ggml_pool_2d(m, prelu310, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs312 = ggml_reshape_2d(m, ggml_cont(m, pool311), 128, 1); + tensor fc313 = linear(m["backbone.level2.15.f_glo.fc.0"], rs312); + tensor act314 = ggml_relu(m, fc313); + tensor fc315 = linear(m["backbone.level2.15.f_glo.fc.2"], act314); + tensor act316 = ggml_sigmoid(m, fc315); + tensor rs317 = ggml_reshape_4d(m, ggml_cont(m, act316), 1, 1, 128, 1); + tensor t318 = ggml_mul(m, as_f32(m, prelu310), as_f32(m, rs317)); + tensor add319 = ggml_add(m, add303, t318); + tensor conv320 = conv_2d(m["backbone.level2.16.conv1x1.conv"], add319, 1, 0); + tensor prelu321 = prelu(m["backbone.level2.16.conv1x1.activate"], conv320); + tensor dwconv322 = conv_2d_depthwise(m["backbone.level2.16.f_loc"], prelu321, 1, 1); + tensor dwconv323 = conv_2d_depthwise(m["backbone.level2.16.f_sur"], prelu321, 1, 4, 4); + tensor cat324 = ggml_concat(m, dwconv322, dwconv323, 2); + tensor bn325 = batch_norm_2d(m["backbone.level2.16.bn"], cat324); + tensor prelu326 = prelu(m["backbone.level2.16.activate"], bn325); + tensor pool327 = ggml_pool_2d(m, prelu326, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs328 = ggml_reshape_2d(m, ggml_cont(m, pool327), 128, 1); + tensor fc329 = linear(m["backbone.level2.16.f_glo.fc.0"], rs328); + tensor act330 = ggml_relu(m, fc329); + tensor fc331 = linear(m["backbone.level2.16.f_glo.fc.2"], act330); + tensor act332 = ggml_sigmoid(m, fc331); + tensor rs333 = ggml_reshape_4d(m, ggml_cont(m, act332), 1, 1, 128, 1); + tensor t334 = ggml_mul(m, as_f32(m, prelu326), as_f32(m, rs333)); + tensor add335 = ggml_add(m, add319, t334); + tensor conv336 = conv_2d(m["backbone.level2.17.conv1x1.conv"], add335, 1, 0); + tensor prelu337 = prelu(m["backbone.level2.17.conv1x1.activate"], conv336); + tensor dwconv338 = conv_2d_depthwise(m["backbone.level2.17.f_loc"], prelu337, 1, 1); + tensor dwconv339 = conv_2d_depthwise(m["backbone.level2.17.f_sur"], prelu337, 1, 4, 4); + tensor cat340 = ggml_concat(m, dwconv338, dwconv339, 2); + tensor bn341 = batch_norm_2d(m["backbone.level2.17.bn"], cat340); + tensor prelu342 = prelu(m["backbone.level2.17.activate"], bn341); + tensor pool343 = ggml_pool_2d(m, prelu342, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs344 = ggml_reshape_2d(m, ggml_cont(m, pool343), 128, 1); + tensor fc345 = linear(m["backbone.level2.17.f_glo.fc.0"], rs344); + tensor act346 = ggml_relu(m, fc345); + tensor fc347 = linear(m["backbone.level2.17.f_glo.fc.2"], act346); + tensor act348 = ggml_sigmoid(m, fc347); + tensor rs349 = ggml_reshape_4d(m, ggml_cont(m, act348), 1, 1, 128, 1); + tensor t350 = ggml_mul(m, as_f32(m, prelu342), as_f32(m, rs349)); + tensor add351 = ggml_add(m, add335, t350); + tensor conv352 = conv_2d(m["backbone.level2.18.conv1x1.conv"], add351, 1, 0); + tensor prelu353 = prelu(m["backbone.level2.18.conv1x1.activate"], conv352); + tensor dwconv354 = conv_2d_depthwise(m["backbone.level2.18.f_loc"], prelu353, 1, 1); + tensor dwconv355 = conv_2d_depthwise(m["backbone.level2.18.f_sur"], prelu353, 1, 4, 4); + tensor cat356 = ggml_concat(m, dwconv354, dwconv355, 2); + tensor bn357 = batch_norm_2d(m["backbone.level2.18.bn"], cat356); + tensor prelu358 = prelu(m["backbone.level2.18.activate"], bn357); + tensor pool359 = ggml_pool_2d(m, prelu358, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs360 = ggml_reshape_2d(m, ggml_cont(m, pool359), 128, 1); + tensor fc361 = linear(m["backbone.level2.18.f_glo.fc.0"], rs360); + tensor act362 = ggml_relu(m, fc361); + tensor fc363 = linear(m["backbone.level2.18.f_glo.fc.2"], act362); + tensor act364 = ggml_sigmoid(m, fc363); + tensor rs365 = ggml_reshape_4d(m, ggml_cont(m, act364), 1, 1, 128, 1); + tensor t366 = ggml_mul(m, as_f32(m, prelu358), as_f32(m, rs365)); + tensor add367 = ggml_add(m, add351, t366); + tensor conv368 = conv_2d(m["backbone.level2.19.conv1x1.conv"], add367, 1, 0); + tensor prelu369 = prelu(m["backbone.level2.19.conv1x1.activate"], conv368); + tensor dwconv370 = conv_2d_depthwise(m["backbone.level2.19.f_loc"], prelu369, 1, 1); + tensor dwconv371 = conv_2d_depthwise(m["backbone.level2.19.f_sur"], prelu369, 1, 4, 4); + tensor cat372 = ggml_concat(m, dwconv370, dwconv371, 2); + tensor bn373 = batch_norm_2d(m["backbone.level2.19.bn"], cat372); + tensor prelu374 = prelu(m["backbone.level2.19.activate"], bn373); + tensor pool375 = ggml_pool_2d(m, prelu374, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs376 = ggml_reshape_2d(m, ggml_cont(m, pool375), 128, 1); + tensor fc377 = linear(m["backbone.level2.19.f_glo.fc.0"], rs376); + tensor act378 = ggml_relu(m, fc377); + tensor fc379 = linear(m["backbone.level2.19.f_glo.fc.2"], act378); + tensor act380 = ggml_sigmoid(m, fc379); + tensor rs381 = ggml_reshape_4d(m, ggml_cont(m, act380), 1, 1, 128, 1); + tensor t382 = ggml_mul(m, as_f32(m, prelu374), as_f32(m, rs381)); + tensor add383 = ggml_add(m, add367, t382); + tensor conv384 = conv_2d(m["backbone.level2.20.conv1x1.conv"], add383, 1, 0); + tensor prelu385 = prelu(m["backbone.level2.20.conv1x1.activate"], conv384); + tensor dwconv386 = conv_2d_depthwise(m["backbone.level2.20.f_loc"], prelu385, 1, 1); + tensor dwconv387 = conv_2d_depthwise(m["backbone.level2.20.f_sur"], prelu385, 1, 4, 4); + tensor cat388 = ggml_concat(m, dwconv386, dwconv387, 2); + tensor bn389 = batch_norm_2d(m["backbone.level2.20.bn"], cat388); + tensor prelu390 = prelu(m["backbone.level2.20.activate"], bn389); + tensor pool391 = ggml_pool_2d(m, prelu390, GGML_OP_POOL_AVG, 85, 85, 85, 85, 0, 0); + tensor rs392 = ggml_reshape_2d(m, ggml_cont(m, pool391), 128, 1); + tensor fc393 = linear(m["backbone.level2.20.f_glo.fc.0"], rs392); + tensor act394 = ggml_relu(m, fc393); + tensor fc395 = linear(m["backbone.level2.20.f_glo.fc.2"], act394); + tensor act396 = ggml_sigmoid(m, fc395); + tensor rs397 = ggml_reshape_4d(m, ggml_cont(m, act396), 1, 1, 128, 1); + tensor t398 = ggml_mul(m, as_f32(m, prelu390), as_f32(m, rs397)); + tensor add399 = ggml_add(m, add383, t398); + tensor cat400 = ggml_concat(m, t79, add399, 2); + tensor bn401 = batch_norm_2d(m["backbone.norm_prelu_2.0"], cat400); + tensor prelu402 = prelu(m["backbone.norm_prelu_2.1"], bn401); + tensor conv403 = conv_2d(m["decode_head.conv_seg"], prelu402, 1, 0); + + tensor out_0 = compute_graph_output(m, contiguous_2d_to_cwhn(m, conv403), "out_0"); + return out_0; +} + +FamCgnet_params FamCgnet_detect_params(model_file const& f) { + FamCgnet_params p{}; + // GGUF general.architecture 검증 (모델은 정적 unroll 이라 추가 하이퍼파라미터 없음). + if (std::string_view arch = f.arch(); arch != "famcgnet") { + throw except( + "Architecture expected to be 'famcgnet', but was '{}' ({})", + arch, f.path); + } + return p; +} + +} // namespace visp diff --git a/src/visp/arch/FamCgnet.h b/src/visp/arch/FamCgnet.h new file mode 100644 index 0000000..3a5fb1b --- /dev/null +++ b/src/visp/arch/FamCgnet.h @@ -0,0 +1,14 @@ +// GENERATED BY SuperGate GTX Compiler (ggml/vision.cpp backend), DO NOT EDIT! +#pragma once +#include "visp/ml.h" + +namespace visp { + +struct FamCgnet_params { + // TODO(ggml): 모델 하이퍼파라미터 (block_count 등) +}; + +tensor FamCgnet_forward(model_ref m, tensor x, FamCgnet_params const& p); +FamCgnet_params FamCgnet_detect_params(model_file const& f); + +} // namespace visp diff --git a/src/visp/arch/FamDeeplabv3.cpp b/src/visp/arch/FamDeeplabv3.cpp new file mode 100644 index 0000000..55efe2a --- /dev/null +++ b/src/visp/arch/FamDeeplabv3.cpp @@ -0,0 +1,174 @@ +// GENERATED BY SuperGate GTX Compiler (ggml/vision.cpp backend), DO NOT EDIT! +#include "visp/arch/FamDeeplabv3.h" +#include "visp/ml.h" +#include "visp/nn.h" +#include "visp/vision.h" +#include "util/string.h" +#include +#include // clamp 의 한쪽 경계가 없으면 ±INFINITY 를 낸다 + +#include + +namespace visp { + +tensor FamDeeplabv3_forward(model_ref m, tensor x, FamDeeplabv3_params const& p) { + (void)p; + // 입력 레이아웃 변환 (TODO: cwhn/whcn 자동 판별) + x = cwhn_to_contiguous_2d(m, x); + + tensor conv1 = conv_2d(m["backbone.stem.0"], x, 2, 1); + tensor act2 = ggml_relu(m, conv1); + tensor conv3 = conv_2d(m["backbone.stem.3"], act2, 1, 1); + tensor act4 = ggml_relu(m, conv3); + tensor conv5 = conv_2d(m["backbone.stem.6"], act4, 1, 1); + tensor act6 = ggml_relu(m, conv5); + tensor pool7 = ggml_pool_2d(m, act6, GGML_OP_POOL_MAX, 3, 3, 2, 2, 1, 1); + tensor conv8 = conv_2d(m["backbone.layer1.0.conv1"], pool7, 1, 0); + tensor act9 = ggml_relu(m, conv8); + tensor conv10 = conv_2d(m["backbone.layer1.0.conv2"], act9, 1, 1); + tensor act11 = ggml_relu(m, conv10); + tensor conv12 = conv_2d(m["backbone.layer1.0.conv3"], act11, 1, 0); + tensor conv13 = conv_2d(m["backbone.layer1.0.downsample.0"], pool7, 1, 0); + tensor add14 = ggml_add(m, conv12, conv13); + tensor act15 = ggml_relu(m, add14); + tensor conv16 = conv_2d(m["backbone.layer1.1.conv1"], act15, 1, 0); + tensor act17 = ggml_relu(m, conv16); + tensor conv18 = conv_2d(m["backbone.layer1.1.conv2"], act17, 1, 1); + tensor act19 = ggml_relu(m, conv18); + tensor conv20 = conv_2d(m["backbone.layer1.1.conv3"], act19, 1, 0); + tensor add21 = ggml_add(m, conv20, act15); + tensor act22 = ggml_relu(m, add21); + tensor conv23 = conv_2d(m["backbone.layer1.2.conv1"], act22, 1, 0); + tensor act24 = ggml_relu(m, conv23); + tensor conv25 = conv_2d(m["backbone.layer1.2.conv2"], act24, 1, 1); + tensor act26 = ggml_relu(m, conv25); + tensor conv27 = conv_2d(m["backbone.layer1.2.conv3"], act26, 1, 0); + tensor add28 = ggml_add(m, conv27, act22); + tensor act29 = ggml_relu(m, add28); + tensor conv30 = conv_2d(m["backbone.layer2.0.conv1"], act29, 1, 0); + tensor act31 = ggml_relu(m, conv30); + tensor conv32 = conv_2d(m["backbone.layer2.0.conv2"], act31, 2, 1); + tensor act33 = ggml_relu(m, conv32); + tensor conv34 = conv_2d(m["backbone.layer2.0.conv3"], act33, 1, 0); + tensor conv35 = conv_2d(m["backbone.layer2.0.downsample.0"], act29, 2, 0); + tensor add36 = ggml_add(m, conv34, conv35); + tensor act37 = ggml_relu(m, add36); + tensor conv38 = conv_2d(m["backbone.layer2.1.conv1"], act37, 1, 0); + tensor act39 = ggml_relu(m, conv38); + tensor conv40 = conv_2d(m["backbone.layer2.1.conv2"], act39, 1, 1); + tensor act41 = ggml_relu(m, conv40); + tensor conv42 = conv_2d(m["backbone.layer2.1.conv3"], act41, 1, 0); + tensor add43 = ggml_add(m, conv42, act37); + tensor act44 = ggml_relu(m, add43); + tensor conv45 = conv_2d(m["backbone.layer2.2.conv1"], act44, 1, 0); + tensor act46 = ggml_relu(m, conv45); + tensor conv47 = conv_2d(m["backbone.layer2.2.conv2"], act46, 1, 1); + tensor act48 = ggml_relu(m, conv47); + tensor conv49 = conv_2d(m["backbone.layer2.2.conv3"], act48, 1, 0); + tensor add50 = ggml_add(m, conv49, act44); + tensor act51 = ggml_relu(m, add50); + tensor conv52 = conv_2d(m["backbone.layer2.3.conv1"], act51, 1, 0); + tensor act53 = ggml_relu(m, conv52); + tensor conv54 = conv_2d(m["backbone.layer2.3.conv2"], act53, 1, 1); + tensor act55 = ggml_relu(m, conv54); + tensor conv56 = conv_2d(m["backbone.layer2.3.conv3"], act55, 1, 0); + tensor add57 = ggml_add(m, conv56, act51); + tensor act58 = ggml_relu(m, add57); + tensor conv59 = conv_2d(m["backbone.layer3.0.conv1"], act58, 1, 0); + tensor act60 = ggml_relu(m, conv59); + tensor conv61 = conv_2d(m["backbone.layer3.0.conv2"], act60, 1, 1); + tensor act62 = ggml_relu(m, conv61); + tensor conv63 = conv_2d(m["backbone.layer3.0.conv3"], act62, 1, 0); + tensor conv64 = conv_2d(m["backbone.layer3.0.downsample.0"], act58, 1, 0); + tensor add65 = ggml_add(m, conv63, conv64); + tensor act66 = ggml_relu(m, add65); + tensor conv67 = conv_2d(m["backbone.layer3.1.conv1"], act66, 1, 0); + tensor act68 = ggml_relu(m, conv67); + tensor conv69 = conv_2d(m["backbone.layer3.1.conv2"], act68, 1, 2, 2); + tensor act70 = ggml_relu(m, conv69); + tensor conv71 = conv_2d(m["backbone.layer3.1.conv3"], act70, 1, 0); + tensor add72 = ggml_add(m, conv71, act66); + tensor act73 = ggml_relu(m, add72); + tensor conv74 = conv_2d(m["backbone.layer3.2.conv1"], act73, 1, 0); + tensor act75 = ggml_relu(m, conv74); + tensor conv76 = conv_2d(m["backbone.layer3.2.conv2"], act75, 1, 2, 2); + tensor act77 = ggml_relu(m, conv76); + tensor conv78 = conv_2d(m["backbone.layer3.2.conv3"], act77, 1, 0); + tensor add79 = ggml_add(m, conv78, act73); + tensor act80 = ggml_relu(m, add79); + tensor conv81 = conv_2d(m["backbone.layer3.3.conv1"], act80, 1, 0); + tensor act82 = ggml_relu(m, conv81); + tensor conv83 = conv_2d(m["backbone.layer3.3.conv2"], act82, 1, 2, 2); + tensor act84 = ggml_relu(m, conv83); + tensor conv85 = conv_2d(m["backbone.layer3.3.conv3"], act84, 1, 0); + tensor add86 = ggml_add(m, conv85, act80); + tensor act87 = ggml_relu(m, add86); + tensor conv88 = conv_2d(m["backbone.layer3.4.conv1"], act87, 1, 0); + tensor act89 = ggml_relu(m, conv88); + tensor conv90 = conv_2d(m["backbone.layer3.4.conv2"], act89, 1, 2, 2); + tensor act91 = ggml_relu(m, conv90); + tensor conv92 = conv_2d(m["backbone.layer3.4.conv3"], act91, 1, 0); + tensor add93 = ggml_add(m, conv92, act87); + tensor act94 = ggml_relu(m, add93); + tensor conv95 = conv_2d(m["backbone.layer3.5.conv1"], act94, 1, 0); + tensor act96 = ggml_relu(m, conv95); + tensor conv97 = conv_2d(m["backbone.layer3.5.conv2"], act96, 1, 2, 2); + tensor act98 = ggml_relu(m, conv97); + tensor conv99 = conv_2d(m["backbone.layer3.5.conv3"], act98, 1, 0); + tensor add100 = ggml_add(m, conv99, act94); + tensor act101 = ggml_relu(m, add100); + tensor conv102 = conv_2d(m["backbone.layer4.0.conv1"], act101, 1, 0); + tensor act103 = ggml_relu(m, conv102); + tensor conv104 = conv_2d(m["backbone.layer4.0.conv2"], act103, 1, 2, 2); + tensor act105 = ggml_relu(m, conv104); + tensor conv106 = conv_2d(m["backbone.layer4.0.conv3"], act105, 1, 0); + tensor conv107 = conv_2d(m["backbone.layer4.0.downsample.0"], act101, 1, 0); + tensor add108 = ggml_add(m, conv106, conv107); + tensor act109 = ggml_relu(m, add108); + tensor conv110 = conv_2d(m["backbone.layer4.1.conv1"], act109, 1, 0); + tensor act111 = ggml_relu(m, conv110); + tensor conv112 = conv_2d(m["backbone.layer4.1.conv2"], act111, 1, 4, 4); + tensor act113 = ggml_relu(m, conv112); + tensor conv114 = conv_2d(m["backbone.layer4.1.conv3"], act113, 1, 0); + tensor add115 = ggml_add(m, conv114, act109); + tensor act116 = ggml_relu(m, add115); + tensor conv117 = conv_2d(m["backbone.layer4.2.conv1"], act116, 1, 0); + tensor act118 = ggml_relu(m, conv117); + tensor conv119 = conv_2d(m["backbone.layer4.2.conv2"], act118, 1, 4, 4); + tensor act120 = ggml_relu(m, conv119); + tensor conv121 = conv_2d(m["backbone.layer4.2.conv3"], act120, 1, 0); + tensor add122 = ggml_add(m, conv121, act116); + tensor act123 = ggml_relu(m, add122); + tensor pool124 = ggml_pool_2d(m, act123, GGML_OP_POOL_AVG, 128, 64, 128, 64, 0, 0); + tensor conv125 = conv_2d(m["decode_head.image_pool.1.conv"], pool124, 1, 0); + tensor act126 = ggml_relu(m, conv125); + tensor up127 = ggml_interpolate(m, act126, 128, 64, 512, 1, GGML_SCALE_MODE_BILINEAR) /* interpolate -> target ne, mode=bilinear */; + tensor conv128 = conv_2d(m["decode_head.aspp_modules.0.conv"], act123, 1, 0); + tensor act129 = ggml_relu(m, conv128); + tensor conv130 = conv_2d(m["decode_head.aspp_modules.1.conv"], act123, 1, 12, 12); + tensor act131 = ggml_relu(m, conv130); + tensor conv132 = conv_2d(m["decode_head.aspp_modules.2.conv"], act123, 1, 24, 24); + tensor act133 = ggml_relu(m, conv132); + tensor conv134 = conv_2d(m["decode_head.aspp_modules.3.conv"], act123, 1, 36, 36); + tensor act135 = ggml_relu(m, conv134); + tensor cat136 = ggml_concat(m, ggml_concat(m, ggml_concat(m, ggml_concat(m, up127, act129, 2), act131, 2), act133, 2), act135, 2); + tensor conv137 = conv_2d(m["decode_head.bottleneck.conv"], cat136, 1, 1); + tensor act138 = ggml_relu(m, conv137); + tensor conv139 = conv_2d(m["decode_head.conv_seg"], act138, 1, 0); + + tensor out_0 = compute_graph_output(m, contiguous_2d_to_cwhn(m, conv139), "out_0"); + return out_0; +} + +FamDeeplabv3_params FamDeeplabv3_detect_params(model_file const& f) { + FamDeeplabv3_params p{}; + // GGUF general.architecture 검증 (모델은 정적 unroll 이라 추가 하이퍼파라미터 없음). + if (std::string_view arch = f.arch(); arch != "famdeeplabv3") { + throw except( + "Architecture expected to be 'famdeeplabv3', but was '{}' ({})", + arch, f.path); + } + return p; +} + +} // namespace visp diff --git a/src/visp/arch/FamDeeplabv3.h b/src/visp/arch/FamDeeplabv3.h new file mode 100644 index 0000000..868d13b --- /dev/null +++ b/src/visp/arch/FamDeeplabv3.h @@ -0,0 +1,14 @@ +// GENERATED BY SuperGate GTX Compiler (ggml/vision.cpp backend), DO NOT EDIT! +#pragma once +#include "visp/ml.h" + +namespace visp { + +struct FamDeeplabv3_params { + // TODO(ggml): 모델 하이퍼파라미터 (block_count 등) +}; + +tensor FamDeeplabv3_forward(model_ref m, tensor x, FamDeeplabv3_params const& p); +FamDeeplabv3_params FamDeeplabv3_detect_params(model_file const& f); + +} // namespace visp diff --git a/src/visp/arch/ResNet.cpp b/src/visp/arch/ResNet.cpp new file mode 100644 index 0000000..63dde78 --- /dev/null +++ b/src/visp/arch/ResNet.cpp @@ -0,0 +1,83 @@ +// GENERATED BY SuperGate GTX Compiler (ggml/vision.cpp backend), DO NOT EDIT! +#include "visp/arch/ResNet.h" +#include "visp/ml.h" +#include "visp/nn.h" +#include "visp/vision.h" +#include "util/string.h" +#include + +#include + +namespace visp { + +tensor ResNet_forward(model_ref m, tensor x, ResNet_params const& p) { + (void)p; + // 입력 레이아웃 변환 (TODO: cwhn/whcn 자동 판별) + x = cwhn_to_contiguous_2d(m, x); + + tensor conv1 = conv_2d(m["conv1"], x, 2, 3); + tensor act2 = ggml_relu(m, conv1); + tensor pool3 = ggml_pool_2d(m, act2, GGML_OP_POOL_MAX, 3, 3, 2, 2, 1, 1); + tensor conv4 = conv_2d(m["layer1.0.conv1"], pool3, 1, 1); + tensor act5 = ggml_relu(m, conv4); + tensor conv6 = conv_2d(m["layer1.0.conv2"], act5, 1, 1); + tensor add7 = ggml_add(m, conv6, pool3); + tensor act8 = ggml_relu(m, add7); + tensor conv9 = conv_2d(m["layer1.1.conv1"], act8, 1, 1); + tensor act10 = ggml_relu(m, conv9); + tensor conv11 = conv_2d(m["layer1.1.conv2"], act10, 1, 1); + tensor add12 = ggml_add(m, conv11, act8); + tensor act13 = ggml_relu(m, add12); + tensor conv14 = conv_2d(m["layer2.0.conv1"], act13, 2, 1); + tensor act15 = ggml_relu(m, conv14); + tensor conv16 = conv_2d(m["layer2.0.conv2"], act15, 1, 1); + tensor conv17 = conv_2d(m["layer2.0.downsample.0"], act13, 2, 0); + tensor add18 = ggml_add(m, conv16, conv17); + tensor act19 = ggml_relu(m, add18); + tensor conv20 = conv_2d(m["layer2.1.conv1"], act19, 1, 1); + tensor act21 = ggml_relu(m, conv20); + tensor conv22 = conv_2d(m["layer2.1.conv2"], act21, 1, 1); + tensor add23 = ggml_add(m, conv22, act19); + tensor act24 = ggml_relu(m, add23); + tensor conv25 = conv_2d(m["layer3.0.conv1"], act24, 2, 1); + tensor act26 = ggml_relu(m, conv25); + tensor conv27 = conv_2d(m["layer3.0.conv2"], act26, 1, 1); + tensor conv28 = conv_2d(m["layer3.0.downsample.0"], act24, 2, 0); + tensor add29 = ggml_add(m, conv27, conv28); + tensor act30 = ggml_relu(m, add29); + tensor conv31 = conv_2d(m["layer3.1.conv1"], act30, 1, 1); + tensor act32 = ggml_relu(m, conv31); + tensor conv33 = conv_2d(m["layer3.1.conv2"], act32, 1, 1); + tensor add34 = ggml_add(m, conv33, act30); + tensor act35 = ggml_relu(m, add34); + tensor conv36 = conv_2d(m["layer4.0.conv1"], act35, 2, 1); + tensor act37 = ggml_relu(m, conv36); + tensor conv38 = conv_2d(m["layer4.0.conv2"], act37, 1, 1); + tensor conv39 = conv_2d(m["layer4.0.downsample.0"], act35, 2, 0); + tensor add40 = ggml_add(m, conv38, conv39); + tensor act41 = ggml_relu(m, add40); + tensor conv42 = conv_2d(m["layer4.1.conv1"], act41, 1, 1); + tensor act43 = ggml_relu(m, conv42); + tensor conv44 = conv_2d(m["layer4.1.conv2"], act43, 1, 1); + tensor add45 = ggml_add(m, conv44, act41); + tensor act46 = ggml_relu(m, add45); + tensor pool47 = ggml_pool_2d(m, act46, GGML_OP_POOL_AVG, 7, 7, 7, 7, 0, 0); + tensor flat48 = ggml_reshape_2d(m, ggml_cont(m, pool47), 512, 1); + tensor fc49 = linear(m["fc"], flat48); + + tensor out_0 = compute_graph_output(m, contiguous_2d_to_cwhn(m, fc49), "out_0"); + return out_0; +} + +ResNet_params ResNet_detect_params(model_file const& f) { + ResNet_params p{}; + // GGUF general.architecture 검증 (모델은 정적 unroll 이라 추가 하이퍼파라미터 없음). + if (std::string_view arch = f.arch(); arch != "resnet") { + throw except( + "Architecture expected to be 'resnet', but was '{}' ({})", + arch, f.path); + } + return p; +} + +} // namespace visp diff --git a/src/visp/arch/ResNet.h b/src/visp/arch/ResNet.h new file mode 100644 index 0000000..fb2f1af --- /dev/null +++ b/src/visp/arch/ResNet.h @@ -0,0 +1,14 @@ +// GENERATED BY SuperGate GTX Compiler (ggml/vision.cpp backend), DO NOT EDIT! +#pragma once +#include "visp/ml.h" + +namespace visp { + +struct ResNet_params { + // TODO(ggml): 모델 하이퍼파라미터 (block_count 등) +}; + +tensor ResNet_forward(model_ref m, tensor x, ResNet_params const& p); +ResNet_params ResNet_detect_params(model_file const& f); + +} // namespace visp diff --git a/src/visp/arch/Yolo26m.cpp b/src/visp/arch/Yolo26m.cpp deleted file mode 100644 index 15f7229..0000000 --- a/src/visp/arch/Yolo26m.cpp +++ /dev/null @@ -1,436 +0,0 @@ -// GENERATED BY SuperGate GTX Compiler (ggml/vision.cpp backend), DO NOT EDIT! -#include "visp/arch/Yolo26m.h" -#include "visp/ml.h" -#include "visp/nn.h" -#include "visp/vision.h" -#include "util/string.h" -#include - -#include - -namespace visp { - -tensor Yolo26m_forward(model_ref m, tensor x, Yolo26m_params const& p) { - (void)p; - // 입력 레이아웃 변환 (TODO: cwhn/whcn 자동 판별) - x = cwhn_to_contiguous_2d(m, x); - - tensor const1 = ggml_arange(m, 0.5f, 1.0f, 1.0f) /* const 0.5 */; - tensor const3 = ggml_arange(m, 0.5f, 1.0f, 1.0f) /* const 0.5 */; - tensor const5 = ggml_arange(m, 0.5f, 1.0f, 1.0f) /* const 0.5 */; - tensor const7 = ggml_arange(m, 0.1767766922712326f, 0.6767766922712326f, 1.0f) /* const 0.1767766922712326 */; - tensor const8 = ggml_arange(m, 0.1767766922712326f, 0.6767766922712326f, 1.0f) /* const 0.1767766922712326 */; - tensor conv9 = conv_2d(m["model.0.conv"], x, 2, 1); - tensor act10 = ggml_silu_inplace(m, conv9); - tensor conv11 = conv_2d(m["model.1.conv"], act10, 2, 1); - tensor act12 = ggml_silu_inplace(m, conv11); - tensor conv13 = conv_2d(m["model.2.cv1.conv"], act12, 1, 0); - tensor act14 = ggml_silu_inplace(m, conv13); - tensor sl15 = ggml_view_4d(m, act14, 160, 160, 64, 1, act14->nb[1], act14->nb[2], act14->nb[3], 0); - tensor sl16 = ggml_view_4d(m, act14, 160, 160, 64, 1, act14->nb[1], act14->nb[2], act14->nb[3], 64*act14->nb[2]); - tensor conv17 = conv_2d(m["model.2.m.0.cv1.conv"], sl16, 1, 0); - tensor act18 = ggml_silu_inplace(m, conv17); - tensor conv19 = conv_2d(m["model.2.m.0.m.0.cv1.conv"], act18, 1, 1); - tensor act20 = ggml_silu_inplace(m, conv19); - tensor conv21 = conv_2d(m["model.2.m.0.m.0.cv2.conv"], act20, 1, 1); - tensor act22 = ggml_silu_inplace(m, conv21); - tensor add23 = ggml_add(m, act18, act22); - tensor conv24 = conv_2d(m["model.2.m.0.m.1.cv1.conv"], add23, 1, 1); - tensor act25 = ggml_silu_inplace(m, conv24); - tensor conv26 = conv_2d(m["model.2.m.0.m.1.cv2.conv"], act25, 1, 1); - tensor act27 = ggml_silu_inplace(m, conv26); - tensor add28 = ggml_add(m, add23, act27); - tensor conv29 = conv_2d(m["model.2.m.0.cv2.conv"], sl16, 1, 0); - tensor act30 = ggml_silu_inplace(m, conv29); - tensor cat31 = ggml_concat(m, add28, act30, 2); - tensor conv32 = conv_2d(m["model.2.m.0.cv3.conv"], cat31, 1, 0); - tensor act33 = ggml_silu_inplace(m, conv32); - tensor cat34 = ggml_concat(m, ggml_concat(m, sl15, sl16, 2), act33, 2); - tensor conv35 = conv_2d(m["model.2.cv2.conv"], cat34, 1, 0); - tensor act36 = ggml_silu_inplace(m, conv35); - tensor conv37 = conv_2d(m["model.3.conv"], act36, 2, 1); - tensor act38 = ggml_silu_inplace(m, conv37); - tensor conv39 = conv_2d(m["model.4.cv1.conv"], act38, 1, 0); - tensor act40 = ggml_silu_inplace(m, conv39); - tensor sl41 = ggml_view_4d(m, act40, 80, 80, 128, 1, act40->nb[1], act40->nb[2], act40->nb[3], 0); - tensor sl42 = ggml_view_4d(m, act40, 80, 80, 128, 1, act40->nb[1], act40->nb[2], act40->nb[3], 128*act40->nb[2]); - tensor conv43 = conv_2d(m["model.4.m.0.cv1.conv"], sl42, 1, 0); - tensor act44 = ggml_silu_inplace(m, conv43); - tensor conv45 = conv_2d(m["model.4.m.0.m.0.cv1.conv"], act44, 1, 1); - tensor act46 = ggml_silu_inplace(m, conv45); - tensor conv47 = conv_2d(m["model.4.m.0.m.0.cv2.conv"], act46, 1, 1); - tensor act48 = ggml_silu_inplace(m, conv47); - tensor add49 = ggml_add(m, act44, act48); - tensor conv50 = conv_2d(m["model.4.m.0.m.1.cv1.conv"], add49, 1, 1); - tensor act51 = ggml_silu_inplace(m, conv50); - tensor conv52 = conv_2d(m["model.4.m.0.m.1.cv2.conv"], act51, 1, 1); - tensor act53 = ggml_silu_inplace(m, conv52); - tensor add54 = ggml_add(m, add49, act53); - tensor conv55 = conv_2d(m["model.4.m.0.cv2.conv"], sl42, 1, 0); - tensor act56 = ggml_silu_inplace(m, conv55); - tensor cat57 = ggml_concat(m, add54, act56, 2); - tensor conv58 = conv_2d(m["model.4.m.0.cv3.conv"], cat57, 1, 0); - tensor act59 = ggml_silu_inplace(m, conv58); - tensor cat60 = ggml_concat(m, ggml_concat(m, sl41, sl42, 2), act59, 2); - tensor conv61 = conv_2d(m["model.4.cv2.conv"], cat60, 1, 0); - tensor act62 = ggml_silu_inplace(m, conv61); - tensor conv63 = conv_2d(m["model.5.conv"], act62, 2, 1); - tensor act64 = ggml_silu_inplace(m, conv63); - tensor conv65 = conv_2d(m["model.6.cv1.conv"], act64, 1, 0); - tensor act66 = ggml_silu_inplace(m, conv65); - tensor sl67 = ggml_view_4d(m, act66, 40, 40, 256, 1, act66->nb[1], act66->nb[2], act66->nb[3], 0); - tensor sl68 = ggml_view_4d(m, act66, 40, 40, 256, 1, act66->nb[1], act66->nb[2], act66->nb[3], 256*act66->nb[2]); - tensor conv69 = conv_2d(m["model.6.m.0.cv1.conv"], sl68, 1, 0); - tensor act70 = ggml_silu_inplace(m, conv69); - tensor conv71 = conv_2d(m["model.6.m.0.m.0.cv1.conv"], act70, 1, 1); - tensor act72 = ggml_silu_inplace(m, conv71); - tensor conv73 = conv_2d(m["model.6.m.0.m.0.cv2.conv"], act72, 1, 1); - tensor act74 = ggml_silu_inplace(m, conv73); - tensor add75 = ggml_add(m, act70, act74); - tensor conv76 = conv_2d(m["model.6.m.0.m.1.cv1.conv"], add75, 1, 1); - tensor act77 = ggml_silu_inplace(m, conv76); - tensor conv78 = conv_2d(m["model.6.m.0.m.1.cv2.conv"], act77, 1, 1); - tensor act79 = ggml_silu_inplace(m, conv78); - tensor add80 = ggml_add(m, add75, act79); - tensor conv81 = conv_2d(m["model.6.m.0.cv2.conv"], sl68, 1, 0); - tensor act82 = ggml_silu_inplace(m, conv81); - tensor cat83 = ggml_concat(m, add80, act82, 2); - tensor conv84 = conv_2d(m["model.6.m.0.cv3.conv"], cat83, 1, 0); - tensor act85 = ggml_silu_inplace(m, conv84); - tensor cat86 = ggml_concat(m, ggml_concat(m, sl67, sl68, 2), act85, 2); - tensor conv87 = conv_2d(m["model.6.cv2.conv"], cat86, 1, 0); - tensor act88 = ggml_silu_inplace(m, conv87); - tensor conv89 = conv_2d(m["model.7.conv"], act88, 2, 1); - tensor act90 = ggml_silu_inplace(m, conv89); - tensor conv91 = conv_2d(m["model.8.cv1.conv"], act90, 1, 0); - tensor act92 = ggml_silu_inplace(m, conv91); - tensor sl93 = ggml_view_4d(m, act92, 20, 20, 256, 1, act92->nb[1], act92->nb[2], act92->nb[3], 0); - tensor sl94 = ggml_view_4d(m, act92, 20, 20, 256, 1, act92->nb[1], act92->nb[2], act92->nb[3], 256*act92->nb[2]); - tensor conv95 = conv_2d(m["model.8.m.0.cv1.conv"], sl94, 1, 0); - tensor act96 = ggml_silu_inplace(m, conv95); - tensor conv97 = conv_2d(m["model.8.m.0.m.0.cv1.conv"], act96, 1, 1); - tensor act98 = ggml_silu_inplace(m, conv97); - tensor conv99 = conv_2d(m["model.8.m.0.m.0.cv2.conv"], act98, 1, 1); - tensor act100 = ggml_silu_inplace(m, conv99); - tensor add101 = ggml_add(m, act96, act100); - tensor conv102 = conv_2d(m["model.8.m.0.m.1.cv1.conv"], add101, 1, 1); - tensor act103 = ggml_silu_inplace(m, conv102); - tensor conv104 = conv_2d(m["model.8.m.0.m.1.cv2.conv"], act103, 1, 1); - tensor act105 = ggml_silu_inplace(m, conv104); - tensor add106 = ggml_add(m, add101, act105); - tensor conv107 = conv_2d(m["model.8.m.0.cv2.conv"], sl94, 1, 0); - tensor act108 = ggml_silu_inplace(m, conv107); - tensor cat109 = ggml_concat(m, add106, act108, 2); - tensor conv110 = conv_2d(m["model.8.m.0.cv3.conv"], cat109, 1, 0); - tensor act111 = ggml_silu_inplace(m, conv110); - tensor cat112 = ggml_concat(m, ggml_concat(m, sl93, sl94, 2), act111, 2); - tensor conv113 = conv_2d(m["model.8.cv2.conv"], cat112, 1, 0); - tensor act114 = ggml_silu_inplace(m, conv113); - tensor conv115 = conv_2d(m["model.9.cv1.conv"], act114, 1, 0); - tensor pool116 = ggml_pool_2d(m, conv115, GGML_OP_POOL_MAX, 5, 5, 1, 1, 2, 2); - tensor pool117 = ggml_pool_2d(m, pool116, GGML_OP_POOL_MAX, 5, 5, 1, 1, 2, 2); - tensor pool118 = ggml_pool_2d(m, pool117, GGML_OP_POOL_MAX, 5, 5, 1, 1, 2, 2); - tensor cat119 = ggml_concat(m, ggml_concat(m, ggml_concat(m, conv115, pool116, 2), pool117, 2), pool118, 2); - tensor conv120 = conv_2d(m["model.9.cv2.conv"], cat119, 1, 0); - tensor act121 = ggml_silu_inplace(m, conv120); - tensor add122 = ggml_add(m, act121, act114); - tensor conv123 = conv_2d(m["model.10.cv1.conv"], add122, 1, 0); - tensor act124 = ggml_silu_inplace(m, conv123); - tensor sl125 = ggml_view_4d(m, act124, 20, 20, 256, 1, act124->nb[1], act124->nb[2], act124->nb[3], 0); - tensor sl126 = ggml_view_4d(m, act124, 20, 20, 256, 1, act124->nb[1], act124->nb[2], act124->nb[3], 256*act124->nb[2]); - tensor conv128 = conv_2d(m["model.10.m.0.attn.qkv.conv"], sl126, 1, 0); - tensor rs129 = ggml_reshape_4d(m, ggml_cont(m, conv128), 400, 128, 4, 1); - tensor sl130 = ggml_view_4d(m, rs129, 400, 32, 4, 1, rs129->nb[1], rs129->nb[2], rs129->nb[3], 0); - tensor sl131 = ggml_view_4d(m, rs129, 400, 32, 4, 1, rs129->nb[1], rs129->nb[2], rs129->nb[3], 32*rs129->nb[1]); - tensor sl132 = ggml_view_4d(m, rs129, 400, 64, 4, 1, rs129->nb[1], rs129->nb[2], rs129->nb[3], 64*rs129->nb[1]); - tensor t133 = ggml_cont(m, ggml_permute(m, sl130, 1, 0, 2, 3)); - tensor mm134 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, sl131, 1, 0, 2, 3)), t133); - tensor t135 = ggml_mul(m, mm134, const8); - tensor sm136 = ggml_soft_max(m, t135); - tensor t137 = ggml_cont(m, ggml_permute(m, sm136, 1, 0, 2, 3)); - tensor mm138 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, t137, 1, 0, 2, 3)), sl132); - tensor rs139 = ggml_reshape_4d(m, ggml_cont(m, mm138), 20, 20, 256, 1); - tensor rs140 = ggml_reshape_4d(m, ggml_cont(m, sl132), 20, 20, 256, 1); - tensor dwconv141 = conv_2d_depthwise(m["model.10.m.0.attn.pe.conv"], rs140, 1, 1); - tensor add142 = ggml_add(m, rs139, dwconv141); - tensor conv143 = conv_2d(m["model.10.m.0.attn.proj.conv"], add142, 1, 0); - tensor add144 = ggml_add(m, sl126, conv143); - tensor conv145 = conv_2d(m["model.10.m.0.ffn.0.conv"], add144, 1, 0); - tensor act146 = ggml_silu_inplace(m, conv145); - tensor conv147 = conv_2d(m["model.10.m.0.ffn.1.conv"], act146, 1, 0); - tensor add148 = ggml_add(m, add144, conv147); - tensor cat149 = ggml_concat(m, sl125, add148, 2); - tensor conv150 = conv_2d(m["model.10.cv2.conv"], cat149, 1, 0); - tensor act151 = ggml_silu_inplace(m, conv150); - tensor up152 = ggml_interpolate(m, act151, 40, 40, 512, 1, GGML_SCALE_MODE_NEAREST) /* interpolate -> target ne */; - tensor cat153 = ggml_concat(m, up152, act88, 2); - tensor conv154 = conv_2d(m["model.13.cv1.conv"], cat153, 1, 0); - tensor act155 = ggml_silu_inplace(m, conv154); - tensor sl156 = ggml_view_4d(m, act155, 40, 40, 256, 1, act155->nb[1], act155->nb[2], act155->nb[3], 0); - tensor sl157 = ggml_view_4d(m, act155, 40, 40, 256, 1, act155->nb[1], act155->nb[2], act155->nb[3], 256*act155->nb[2]); - tensor conv158 = conv_2d(m["model.13.m.0.cv1.conv"], sl157, 1, 0); - tensor act159 = ggml_silu_inplace(m, conv158); - tensor conv160 = conv_2d(m["model.13.m.0.m.0.cv1.conv"], act159, 1, 1); - tensor act161 = ggml_silu_inplace(m, conv160); - tensor conv162 = conv_2d(m["model.13.m.0.m.0.cv2.conv"], act161, 1, 1); - tensor act163 = ggml_silu_inplace(m, conv162); - tensor add164 = ggml_add(m, act159, act163); - tensor conv165 = conv_2d(m["model.13.m.0.m.1.cv1.conv"], add164, 1, 1); - tensor act166 = ggml_silu_inplace(m, conv165); - tensor conv167 = conv_2d(m["model.13.m.0.m.1.cv2.conv"], act166, 1, 1); - tensor act168 = ggml_silu_inplace(m, conv167); - tensor add169 = ggml_add(m, add164, act168); - tensor conv170 = conv_2d(m["model.13.m.0.cv2.conv"], sl157, 1, 0); - tensor act171 = ggml_silu_inplace(m, conv170); - tensor cat172 = ggml_concat(m, add169, act171, 2); - tensor conv173 = conv_2d(m["model.13.m.0.cv3.conv"], cat172, 1, 0); - tensor act174 = ggml_silu_inplace(m, conv173); - tensor cat175 = ggml_concat(m, ggml_concat(m, sl156, sl157, 2), act174, 2); - tensor conv176 = conv_2d(m["model.13.cv2.conv"], cat175, 1, 0); - tensor act177 = ggml_silu_inplace(m, conv176); - tensor up178 = ggml_interpolate(m, act177, 80, 80, 512, 1, GGML_SCALE_MODE_NEAREST) /* interpolate -> target ne */; - tensor cat179 = ggml_concat(m, up178, act62, 2); - tensor conv180 = conv_2d(m["model.16.cv1.conv"], cat179, 1, 0); - tensor act181 = ggml_silu_inplace(m, conv180); - tensor sl182 = ggml_view_4d(m, act181, 80, 80, 128, 1, act181->nb[1], act181->nb[2], act181->nb[3], 0); - tensor sl183 = ggml_view_4d(m, act181, 80, 80, 128, 1, act181->nb[1], act181->nb[2], act181->nb[3], 128*act181->nb[2]); - tensor conv184 = conv_2d(m["model.16.m.0.cv1.conv"], sl183, 1, 0); - tensor act185 = ggml_silu_inplace(m, conv184); - tensor conv186 = conv_2d(m["model.16.m.0.m.0.cv1.conv"], act185, 1, 1); - tensor act187 = ggml_silu_inplace(m, conv186); - tensor conv188 = conv_2d(m["model.16.m.0.m.0.cv2.conv"], act187, 1, 1); - tensor act189 = ggml_silu_inplace(m, conv188); - tensor add190 = ggml_add(m, act185, act189); - tensor conv191 = conv_2d(m["model.16.m.0.m.1.cv1.conv"], add190, 1, 1); - tensor act192 = ggml_silu_inplace(m, conv191); - tensor conv193 = conv_2d(m["model.16.m.0.m.1.cv2.conv"], act192, 1, 1); - tensor act194 = ggml_silu_inplace(m, conv193); - tensor add195 = ggml_add(m, add190, act194); - tensor conv196 = conv_2d(m["model.16.m.0.cv2.conv"], sl183, 1, 0); - tensor act197 = ggml_silu_inplace(m, conv196); - tensor cat198 = ggml_concat(m, add195, act197, 2); - tensor conv199 = conv_2d(m["model.16.m.0.cv3.conv"], cat198, 1, 0); - tensor act200 = ggml_silu_inplace(m, conv199); - tensor cat201 = ggml_concat(m, ggml_concat(m, sl182, sl183, 2), act200, 2); - tensor conv202 = conv_2d(m["model.16.cv2.conv"], cat201, 1, 0); - tensor act203 = ggml_silu_inplace(m, conv202); - tensor conv204 = conv_2d(m["model.17.conv"], act203, 2, 1); - tensor act205 = ggml_silu_inplace(m, conv204); - tensor cat206 = ggml_concat(m, act205, act177, 2); - tensor conv207 = conv_2d(m["model.19.cv1.conv"], cat206, 1, 0); - tensor act208 = ggml_silu_inplace(m, conv207); - tensor sl209 = ggml_view_4d(m, act208, 40, 40, 256, 1, act208->nb[1], act208->nb[2], act208->nb[3], 0); - tensor sl210 = ggml_view_4d(m, act208, 40, 40, 256, 1, act208->nb[1], act208->nb[2], act208->nb[3], 256*act208->nb[2]); - tensor conv211 = conv_2d(m["model.19.m.0.cv1.conv"], sl210, 1, 0); - tensor act212 = ggml_silu_inplace(m, conv211); - tensor conv213 = conv_2d(m["model.19.m.0.m.0.cv1.conv"], act212, 1, 1); - tensor act214 = ggml_silu_inplace(m, conv213); - tensor conv215 = conv_2d(m["model.19.m.0.m.0.cv2.conv"], act214, 1, 1); - tensor act216 = ggml_silu_inplace(m, conv215); - tensor add217 = ggml_add(m, act212, act216); - tensor conv218 = conv_2d(m["model.19.m.0.m.1.cv1.conv"], add217, 1, 1); - tensor act219 = ggml_silu_inplace(m, conv218); - tensor conv220 = conv_2d(m["model.19.m.0.m.1.cv2.conv"], act219, 1, 1); - tensor act221 = ggml_silu_inplace(m, conv220); - tensor add222 = ggml_add(m, add217, act221); - tensor conv223 = conv_2d(m["model.19.m.0.cv2.conv"], sl210, 1, 0); - tensor act224 = ggml_silu_inplace(m, conv223); - tensor cat225 = ggml_concat(m, add222, act224, 2); - tensor conv226 = conv_2d(m["model.19.m.0.cv3.conv"], cat225, 1, 0); - tensor act227 = ggml_silu_inplace(m, conv226); - tensor cat228 = ggml_concat(m, ggml_concat(m, sl209, sl210, 2), act227, 2); - tensor conv229 = conv_2d(m["model.19.cv2.conv"], cat228, 1, 0); - tensor act230 = ggml_silu_inplace(m, conv229); - tensor conv231 = conv_2d(m["model.20.conv"], act230, 2, 1); - tensor act232 = ggml_silu_inplace(m, conv231); - tensor cat233 = ggml_concat(m, act232, act151, 2); - tensor conv234 = conv_2d(m["model.22.cv1.conv"], cat233, 1, 0); - tensor act235 = ggml_silu_inplace(m, conv234); - tensor sl236 = ggml_view_4d(m, act235, 20, 20, 256, 1, act235->nb[1], act235->nb[2], act235->nb[3], 0); - tensor sl237 = ggml_view_4d(m, act235, 20, 20, 256, 1, act235->nb[1], act235->nb[2], act235->nb[3], 256*act235->nb[2]); - tensor conv238 = conv_2d(m["model.22.m.0.0.cv1.conv"], sl237, 1, 1); - tensor act239 = ggml_silu_inplace(m, conv238); - tensor conv240 = conv_2d(m["model.22.m.0.0.cv2.conv"], act239, 1, 1); - tensor act241 = ggml_silu_inplace(m, conv240); - tensor add242 = ggml_add(m, sl237, act241); - tensor conv244 = conv_2d(m["model.22.m.0.1.attn.qkv.conv"], add242, 1, 0); - tensor rs245 = ggml_reshape_4d(m, ggml_cont(m, conv244), 400, 128, 4, 1); - tensor sl246 = ggml_view_4d(m, rs245, 400, 32, 4, 1, rs245->nb[1], rs245->nb[2], rs245->nb[3], 0); - tensor sl247 = ggml_view_4d(m, rs245, 400, 32, 4, 1, rs245->nb[1], rs245->nb[2], rs245->nb[3], 32*rs245->nb[1]); - tensor sl248 = ggml_view_4d(m, rs245, 400, 64, 4, 1, rs245->nb[1], rs245->nb[2], rs245->nb[3], 64*rs245->nb[1]); - tensor t249 = ggml_cont(m, ggml_permute(m, sl246, 1, 0, 2, 3)); - tensor mm250 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, sl247, 1, 0, 2, 3)), t249); - tensor t251 = ggml_mul(m, mm250, const7); - tensor sm252 = ggml_soft_max(m, t251); - tensor t253 = ggml_cont(m, ggml_permute(m, sm252, 1, 0, 2, 3)); - tensor mm254 = ggml_mul_mat(m, ggml_cont(m, ggml_permute(m, t253, 1, 0, 2, 3)), sl248); - tensor rs255 = ggml_reshape_4d(m, ggml_cont(m, mm254), 20, 20, 256, 1); - tensor rs256 = ggml_reshape_4d(m, ggml_cont(m, sl248), 20, 20, 256, 1); - tensor dwconv257 = conv_2d_depthwise(m["model.22.m.0.1.attn.pe.conv"], rs256, 1, 1); - tensor add258 = ggml_add(m, rs255, dwconv257); - tensor conv259 = conv_2d(m["model.22.m.0.1.attn.proj.conv"], add258, 1, 0); - tensor add260 = ggml_add(m, add242, conv259); - tensor conv261 = conv_2d(m["model.22.m.0.1.ffn.0.conv"], add260, 1, 0); - tensor act262 = ggml_silu_inplace(m, conv261); - tensor conv263 = conv_2d(m["model.22.m.0.1.ffn.1.conv"], act262, 1, 0); - tensor add264 = ggml_add(m, add260, conv263); - tensor cat265 = ggml_concat(m, ggml_concat(m, sl236, sl237, 2), add264, 2); - tensor conv266 = conv_2d(m["model.22.cv2.conv"], cat265, 1, 0); - tensor act267 = ggml_silu_inplace(m, conv266); - tensor conv268 = conv_2d(m["model.23.cv2.0.0.conv"], act203, 1, 1); - tensor act269 = ggml_silu_inplace(m, conv268); - tensor conv270 = conv_2d(m["model.23.cv2.0.1.conv"], act269, 1, 1); - tensor act271 = ggml_silu_inplace(m, conv270); - tensor conv272 = conv_2d(m["model.23.cv2.0.2"], act271, 1, 0); - tensor rs273 = ggml_reshape_3d(m, ggml_cont(m, conv272), 6400, 4, 1); - tensor conv274 = conv_2d(m["model.23.cv2.1.0.conv"], act230, 1, 1); - tensor act275 = ggml_silu_inplace(m, conv274); - tensor conv276 = conv_2d(m["model.23.cv2.1.1.conv"], act275, 1, 1); - tensor act277 = ggml_silu_inplace(m, conv276); - tensor conv278 = conv_2d(m["model.23.cv2.1.2"], act277, 1, 0); - tensor rs279 = ggml_reshape_3d(m, ggml_cont(m, conv278), 1600, 4, 1); - tensor conv280 = conv_2d(m["model.23.cv2.2.0.conv"], act267, 1, 1); - tensor act281 = ggml_silu_inplace(m, conv280); - tensor conv282 = conv_2d(m["model.23.cv2.2.1.conv"], act281, 1, 1); - tensor act283 = ggml_silu_inplace(m, conv282); - tensor conv284 = conv_2d(m["model.23.cv2.2.2"], act283, 1, 0); - tensor rs285 = ggml_reshape_3d(m, ggml_cont(m, conv284), 400, 4, 1); - tensor cat286 = ggml_concat(m, ggml_concat(m, rs273, rs279, 0), rs285, 0); - tensor dwconv287 = conv_2d_depthwise(m["model.23.cv3.0.0.0.conv"], act203, 1, 1); - tensor act288 = ggml_silu_inplace(m, dwconv287); - tensor conv289 = conv_2d(m["model.23.cv3.0.0.1.conv"], act288, 1, 0); - tensor act290 = ggml_silu_inplace(m, conv289); - tensor dwconv291 = conv_2d_depthwise(m["model.23.cv3.0.1.0.conv"], act290, 1, 1); - tensor act292 = ggml_silu_inplace(m, dwconv291); - tensor conv293 = conv_2d(m["model.23.cv3.0.1.1.conv"], act292, 1, 0); - tensor act294 = ggml_silu_inplace(m, conv293); - tensor conv295 = conv_2d(m["model.23.cv3.0.2"], act294, 1, 0); - tensor rs296 = ggml_reshape_3d(m, ggml_cont(m, conv295), 6400, 80, 1); - tensor dwconv297 = conv_2d_depthwise(m["model.23.cv3.1.0.0.conv"], act230, 1, 1); - tensor act298 = ggml_silu_inplace(m, dwconv297); - tensor conv299 = conv_2d(m["model.23.cv3.1.0.1.conv"], act298, 1, 0); - tensor act300 = ggml_silu_inplace(m, conv299); - tensor dwconv301 = conv_2d_depthwise(m["model.23.cv3.1.1.0.conv"], act300, 1, 1); - tensor act302 = ggml_silu_inplace(m, dwconv301); - tensor conv303 = conv_2d(m["model.23.cv3.1.1.1.conv"], act302, 1, 0); - tensor act304 = ggml_silu_inplace(m, conv303); - tensor conv305 = conv_2d(m["model.23.cv3.1.2"], act304, 1, 0); - tensor rs306 = ggml_reshape_3d(m, ggml_cont(m, conv305), 1600, 80, 1); - tensor dwconv307 = conv_2d_depthwise(m["model.23.cv3.2.0.0.conv"], act267, 1, 1); - tensor act308 = ggml_silu_inplace(m, dwconv307); - tensor conv309 = conv_2d(m["model.23.cv3.2.0.1.conv"], act308, 1, 0); - tensor act310 = ggml_silu_inplace(m, conv309); - tensor dwconv311 = conv_2d_depthwise(m["model.23.cv3.2.1.0.conv"], act310, 1, 1); - tensor act312 = ggml_silu_inplace(m, dwconv311); - tensor conv313 = conv_2d(m["model.23.cv3.2.1.1.conv"], act312, 1, 0); - tensor act314 = ggml_silu_inplace(m, conv313); - tensor conv315 = conv_2d(m["model.23.cv3.2.2"], act314, 1, 0); - tensor rs316 = ggml_reshape_3d(m, ggml_cont(m, conv315), 400, 80, 1); - tensor cat317 = ggml_concat(m, ggml_concat(m, rs296, rs306, 0), rs316, 0); - tensor conv318 = conv_2d(m["model.23.one2one_cv2.0.0.conv"], act203, 1, 1); - tensor act319 = ggml_silu_inplace(m, conv318); - tensor conv320 = conv_2d(m["model.23.one2one_cv2.0.1.conv"], act319, 1, 1); - tensor act321 = ggml_silu_inplace(m, conv320); - tensor conv322 = conv_2d(m["model.23.one2one_cv2.0.2"], act321, 1, 0); - tensor rs323 = ggml_reshape_3d(m, ggml_cont(m, conv322), 6400, 4, 1); - tensor conv324 = conv_2d(m["model.23.one2one_cv2.1.0.conv"], act230, 1, 1); - tensor act325 = ggml_silu_inplace(m, conv324); - tensor conv326 = conv_2d(m["model.23.one2one_cv2.1.1.conv"], act325, 1, 1); - tensor act327 = ggml_silu_inplace(m, conv326); - tensor conv328 = conv_2d(m["model.23.one2one_cv2.1.2"], act327, 1, 0); - tensor rs329 = ggml_reshape_3d(m, ggml_cont(m, conv328), 1600, 4, 1); - tensor conv330 = conv_2d(m["model.23.one2one_cv2.2.0.conv"], act267, 1, 1); - tensor act331 = ggml_silu_inplace(m, conv330); - tensor conv332 = conv_2d(m["model.23.one2one_cv2.2.1.conv"], act331, 1, 1); - tensor act333 = ggml_silu_inplace(m, conv332); - tensor conv334 = conv_2d(m["model.23.one2one_cv2.2.2"], act333, 1, 0); - tensor rs335 = ggml_reshape_3d(m, ggml_cont(m, conv334), 400, 4, 1); - tensor cat336 = ggml_concat(m, ggml_concat(m, rs323, rs329, 0), rs335, 0); - tensor dwconv337 = conv_2d_depthwise(m["model.23.one2one_cv3.0.0.0.conv"], act203, 1, 1); - tensor act338 = ggml_silu_inplace(m, dwconv337); - tensor conv339 = conv_2d(m["model.23.one2one_cv3.0.0.1.conv"], act338, 1, 0); - tensor act340 = ggml_silu_inplace(m, conv339); - tensor dwconv341 = conv_2d_depthwise(m["model.23.one2one_cv3.0.1.0.conv"], act340, 1, 1); - tensor act342 = ggml_silu_inplace(m, dwconv341); - tensor conv343 = conv_2d(m["model.23.one2one_cv3.0.1.1.conv"], act342, 1, 0); - tensor act344 = ggml_silu_inplace(m, conv343); - tensor conv345 = conv_2d(m["model.23.one2one_cv3.0.2"], act344, 1, 0); - tensor rs346 = ggml_reshape_3d(m, ggml_cont(m, conv345), 6400, 80, 1); - tensor dwconv347 = conv_2d_depthwise(m["model.23.one2one_cv3.1.0.0.conv"], act230, 1, 1); - tensor act348 = ggml_silu_inplace(m, dwconv347); - tensor conv349 = conv_2d(m["model.23.one2one_cv3.1.0.1.conv"], act348, 1, 0); - tensor act350 = ggml_silu_inplace(m, conv349); - tensor dwconv351 = conv_2d_depthwise(m["model.23.one2one_cv3.1.1.0.conv"], act350, 1, 1); - tensor act352 = ggml_silu_inplace(m, dwconv351); - tensor conv353 = conv_2d(m["model.23.one2one_cv3.1.1.1.conv"], act352, 1, 0); - tensor act354 = ggml_silu_inplace(m, conv353); - tensor conv355 = conv_2d(m["model.23.one2one_cv3.1.2"], act354, 1, 0); - tensor rs356 = ggml_reshape_3d(m, ggml_cont(m, conv355), 1600, 80, 1); - tensor dwconv357 = conv_2d_depthwise(m["model.23.one2one_cv3.2.0.0.conv"], act267, 1, 1); - tensor act358 = ggml_silu_inplace(m, dwconv357); - tensor conv359 = conv_2d(m["model.23.one2one_cv3.2.0.1.conv"], act358, 1, 0); - tensor act360 = ggml_silu_inplace(m, conv359); - tensor dwconv361 = conv_2d_depthwise(m["model.23.one2one_cv3.2.1.0.conv"], act360, 1, 1); - tensor act362 = ggml_silu_inplace(m, dwconv361); - tensor conv363 = conv_2d(m["model.23.one2one_cv3.2.1.1.conv"], act362, 1, 0); - tensor act364 = ggml_silu_inplace(m, conv363); - tensor conv365 = conv_2d(m["model.23.one2one_cv3.2.2"], act364, 1, 0); - tensor rs366 = ggml_reshape_3d(m, ggml_cont(m, conv365), 400, 80, 1); - tensor cat367 = ggml_concat(m, ggml_concat(m, rs346, rs356, 0), rs366, 0); - tensor ar370 = ggml_arange(m, 0.0f, 80.0f, 1.0f); - tensor add371 = ggml_add(m, ar370, const5); - tensor mg372 = ggml_repeat(m, add371, ggml_new_tensor_2d(m, GGML_TYPE_F32, 80, 80)); - tensor stk373 = ggml_concat(m, ggml_reshape_3d(m, mg372, 1, 80, 80), ggml_reshape_3d(m, mg372, 1, 80, 80), 0); - tensor rs374 = ggml_reshape_2d(m, ggml_cont(m, stk373), 2, 6400); - tensor full376 = ggml_fill(m, ggml_new_tensor_2d(m, GGML_TYPE_F32, 1, 6400), 8.0f); - tensor ar379 = ggml_arange(m, 0.0f, 40.0f, 1.0f); - tensor add380 = ggml_add(m, ar379, const3); - tensor mg381 = ggml_repeat(m, add380, ggml_new_tensor_2d(m, GGML_TYPE_F32, 40, 40)); - tensor stk382 = ggml_concat(m, ggml_reshape_3d(m, mg381, 1, 40, 40), ggml_reshape_3d(m, mg381, 1, 40, 40), 0); - tensor rs383 = ggml_reshape_2d(m, ggml_cont(m, stk382), 2, 1600); - tensor full385 = ggml_fill(m, ggml_new_tensor_2d(m, GGML_TYPE_F32, 1, 1600), 16.0f); - tensor ar388 = ggml_arange(m, 0.0f, 20.0f, 1.0f); - tensor add389 = ggml_add(m, ar388, const1); - tensor mg390 = ggml_repeat(m, add389, ggml_new_tensor_2d(m, GGML_TYPE_F32, 20, 20)); - tensor stk391 = ggml_concat(m, ggml_reshape_3d(m, mg390, 1, 20, 20), ggml_reshape_3d(m, mg390, 1, 20, 20), 0); - tensor rs392 = ggml_reshape_2d(m, ggml_cont(m, stk391), 2, 400); - tensor full394 = ggml_fill(m, ggml_new_tensor_2d(m, GGML_TYPE_F32, 1, 400), 32.0f); - tensor cat395 = ggml_concat(m, ggml_concat(m, rs374, rs383, 1), rs392, 1); - tensor cat396 = ggml_concat(m, ggml_concat(m, full376, full385, 1), full394, 1); - tensor t397 = ggml_cont(m, ggml_permute(m, cat395, 1, 0, 2, 3)); - tensor t398 = ggml_cont(m, ggml_permute(m, cat396, 1, 0, 2, 3)); - tensor uns399 = ggml_reshape_3d(m, ggml_cont(m, t397), 8400, 2, 1); - tensor sl400 = ggml_view_3d(m, cat336, 8400, 2, 1, cat336->nb[1], cat336->nb[2], 0); - tensor sl401 = ggml_view_3d(m, cat336, 8400, 2, 1, cat336->nb[1], cat336->nb[2], 2*cat336->nb[1]); - tensor t402 = ggml_sub(m, uns399, sl400); - tensor add403 = ggml_add(m, uns399, sl401); - tensor cat404 = ggml_concat(m, t402, add403, 1); - tensor t405 = ggml_mul(m, cat404, t398); - tensor t406 = ggml_sigmoid(m, ggml_concat(m, ggml_concat(m, rs346, rs356, 0), rs366, 0)); - tensor cat407 = ggml_concat(m, t405, t406, 1); - - tensor out_0 = compute_graph_output(m, contiguous_2d_to_cwhn(m, cat286), "out_0"); - tensor out_1 = compute_graph_output(m, contiguous_2d_to_cwhn(m, act203), "out_1"); - tensor out_2 = compute_graph_output(m, contiguous_2d_to_cwhn(m, act230), "out_2"); - tensor out_3 = compute_graph_output(m, contiguous_2d_to_cwhn(m, act267), "out_3"); - tensor out_4 = compute_graph_output(m, contiguous_2d_to_cwhn(m, cat317), "out_4"); - tensor out_5 = compute_graph_output(m, contiguous_2d_to_cwhn(m, cat336), "out_5"); - tensor out_6 = compute_graph_output(m, contiguous_2d_to_cwhn(m, act203), "out_6"); - tensor out_7 = compute_graph_output(m, contiguous_2d_to_cwhn(m, act230), "out_7"); - tensor out_8 = compute_graph_output(m, contiguous_2d_to_cwhn(m, act267), "out_8"); - tensor out_9 = compute_graph_output(m, contiguous_2d_to_cwhn(m, cat367), "out_9"); - return out_9; -} - -Yolo26m_params Yolo26m_detect_params(model_file const& f) { - Yolo26m_params p{}; - // GGUF general.architecture 검증 (모델은 정적 unroll 이라 추가 하이퍼파라미터 없음). - if (std::string_view arch = f.arch(); arch != "yolo26m") { - throw except( - "Architecture expected to be 'yolo26m', but was '{}' ({})", - arch, f.path); - } - return p; -} - -} // namespace visp diff --git a/src/visp/arch/famann_register.cpp b/src/visp/arch/famann_register.cpp new file mode 100644 index 0000000..9fe26db --- /dev/null +++ b/src/visp/arch/famann_register.cpp @@ -0,0 +1,30 @@ +// GENERATED BY tools/install_arch.py — 손으로 고치지 마라(재설치하면 덮어쓴다). +// +// 이 파일은 **아무도 부르지 않는다.** 전역 객체 생성자가 프로그램 시작 시 스스로 +// `arch_registry` 에 등록한다. 그래서 CLI 에 case 를 추가할 필요가 없다. +#include "visp/arch/FamAnn.h" +#include "visp/arch_registry.h" + + +namespace { + +visp::tensor forward(visp::model_ref m, visp::tensor x, visp::model_file const& f) { + // `FamAnn_params` 는 arch 마다 타입이 달라 레지스트리 시그니처에 못 싣는다 — + // 여기서 만들어 소화한다. + return visp::FamAnn_forward(m, x, visp::FamAnn_detect_params(f)); +} + +visp::arch_task make_task() { + visp::arch_task t; + t.kind = visp::arch_kind::raw; + t.input_size = 512; + t.input_w = 1024; + t.input_h = 512; + t.mean = {123.675f, 116.28f, 103.53f}; + t.stdv = {58.395f, 57.12f, 57.375f}; + return t; +} + +const visp::arch_registrar reg{"famann", &forward, make_task()}; + +} // namespace diff --git a/src/visp/arch/famapcnet_register.cpp b/src/visp/arch/famapcnet_register.cpp new file mode 100644 index 0000000..546842d --- /dev/null +++ b/src/visp/arch/famapcnet_register.cpp @@ -0,0 +1,30 @@ +// GENERATED BY tools/install_arch.py — 손으로 고치지 마라(재설치하면 덮어쓴다). +// +// 이 파일은 **아무도 부르지 않는다.** 전역 객체 생성자가 프로그램 시작 시 스스로 +// `arch_registry` 에 등록한다. 그래서 CLI 에 case 를 추가할 필요가 없다. +#include "visp/arch/FamApcnet.h" +#include "visp/arch_registry.h" + + +namespace { + +visp::tensor forward(visp::model_ref m, visp::tensor x, visp::model_file const& f) { + // `FamApcnet_params` 는 arch 마다 타입이 달라 레지스트리 시그니처에 못 싣는다 — + // 여기서 만들어 소화한다. + return visp::FamApcnet_forward(m, x, visp::FamApcnet_detect_params(f)); +} + +visp::arch_task make_task() { + visp::arch_task t; + t.kind = visp::arch_kind::raw; + t.input_size = 512; + t.input_w = 1024; + t.input_h = 512; + t.mean = {123.675f, 116.28f, 103.53f}; + t.stdv = {58.395f, 57.12f, 57.375f}; + return t; +} + +const visp::arch_registrar reg{"famapcnet", &forward, make_task()}; + +} // namespace diff --git a/src/visp/arch/fambeit_register.cpp b/src/visp/arch/fambeit_register.cpp new file mode 100644 index 0000000..f9434ed --- /dev/null +++ b/src/visp/arch/fambeit_register.cpp @@ -0,0 +1,30 @@ +// GENERATED BY tools/install_arch.py — 손으로 고치지 마라(재설치하면 덮어쓴다). +// +// 이 파일은 **아무도 부르지 않는다.** 전역 객체 생성자가 프로그램 시작 시 스스로 +// `arch_registry` 에 등록한다. 그래서 CLI 에 case 를 추가할 필요가 없다. +#include "visp/arch/FamBeit.h" +#include "visp/arch_registry.h" + + +namespace { + +visp::tensor forward(visp::model_ref m, visp::tensor x, visp::model_file const& f) { + // `FamBeit_params` 는 arch 마다 타입이 달라 레지스트리 시그니처에 못 싣는다 — + // 여기서 만들어 소화한다. + return visp::FamBeit_forward(m, x, visp::FamBeit_detect_params(f)); +} + +visp::arch_task make_task() { + visp::arch_task t; + t.kind = visp::arch_kind::raw; + t.input_size = 640; + t.input_w = 640; + t.input_h = 640; + t.mean = {123.675f, 116.28f, 103.53f}; + t.stdv = {58.395f, 57.12f, 57.375f}; + return t; +} + +const visp::arch_registrar reg{"fambeit", &forward, make_task()}; + +} // namespace diff --git a/src/visp/arch/fambisenetv1_register.cpp b/src/visp/arch/fambisenetv1_register.cpp new file mode 100644 index 0000000..2544311 --- /dev/null +++ b/src/visp/arch/fambisenetv1_register.cpp @@ -0,0 +1,30 @@ +// GENERATED BY tools/install_arch.py — 손으로 고치지 마라(재설치하면 덮어쓴다). +// +// 이 파일은 **아무도 부르지 않는다.** 전역 객체 생성자가 프로그램 시작 시 스스로 +// `arch_registry` 에 등록한다. 그래서 CLI 에 case 를 추가할 필요가 없다. +#include "visp/arch/FamBisenetv1.h" +#include "visp/arch_registry.h" + + +namespace { + +visp::tensor forward(visp::model_ref m, visp::tensor x, visp::model_file const& f) { + // `FamBisenetv1_params` 는 arch 마다 타입이 달라 레지스트리 시그니처에 못 싣는다 — + // 여기서 만들어 소화한다. + return visp::FamBisenetv1_forward(m, x, visp::FamBisenetv1_detect_params(f)); +} + +visp::arch_task make_task() { + visp::arch_task t; + t.kind = visp::arch_kind::raw; + t.input_size = 1024; + t.input_w = 1024; + t.input_h = 1024; + t.mean = {123.675f, 116.28f, 103.53f}; + t.stdv = {58.395f, 57.12f, 57.375f}; + return t; +} + +const visp::arch_registrar reg{"fambisenetv1", &forward, make_task()}; + +} // namespace diff --git a/src/visp/arch/fambisenetv2_register.cpp b/src/visp/arch/fambisenetv2_register.cpp new file mode 100644 index 0000000..ca8b4db --- /dev/null +++ b/src/visp/arch/fambisenetv2_register.cpp @@ -0,0 +1,30 @@ +// GENERATED BY tools/install_arch.py — 손으로 고치지 마라(재설치하면 덮어쓴다). +// +// 이 파일은 **아무도 부르지 않는다.** 전역 객체 생성자가 프로그램 시작 시 스스로 +// `arch_registry` 에 등록한다. 그래서 CLI 에 case 를 추가할 필요가 없다. +#include "visp/arch/FamBisenetv2.h" +#include "visp/arch_registry.h" + + +namespace { + +visp::tensor forward(visp::model_ref m, visp::tensor x, visp::model_file const& f) { + // `FamBisenetv2_params` 는 arch 마다 타입이 달라 레지스트리 시그니처에 못 싣는다 — + // 여기서 만들어 소화한다. + return visp::FamBisenetv2_forward(m, x, visp::FamBisenetv2_detect_params(f)); +} + +visp::arch_task make_task() { + visp::arch_task t; + t.kind = visp::arch_kind::raw; + t.input_size = 1024; + t.input_w = 1024; + t.input_h = 1024; + t.mean = {123.675f, 116.28f, 103.53f}; + t.stdv = {58.395f, 57.12f, 57.375f}; + return t; +} + +const visp::arch_registrar reg{"fambisenetv2", &forward, make_task()}; + +} // namespace diff --git a/src/visp/arch/famcgnet_register.cpp b/src/visp/arch/famcgnet_register.cpp new file mode 100644 index 0000000..e7efbdc --- /dev/null +++ b/src/visp/arch/famcgnet_register.cpp @@ -0,0 +1,30 @@ +// GENERATED BY tools/install_arch.py — 손으로 고치지 마라(재설치하면 덮어쓴다). +// +// 이 파일은 **아무도 부르지 않는다.** 전역 객체 생성자가 프로그램 시작 시 스스로 +// `arch_registry` 에 등록한다. 그래서 CLI 에 case 를 추가할 필요가 없다. +#include "visp/arch/FamCgnet.h" +#include "visp/arch_registry.h" + + +namespace { + +visp::tensor forward(visp::model_ref m, visp::tensor x, visp::model_file const& f) { + // `FamCgnet_params` 는 arch 마다 타입이 달라 레지스트리 시그니처에 못 싣는다 — + // 여기서 만들어 소화한다. + return visp::FamCgnet_forward(m, x, visp::FamCgnet_detect_params(f)); +} + +visp::arch_task make_task() { + visp::arch_task t; + t.kind = visp::arch_kind::raw; + t.input_size = 680; + t.input_w = 680; + t.input_h = 680; + t.mean = {72.39239876f, 82.90891754f, 73.15835921f}; + t.stdv = {1.0f, 1.0f, 1.0f}; + return t; +} + +const visp::arch_registrar reg{"famcgnet", &forward, make_task()}; + +} // namespace diff --git a/src/visp/arch/famdeeplabv3_register.cpp b/src/visp/arch/famdeeplabv3_register.cpp new file mode 100644 index 0000000..25ba7d9 --- /dev/null +++ b/src/visp/arch/famdeeplabv3_register.cpp @@ -0,0 +1,30 @@ +// GENERATED BY tools/install_arch.py — 손으로 고치지 마라(재설치하면 덮어쓴다). +// +// 이 파일은 **아무도 부르지 않는다.** 전역 객체 생성자가 프로그램 시작 시 스스로 +// `arch_registry` 에 등록한다. 그래서 CLI 에 case 를 추가할 필요가 없다. +#include "visp/arch/FamDeeplabv3.h" +#include "visp/arch_registry.h" + + +namespace { + +visp::tensor forward(visp::model_ref m, visp::tensor x, visp::model_file const& f) { + // `FamDeeplabv3_params` 는 arch 마다 타입이 달라 레지스트리 시그니처에 못 싣는다 — + // 여기서 만들어 소화한다. + return visp::FamDeeplabv3_forward(m, x, visp::FamDeeplabv3_detect_params(f)); +} + +visp::arch_task make_task() { + visp::arch_task t; + t.kind = visp::arch_kind::raw; + t.input_size = 512; + t.input_w = 1024; + t.input_h = 512; + t.mean = {123.675f, 116.28f, 103.53f}; + t.stdv = {58.395f, 57.12f, 57.375f}; + return t; +} + +const visp::arch_registrar reg{"famdeeplabv3", &forward, make_task()}; + +} // namespace diff --git a/src/visp/arch/graph_interpreter.cpp b/src/visp/arch/graph_interpreter.cpp new file mode 100644 index 0000000..bb7447d --- /dev/null +++ b/src/visp/arch/graph_interpreter.cpp @@ -0,0 +1,126 @@ +// 범용 그래프 인터프리터 구현. (graph_interpreter.h 참조) +#include "visp/arch/graph_interpreter.h" +#include "visp/nn.h" + +#include + +#include +#include +#include +#include + +namespace visp { + +static std::vector split(std::string_view s, char d) { + std::vector out; + size_t start = 0; + while (true) { + size_t p = s.find(d, start); + size_t len = (p == std::string_view::npos) ? std::string_view::npos : p - start; + out.emplace_back(std::string(s.substr(start, len))); + if (p == std::string_view::npos) break; + start = p + 1; + } + return out; +} + +static std::vector parse_ids(std::string_view s) { + std::vector v; + for (auto& x : split(s, ',')) if (!x.empty()) v.push_back(std::atoi(x.c_str())); + return v; +} + +graph_spec parse_graph(model_file const& file) { + graph_spec g; + for (auto& line : split(file.get_string("graph.nodes"), '\n')) { + if (line.empty()) continue; + auto f = split(line, '|'); // id|op|inputs|weight|attrs|out_shape + if (f.size() < 2) continue; + graph_node n; + n.id = std::atoi(f[0].c_str()); + n.op = f[1]; + if (f.size() > 2) n.inputs = parse_ids(f[2]); + if (f.size() > 3) n.weight = f[3]; + if (f.size() > 4 && !f[4].empty()) + for (auto& kv : split(f[4], ';')) { + auto e = kv.find('='); + if (e != std::string::npos) + n.attrs[kv.substr(0, e)] = (float)std::atof(kv.c_str() + e + 1); + } + if (f.size() > 5 && !f[5].empty()) + for (auto& x : split(f[5], ',')) n.out_shape.push_back(std::atoll(x.c_str())); + g.nodes.push_back(std::move(n)); + } + g.inputs = parse_ids(file.get_string("graph.inputs")); + g.outputs = parse_ids(file.get_string("graph.outputs")); + return g; +} + +static int iattr(graph_node const& n, char const* k, int def) { + auto it = n.attrs.find(k); + return it == n.attrs.end() ? def : (int)it->second; +} + +tensor graph_interpret(model_ref m, tensor input, graph_spec const& g) { + std::map t; + tensor x0 = cwhn_to_contiguous_2d(m, input); + for (int id : g.inputs) t[id] = x0; + + for (auto& n : g.nodes) { + if (n.op == "input") continue; + auto in = [&](int i) -> tensor { return t.at(n.inputs[(size_t)i]); }; + model_ref mw = n.weight.empty() ? m : m[n.weight.c_str()]; + tensor y = nullptr; + + if (n.op == "conv2d") { + int s = iattr(n, "stride", 1), p = iattr(n, "padding", 0), + d = iattr(n, "dilation", 1), grp = iattr(n, "groups", 1); + y = grp > 1 ? conv_2d_grouped(mw, in(0), s, p, d, grp) + : conv_2d(mw, in(0), s, p, d); + } else if (n.op == "batch_norm") { + y = batch_norm_2d(mw, in(0)); + } else if (n.op == "relu") { + y = ggml_relu(m, in(0)); + } else if (n.op == "elemwise_add") { + y = ggml_add(m, in(0), in(1)); + } else if (n.op == "maxpool") { + int k = iattr(n, "kernel_size", 2), s = iattr(n, "stride", k), p = iattr(n, "padding", 0); + y = ggml_pool_2d(m, in(0), GGML_OP_POOL_MAX, k, k, s, s, p, p); + } else if (n.op == "adaptive_avg_pool2d") { + tensor a = in(0); + y = ggml_pool_2d(m, a, GGML_OP_POOL_AVG, a->ne[0], a->ne[1], a->ne[0], a->ne[1], 0, 0); + } else if (n.op == "flatten") { + tensor a = ggml_cont(m, in(0)); + y = ggml_reshape_2d(m, a, a->ne[0] * a->ne[1] * a->ne[2], a->ne[3]); + } else if (n.op == "dense" || n.op == "linear") { + y = linear(mw, in(0)); + } else if (n.op == "resize") { + // FPN 업샘플 등. 정적 out_shape(torch NCHW) 역순 = ggml ne [W,H,C,N] 로 interpolate. + int64_t ne[4] = {1, 1, 1, 1}; + int k = 0; + for (int i = (int)n.out_shape.size() - 1; i >= 0 && k < 4; --i) ne[k++] = n.out_shape[(size_t)i]; + y = ggml_interpolate(m, in(0), ne[0], ne[1], ne[2], ne[3], GGML_SCALE_MODE_NEAREST); + } else if (n.op == "shape") { + // 메타 op(차원 질의) — resize 는 정적 out_shape 를 쓰므로 tensor 로는 미사용. passthrough. + y = n.inputs.empty() ? nullptr : in(0); + if (!y) continue; + } else { + fprintf(stderr, "[graph] 미지원 op '%s' (n%d) — passthrough\n", n.op.c_str(), n.id); + y = in(0); + } + ggml_format_name(y, "n%d_%s", n.id, n.op.c_str()); + t[n.id] = y; + } + + tensor last = nullptr; + int oi = 0; + for (int oid : g.outputs) { + tensor out = contiguous_2d_to_cwhn(m, t.at(oid)); + char nm[32]; + snprintf(nm, sizeof(nm), "out_%d", oi++); + last = compute_graph_output(m, out, tensor_name(nm)); + } + return last; +} + +} // namespace visp diff --git a/src/visp/arch/graph_interpreter.h b/src/visp/arch/graph_interpreter.h new file mode 100644 index 0000000..062ffea --- /dev/null +++ b/src/visp/arch/graph_interpreter.h @@ -0,0 +1,35 @@ +// 범용 그래프 인터프리터 — gguf 의 그래프 '데이터'(graph.nodes)를 읽어 런타임에 ggml +// 그래프를 조립한다. g2c 가 .cpp 부품을 생성하는 대신, 모델을 노드 리스트로 직렬화하고 +// (tools/graph_export/serialize.py) vision.cpp 가 여기서 해석한다 (ONNX 방식). +#pragma once + +#include "visp/ml.h" + +#include +#include +#include + +namespace visp { + +struct graph_node { + int id = 0; + std::string op; // "conv2d", "batch_norm", "relu", ... + std::vector inputs; // 활성 입력 = producer 노드 id + std::string weight; // 가중치 prefix (m[weight]); 무가중치면 빈 문자열 + std::map attrs; // 스칼라 attr (stride/padding/...) + std::vector out_shape; +}; + +struct graph_spec { + std::vector nodes; // id 오름차순(topo) + std::vector inputs; + std::vector outputs; +}; + +// gguf KV(graph.nodes/inputs/outputs) 파싱 +graph_spec parse_graph(model_file const& file); + +// 그래프 데이터 → ggml 그래프 조립. out_0..N 등록, 마지막 출력 반환. +tensor graph_interpret(model_ref m, tensor input, graph_spec const& g); + +} // namespace visp diff --git a/src/visp/arch/resnet_register.cpp b/src/visp/arch/resnet_register.cpp new file mode 100644 index 0000000..fe793f3 --- /dev/null +++ b/src/visp/arch/resnet_register.cpp @@ -0,0 +1,30 @@ +// GENERATED BY tools/install_arch.py — 손으로 고치지 마라(재설치하면 덮어쓴다). +// +// 이 파일은 **아무도 부르지 않는다.** 전역 객체 생성자가 프로그램 시작 시 스스로 +// `arch_registry` 에 등록한다. 그래서 CLI 에 case 를 추가할 필요가 없다. +#include "visp/arch/ResNet.h" +#include "visp/arch_registry.h" + + +namespace { + +visp::tensor forward(visp::model_ref m, visp::tensor x, visp::model_file const& f) { + // `ResNet_params` 는 arch 마다 타입이 달라 레지스트리 시그니처에 못 싣는다 — + // 여기서 만들어 소화한다. + return visp::ResNet_forward(m, x, visp::ResNet_detect_params(f)); +} + +visp::arch_task make_task() { + visp::arch_task t; + t.kind = visp::arch_kind::raw; + t.input_size = 224; + t.input_w = 224; + t.input_h = 224; + t.mean = {123.675f, 116.28f, 103.53f}; + t.stdv = {58.395f, 57.12f, 57.375f}; + return t; +} + +const visp::arch_registrar reg{"resnet", &forward, make_task()}; + +} // namespace diff --git a/src/visp/arch/yolo26m_register.cpp b/src/visp/arch/yolo26m_register.cpp deleted file mode 100644 index f5e8b07..0000000 --- a/src/visp/arch/yolo26m_register.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// GENERATED BY tools/install_arch.py — 손으로 고치지 마라(재설치하면 덮어쓴다). -// -// 이 파일은 **아무도 부르지 않는다.** 전역 객체 생성자가 프로그램 시작 시 스스로 -// `arch_registry` 에 등록한다. 그래서 CLI 에 case 를 추가할 필요가 없다. -#include "visp/arch/Yolo26m.h" -#include "visp/arch_registry.h" - -namespace { - -visp::tensor forward(visp::model_ref m, visp::tensor x, visp::model_file const& f) { - // `Yolo26m_params` 는 arch 마다 타입이 달라 레지스트리 시그니처에 못 싣는다 — - // 여기서 만들어 소화한다. - return visp::Yolo26m_forward(m, x, visp::Yolo26m_detect_params(f)); -} - -visp::arch_task make_task() { - visp::arch_task t; - t.kind = visp::arch_kind::detect_yolo; - t.num_classes = 80; - t.strides = {8.0f, 16.0f, 32.0f}; - t.nms_free = true; - t.score_thr = 0.25f; - t.input_size = 640; - t.mean = {0.0f, 0.0f, 0.0f}; - t.stdv = {255.0f, 255.0f, 255.0f}; - t.class_names = { - "person", "bicycle", "car", "motorcycle", "airplane", "bus", - "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", - "parking meter", "bench", "bird", "cat", "dog", "horse", - "sheep", "cow", "elephant", "bear", "zebra", "giraffe", - "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", - "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", - "fork", "knife", "spoon", "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", - "donut", "cake", "chair", "couch", "potted plant", "bed", - "dining table", "toilet", "tv", "laptop", "mouse", "remote", - "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", - "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", - "hair drier", "toothbrush", - }; - return t; -} - -const visp::arch_registrar reg{"yolo26m", &forward, make_task()}; - -} // namespace diff --git a/src/visp/arch/yolov9t.cpp b/src/visp/arch/yolov9t.cpp deleted file mode 100644 index 6a29899..0000000 --- a/src/visp/arch/yolov9t.cpp +++ /dev/null @@ -1,1715 +0,0 @@ -#include "yolov9t.h" -#include "visp/image.h" -#include "visp/ml.h" -#include "visp/nn.h" -#include "visp/util.h" -#include "util/math.h" -#include "visp/vision.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "ggml.h" - -#include -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" -#pragma GCC diagnostic ignored "-Wmissing-field-initializers" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wpragmas" -using namespace visp; - -enum PADDING_MODE { NONE = -INT16_MAX, YES = 1 }; -namespace visp::yolov9t { - -// main(sam3) 의 concat(model_ref, std::array, dim) 시그니처와 달리 -// yolov9t 는 가변 개수 tensor 를 concat 한다 → ggml_concat 순차 fold 로 동일 결과 제공. -static tensor yolo_concat(model_ref m, std::vector ts, int dim) { - tensor out = nullptr; - for (tensor t : ts) { - if (t == nullptr) continue; - out = out ? ggml_concat(m, out, t, dim) : t; - } - return out; -} - - -yolov9t_params yolov9t_detect_params(model_file const& /*file*/) { - yolov9t_params params; - - - params.num_classes = 80; - params.input_size = 640; - params.variant = "tiny"; - - - return params; -} -int autopad(int k, int p = -1, int d = 1) { - if (d > 1) { - k = d * (k - 1) + 1; - } - if (p == -1) { - p = k / 2; - } - return p; -} - - -inline tensor add_bias_if_exists(model_ref m, tensor x, const std::string& full_prefix) { - auto old = m.prefix; - m.prefix = tensor_name(full_prefix.c_str()); - tensor b = m.find("bias"); - m.prefix = old; - if (!b) return x; - - GGML_ASSERT(b->ne[0] == x->ne[0]); - b = ggml_reshape_4d(m, b, b->ne[0], 1, 1, 1); - return ggml_add(m, x, b); -} - - -tensor Conv( - model_ref m, - tensor x, - std::string const& name, - int c1, - int c2, - int k, - int s, - int p, - bool act, - bool bn, - bool debug) { - (void)debug; - int auto_pad = autopad(k, p, 1); - - - tensor_name old_prefix = m.prefix; - - - std::string weight_name = name; - - - if (name.find(".conv") == std::string::npos) { - weight_name += ".conv"; - } - m.prefix = tensor_name(weight_name.c_str()); - - if (!ggml_is_contiguous(x)) { - x = ggml_cont(m, x); - } - - - x = conv_2d(m, x, s, auto_pad); - if (bn){ - std::string bn_name = name.c_str(); - if (auto pos = bn_name.rfind(".conv"); pos != std::string::npos) { - bn_name.replace(pos, std::string(".conv").size(), ".bn"); - } - if (bn_name.find(".bn") == std::string::npos) { - bn_name += ".bn"; - } - m.prefix = tensor_name(bn_name.c_str()); - x = batch_norm_2d(m, x); - } - - if (act) { - x = ggml_silu(m, x); - } - - - m.prefix = old_prefix; - return x; -} - - - -/* -class AConv(nn.Module): - """Average pooling convolution for downsampling""" - def __init__(self, c1, c2): - super().__init__() - self.cv1 = Conv(c1, c2, 3, 2, 1) # 단순한 3x3 stride=2 컨볼루션 - self.avgpool = nn.AvgPool2d(2, 1, 0, False, True) - - def forward(self, x): - pool = self.avgpool(x) - x = self.cv1(pool) - return x -*/ -tensor conv_2d_batch_norm(model_ref m, tensor x, int stride = 1, int pad = 0) { - return conv_2d(m, x, stride, pad); -} -tensor mean_2d(model_ref m, tensor x) { - auto [w, h, c, n] = nelements_whcn(m, x); - x = contiguous_2d_to_whcn(m, x); - x = ggml_reshape_3d(m, x, w * h, c, n); - x = ggml_mean(m, x); - x = is_cwhn(m) ? ggml_reshape_4d(m, x, c, 1, 1, n) : ggml_reshape_4d(m, x, 1, 1, c, n); - return x; -} - -tensor global_avg_pool(model_ref m, tensor x) { - x = mean_2d(m, x); - x = conv_2d_batch_norm(m[1], x); - x = ggml_relu_inplace(m, x); - return named(m, x); -} -tensor AConv(model_ref m, tensor x, std::string const& name, int c1, int c2, bool debug) { - - - - x = contiguous_2d_to_whcn(m, x); - - tensor pooled = ggml_pool_2d( - m, x, - GGML_OP_POOL_AVG, - /*kW*/ 2, /*kH*/ 2, - /*sW*/ 1, /*sH*/ 1, - /*pW*/ 0, /*pH*/ 0 - ); - - pooled = whcn_to_contiguous_2d(m, pooled); - - - - tensor out = Conv(m, pooled, name + ".cv1.conv", c1, c2, /*k=*/3, /*stride=*/2, /*pad=*/1, /*act=*/true); - return out; -} - -tensor ELAN1( - model_ref m, - tensor x, - std::string const& name, - int c1, - int c2, - int c3, - int c4, - bool debug -) { - - - - GGML_ASSERT(c3 % 2 == 0 && "ELAN1: c3 must be even for chunk(2)"); - const int c_half = c3 / 2; - tensor cv1_out = Conv(m, x, name + ".cv1.conv", c1, c3, 1, 1, -1, true); - - - - tensor split1 = slice(m, cv1_out, - /*C*/ {0, c_half}, - /*W*/{}, /*H*/ {}, /*N*/ {}); - tensor split2 = slice(m, cv1_out, - /*C*/ {c_half, c3}, - /*W*/ {}, /*H*/ {}, /*N*/ {}); - - - if (!ggml_is_contiguous(split1)) split1 = ggml_cont(m, split1); - if (!ggml_is_contiguous(split2)) split2 = ggml_cont(m, split2); - - - tensor cv2_out = Conv(m, split2, name + ".cv2.conv", c_half, c4, 3, 1, -1, true); - - - tensor cv3_out = Conv(m, cv2_out, name + ".cv3.conv", c4, c4, 3, 1, -1, true); - - - tensor cat12 = Concat(m, split1, split2, /*dim=*/0); - tensor cat123 = Concat(m, cat12, cv2_out, /*dim=*/0); - tensor cat1234 = Concat(m, cat123, cv3_out, /*dim=*/0); - - - - tensor out = Conv(m, cat1234, name + ".cv4.conv", c3 + (2 * c4), c2, 1, 1, -1, true); - - - return out; -} - -tensor Bottleneck( - model_ref m, - tensor x, - std::string const& name, - int c1, - int c2, - bool shortcut, - int g, - float e, - bool debug) { - int c_ = (int)(c2 * e); - - - tensor cv1_out = Conv(m, x, name + ".cv1.conv", c1, c_, 1, 1, -1, true); - tensor cv2_out = Conv(m, cv1_out, name + ".cv2.conv", c_, c2, 3, 1, -1, true); - - - return cv2_out; -} - -tensor RepConv( - model_ref m, - tensor x, - std::string const& name, - int c1, - int c2, - int k, - int s, - int p, - int g, - int d, - bool act, - bool bn, - bool deploy, - bool debug) { - - - if (k != 3 || p != 1) { - throw std::invalid_argument("RepConv requires k=3 and p=1"); - } - - tensor output; - - if (deploy) { - - output = Conv(m, x, name + ".fused_conv", c1, c2, k, s, p, act, debug); - } else { - - - std::string cv11_name = name; - - tensor conv1_out = Conv(m, x, cv11_name + ".conv1.conv", c1, c2, k, s, p, false, debug); - tensor conv2_out = Conv(m, x, cv11_name + ".conv2.conv", c1, c2, 1, s, (p-k)/2, false, debug); - - output = ggml_add(m, conv1_out, conv2_out); - - - if (act) { - output = ggml_silu(m, output); - } - } - - - return output; -} - - - -tensor RepBottleneck( - model_ref m, - tensor x, - std::string const& name, - int c1, - int c2, - bool shortcut, - int g, - int k, - float e, - bool debug) { - - int c_ = (int)(c2 * e); - - - tensor cv1_out = RepConv(m, x, name + ".cv1", c1, c_, 3, 1, 1, 1, 1, true, false, false, debug); - - - tensor cv2_out = Conv(m, cv1_out, name + ".cv2", c_, c2, 3, 1, -1, true, debug); - - - if (shortcut && c1 == c2) { - tensor result = ggml_add(m, x, cv2_out); - return result; - } - - - return cv2_out; -} - -tensor C3( - model_ref m, - tensor x, - std::string const& name, - int c1, - int c2, - int n, - bool shortcut, - int g, - float e, - bool debug) { - int c_ = (int)(c2 * e); - - - tensor cv1_out = Conv(m, x, name + ".cv1", c1, c_, 1, 1, -1, true); - - - tensor cv2_out = Conv(m, x, name + ".cv2", c1, c_, 1, 1, -1, true); - - - tensor m_out = cv1_out; - for (int i = 0; i < n; ++i) { - std::string bottleneck_name = name + ".m." + std::to_string(i); - m_out = Bottleneck(m, m_out, bottleneck_name, c_, c_, shortcut, g, 1.0); - } - - - tensor concat = ggml_concat(m, m_out, cv2_out, 0); - concat = ggml_cont(m, concat); - - tensor output = Conv(m, concat, name + ".cv3", 2 * c_, c2, 1, 1, -1, true); - - return output; -} - - -tensor RepCSP( - model_ref m, - tensor x, - std::string const& name, - int c1, - int c2, - int n, - bool shortcut, - int g, - float e, - bool debug) { - - int c_ = (int)(c2 * e); - - - tensor cv1_out = Conv(m, x, name + ".cv1", c1, c_, 1, 1, -1, true); - tensor cv2_out = Conv(m, x, name + ".cv2", c1, c_, 1, 1, -1, true); - - - tensor m_out = cv1_out; - for (int i = 0; i < n; ++i) { - std::string bottleneck_name = name + ".m." + std::to_string(i); - m_out = RepBottleneck(m, m_out, bottleneck_name, c_, c_, shortcut, g, 3, 1.0); - } - - - tensor concat = ggml_concat(m, m_out, cv2_out, 0); - concat = ggml_cont(m, concat); - - tensor output = Conv(m, concat, name + ".cv3", 2 * c_, c2, 1, 1, -1, true); - - return output; -} - - - -tensor RepNCSPELAN4( - model_ref m, tensor x, std::string const& name, int c1, int c2, int c3, int c4, int n, bool debug) { - - int c = c3 / 2; - - tensor cv1_out = Conv(m, x, name + ".cv1", c1, c3, /*k=*/1, /*stride=*/1, /*pad=*/0, /*bias=*/true); - - - tensor cv1_out_h0 = slice(m, cv1_out, - /*C*/ slice_t(0, c), - /*W*/ slice_t(), /*H*/ slice_t(), /*N*/ slice_t()); - tensor cv1_out_h1 = slice(m, cv1_out, - /*C*/ slice_t(c, c3), - /*W*/ slice_t(), /*H*/ slice_t(), /*N*/ slice_t()); - - if (!ggml_is_contiguous(cv1_out_h0)) - cv1_out_h0 = ggml_cont(m, cv1_out_h0); - if (!ggml_is_contiguous(cv1_out_h1)) - cv1_out_h1 = ggml_cont(m, cv1_out_h1); - - auto rep_conv = [=](model_ref m, tensor x, std::string name, int c_in, int c_out, int n) { - tensor y2 = RepCSP(m, x, name+ ".0", c_in, c_out, n); - return Conv(m, y2, name + ".1", c_out, c_out, 3, 1, 1, true); - }; - - - tensor cv2 = rep_conv(m, cv1_out_h1, name + ".cv2", c3 / 2, c4, n); - tensor cv3 = rep_conv(m, cv2, name + ".cv3", c4, c4, n); - - tensor cat = yolo_concat(m, {cv1_out_h0, cv1_out_h1, cv2, cv3}, 0); - if (!ggml_is_contiguous(cat)) - { - cat = ggml_cont(m, cat); - } - - - tensor output = Conv(m, cat, name + ".cv4", c3 + (2 * c4), c2, 1, 1, -1, true); - - return output; -} - - - -tensor SPPELAN(model_ref m, tensor x, std::string const& name, int c1, int c2, int c3, int k, bool debug) { - - tensor cv1 = Conv(m, x, name + ".cv1", c1, c3, 1, 1, -1, true); - - - int pad = k / 2; - - - - - auto max_pool = [=](model_ref m, tensor x) { - x = contiguous_2d_to_whcn(m, x); - x = ggml_pool_2d(m, x, GGML_OP_POOL_MAX, k, k, 1, 1, pad, pad); - x = whcn_to_contiguous_2d(m, x); - return x; - }; - tensor cv2 = max_pool(m, cv1); - tensor cv3 = max_pool(m, cv2); - tensor cv4 = max_pool(m, cv3); - - - - - tensor cv5 = yolo_concat(m, {cv1, cv2, cv3, cv4}, 0); - cv5 = ggml_cont(m, cv5); - - tensor output = Conv(m, cv5, name + ".cv5", 4*c3, c2, 1, 1, -1, true); - - - - - - - return output; -} - - -tensor Upsample(model_ref m, tensor x, int scale_factor, bool debug) { - - x = permute_cwhn_to_whcn(m, x); - x = ggml_upscale(m, x, scale_factor, GGML_SCALE_MODE_NEAREST); - x = permute_whcn_to_cwhn(m, x); - x = ggml_cont(m, x); - return x; -} - -tensor Concat(model_ref m, tensor a, tensor b, int axis, bool debug) { - - - int dim = (m.flags & model_build_flag::cwhn) ? 0 : 2; - tensor output = ggml_concat(m, a, b, dim); - - output = ggml_cont(m, output); - return output; -} - - - -std::map yolov9t_backbone(model_ref m, tensor x) { - std::map features; - - tensor x0 = Conv(m, x, "model.0", 3, 16, 3, 2, -1, true, false); - features[0] = x0; - ggml_set_output(x0); - - - tensor x1 = Conv(m, x0, "model.1", 16, 32, 3, 2, -1, true, false); - features[1] = x1; - ggml_set_output(x1); - - - tensor x2 = ELAN1(m, x1, "model.2", 32, 32, 32, 16); - ggml_set_output(x2); - features[2] = x2; - - - tensor x3 = AConv(m, x2, "model.3", 32, 64); - ggml_set_output(x3); - features[3] = x3; - - - tensor x4 = RepNCSPELAN4(m, x3, "model.4", 64, 64, 64, 32, 3, false); - features[4] = x4; - ggml_set_output(x4); - - - tensor x5 = AConv(m, x4, "model.5", 64, 96); - features[5] = x5; - ggml_set_output(x5); - - - tensor x6 = RepNCSPELAN4(m, x5, "model.6", 96, 96, 96, 48, 3); - features[6] = x6; - ggml_set_output(x6); - - - tensor x7 = AConv(m, x6, "model.7", 96, 128); - features[7] = x7; - ggml_set_output(x7); - - - tensor x8 = RepNCSPELAN4(m, x7, "model.8", 128, 128, 128, 64, 3); - features[8] = x8; - ggml_set_output(x8); - - - tensor x9 = SPPELAN(m, x8, "model.9", 128, 128, 64, 5, false); - features[9] = x9; - ggml_set_output(x9); - - - tensor x10 = Upsample(m, x9, 2); - features[10] = x10; - ggml_set_output(x10); - - - - - - - tensor x11 = Concat(m, x10, features[6], 2); - features[11] = x11; - ggml_set_output(x11); - - - - - tensor x12 = RepNCSPELAN4(m, x11, "model.12", 224, 96, 96, 48, 3, false); - features[12] = x12; - ggml_set_output(x12); - - - tensor x13 = Upsample(m, x12, 2); - features[13] = x13; - ggml_set_output(x13); - - - tensor x14 = Concat(m, x13, features[4], 2); - features[14] = x14; - ggml_set_output(x14); - - - tensor x15 = RepNCSPELAN4(m, x14, "model.15", 160, 64, 64, 32, 3); - features[15] = x15; - ggml_set_output(x15); - - - tensor x16 = AConv(m, x15, "model.16", 64, 48); - features[16] = x16; - ggml_set_output(x16); - - - tensor x17 = Concat(m, x16, features[12]); - features[17] = x17; - ggml_set_output(x17); - - - tensor x18 = RepNCSPELAN4(m, x17, "model.18", 144, 96, 96, 48, 3); - features[18] = x18; - ggml_set_output(x18); - - - tensor x19 = AConv(m, x18, "model.19", 96, 64); - features[19] = x19; - ggml_set_output(x19); - - - tensor x20 = Concat(m, x19, features[9]); - features[20] = x20; - ggml_set_output(x20); - - - tensor x21 = RepNCSPELAN4(m, x20, "model.21", 192, 128, 128, 64, 3); - features[21] = x21; - ggml_set_output(x21); - - - - return features; -} - - -tensor dist2bbox(model_ref m, tensor distance, tensor anchor_points, tensor stride_tensor, bool xywh) { - - - distance = ggml_cont(m, distance); - distance = ggml_permute(m, distance, 2, 0, 1, 3); - distance = ggml_cont(m, distance); - GGML_ASSERT(distance->ne[0] == 4); - GGML_ASSERT(anchor_points->ne[0] == 2); - - tensor lt = slice(m, distance, slice_t{0, 2}, slice_t{}, slice_t{}, slice_t{}); - tensor rb = slice(m, distance, slice_t{2, 4}, slice_t{}, slice_t{}, slice_t{}); - - - tensor x1y1 = ggml_sub(m, anchor_points, lt); - tensor x2y2 = ggml_add(m, anchor_points, rb); - - if (xywh) { - tensor c_xy = ggml_scale(m, ggml_add(m, x1y1, x2y2), 0.5f); - tensor wh = ggml_sub(m, x2y2, x1y1); - return ggml_concat(m, c_xy, wh, 0); - } else { - return ggml_concat(m, x1y1, x2y2, 0); - } - -} - -tensor dfl_forward(model_ref m, tensor proj_tensor, tensor x, int reg_max, bool debug) { - - - - if (!ggml_is_contiguous(x)) x = ggml_cont(m, x); - - - int64_t e0 = x->ne[0]; - int64_t e1 = x->ne[1]; - int64_t e2 = x->ne[2]; - int64_t e3 = x->ne[3]; - int64_t total_elements = e0 * e1 * e2 * e3; - int64_t batch = e3; - int64_t expected_per_anchor = int64_t(reg_max) * 4 * batch; - if (expected_per_anchor == 0) { - throw std::runtime_error("Invalid reg_max or batch size in dfl_forward"); - } - if (total_elements % expected_per_anchor != 0) { - throw std::runtime_error("dfl_forward: input size not divisible by reg_max*4*batch"); - } - int64_t anchors = total_elements / expected_per_anchor; - - tensor reshaped = ggml_reshape_4d(m, x, reg_max, 4, anchors, batch); - - tensor soft = ggml_soft_max(m, reshaped); - - if (!ggml_is_contiguous(proj_tensor)) proj_tensor = ggml_cont(m, proj_tensor); - tensor proj = ggml_reshape_4d(m, proj_tensor, reg_max, 1, 1, 1); - - tensor weighted = ggml_mul(m, soft, proj); - tensor summed = ggml_sum_rows(m, weighted); - - if (!ggml_is_contiguous(summed)) summed = ggml_cont(m, summed); - return summed; -} - -DetectOutput inference(model_ref m, - DetectOutput out, - std::vector ch, - int reg_max, - int nc) { - - std::vector reshaped_outputs; - - tensor x_cat = yolo_concat(m, out.features, 1); - - - int64_t total_channels = x_cat->ne[0]; - int64_t total_anchors = x_cat->ne[1]; - - std::vector strides_vec; - for (size_t i = 0; i < out.features.size(); ++i) { - strides_vec.push_back(8.0f * std::pow(2.0f, (float)i)); - } - std::vector anchor_host; - std::vector stride_host; - auto [anchor_points, stride_tensor] = make_anchors(m, out, anchor_host, stride_host, strides_vec, 0.5f); - - - tensor box = ggml_view_4d(m, x_cat, - reg_max * 4, total_anchors, 1, 1, - x_cat->nb[1], x_cat->nb[2], x_cat->nb[3], - 0); - - box = ggml_cont(m, box); - - - tensor cls = ggml_view_4d(m, x_cat, - nc, total_anchors, 1, 1, - x_cat->nb[1], x_cat->nb[2], x_cat->nb[3], - reg_max * 4 * x_cat->nb[0]); - cls = ggml_cont(m, cls); - - - tensor cls_logits = ggml_dup(m, cls); - ggml_set_output(cls_logits); - out.debug_cls_logits = cls_logits; - - - - tensor proj = ggml_new_tensor_1d(m, GGML_TYPE_F32, reg_max); - ggml_set_name(proj, "dfl_proj"); - out.dfl_proj = proj; - out.dfl_proj_host_data.resize(reg_max); - for (int i = 0; i < reg_max; ++i) out.dfl_proj_host_data[i] = float(i); - out.reg_max = reg_max; - - - tensor dfl_output = dfl_forward(m, proj, box, reg_max, false); - - - tensor dbox = dist2bbox(m, dfl_output, anchor_points, stride_tensor, false); - - - tensor strides_bc = ggml_reshape_4d(m, stride_tensor, 1, total_anchors, 1, 1); - - dbox = ggml_mul(m, dbox, strides_bc); - - - - cls = ggml_sigmoid(m, cls); - ggml_set_name(cls, "cls_prob"); - - - out.predictions_bbox = std::move(dbox); - out.predictions_cls = std::move(cls); - out.anchor_points = std::move(anchor_points); - out.strides_points = std::move(stride_tensor); - out.anchor_host_data = std::move(anchor_host); - out.stride_host_data = std::move(stride_host); - return out; -} - -DetectOutput detect_forward(model_ref m, - std::vector features, - std::vector ch, - int nc, - bool training) { - int reg_max = 16; - int c2 = std::max({16, ch[0] / 4, reg_max * 4}); - int c3 = std::max(ch[0], std::min(nc, 100)); - DetectOutput out; - - std::string reg_base = std::string("detect.cv2"); - std::string cls_base = std::string("detect.cv3"); - - for (size_t i = 0; i < features.size(); ++i) { - - tensor_name old_prefix_i = m.prefix; - tensor r0 = Conv(m, features[i], reg_base + "."+std::to_string(i)+".0.conv", ch[i], c2, 3, 1, -1, true, false); - tensor r1 = Conv(m, r0 ,reg_base + "."+std::to_string(i)+".1.conv", c2, c2, 3, 1, -1, true, false); - - m.prefix = tensor_name((reg_base + "."+std::to_string(i)+".2").c_str()); - tensor r2 = conv_2d(m, r1, 1, 0); - - if (!ggml_is_contiguous(r2)) r2 = ggml_cont(m, r2); - - tensor c0 = Conv(m, features[i], cls_base + "."+std::to_string(i)+".0.conv", ch[i], c3, 3, 1, -1, true, false); - tensor c1 = Conv(m, c0, cls_base + "."+std::to_string(i)+".1.conv", c3, c3, 3, 1, -1, true, false); - - m.prefix = tensor_name((cls_base + "." + std::to_string(i) + ".2").c_str()); - tensor c2 = conv_2d(m, c1, 1, 0); - - if (!ggml_is_contiguous(c2)) c2 = ggml_cont(m, c2); - - tensor combined = Concat(m, r2, c2, 0); - - tensor reshaped_combined = ggml_reshape_2d(m, combined, combined->ne[0], combined->ne[1]*combined->ne[2]); - - - combined = ggml_cont(m, combined); - reshaped_combined = ggml_cont(m, reshaped_combined); - out.raw_outputs.push_back(std::move(combined)); - out.features.push_back(std::move(reshaped_combined)); - - m.prefix = old_prefix_i; - } - - if (training) { - out.predictions_cls = nullptr; - out.predictions_bbox = nullptr; - return out; - } - - DetectOutput detect = inference(m, out, ch, reg_max, nc); - - return detect; -} - - -DetectOutput yolov9t_forward(model_ref m, tensor x) { - - std::map features = yolov9t_backbone(m, x); - - - std::vector channels = {64, 96, 128}; - - std::vector features_vector = {features[15], features[18], features[21]}; - DetectOutput d = detect_forward(m, features_vector, channels, 80, false); - - d.features_map = features; - - return d; -} - - -std::vector const& get_coco_class_names() { - static std::vector const class_names = { - "person", "bicycle", "car", - "motorcycle", "airplane", "bus", - "train", "truck", "boat", - "traffic light", "fire hydrant", "stop sign", - "parking meter", "bench", "bird", - "cat", "dog", "horse", - "sheep", "cow", "elephant", - "bear", "zebra", "giraffe", - "backpack", "umbrella", "handbag", - "tie", "suitcase", "frisbee", - "skis", "snowboard", "sports ball", - "kite", "baseball bat", "baseball glove", - "skateboard", "surfboard", "tennis racket", - "bottle", "wine glass", "cup", - "fork", "knife", "spoon", - "bowl", "banana", "apple", - "sandwich", "orange", "broccoli", - "carrot", "hot dog", "pizza", - "donut", "cake", "chair", - "couch", "potted plant", "bed", - "dining table", "toilet", "tv", - "laptop", "mouse", "remote", - "keyboard", "cell phone", "microwave", - "oven", "toaster", "sink", - "refrigerator", "book", "clock", - "vase", "scissors", "teddy bear", - "hair drier", "toothbrush"}; - return class_names; -} - -std::pair make_anchors( - model_ref m, - DetectOutput const& out, - std::vector& anchor_host, - std::vector& stride_host, - std::vector const& strides, - float grid_cell_offset) { - - GGML_ASSERT(out.raw_outputs.size() == strides.size()); - GGML_ASSERT(!out.raw_outputs.empty()); - - - int64_t total_anchors = 0; - for (size_t i = 0; i < out.raw_outputs.size(); ++i) { - auto ne = nelements(out.raw_outputs[i]); - total_anchors += ne[1] * ne[2]; - } - - - - std::vector anchor_data(2 * total_anchors); - std::vector stride_data(total_anchors); - - int64_t offset = 0; - for (size_t i = 0; i < strides.size(); ++i) { - int64_t w = out.raw_outputs[i]->ne[1]; - int64_t h = out.raw_outputs[i]->ne[2]; - - for (int64_t y = 0; y < h; ++y) { - for (int64_t x = 0; x < w; ++x) { - int64_t idx = offset + y * w + x; - anchor_data[idx * 2 + 0] = x + grid_cell_offset; - anchor_data[idx * 2 + 1] = y + grid_cell_offset; - stride_data[idx] = strides[i]; - } - } - offset += w * h; - } - - tensor anchor_points = compute_graph_input( - m, GGML_TYPE_F32, {2, total_anchors, 1, 1}, "anchor_points"); - tensor stride_tensor = compute_graph_input( - m, GGML_TYPE_F32, {1, total_anchors, 1, 1}, "stride_points"); - - anchor_host = std::move(anchor_data); - - stride_host = std::move(stride_data); - - - return std::make_pair(anchor_points, stride_tensor); -} - -int make_divisible(int x, int divisor) { - - if (divisor < 1) { - return x; - } - return (x + divisor - 1) / divisor * divisor; -} -int check_img_size(int imgsz, int s, int floor) { - int new_size = std::max(make_divisible(imgsz, int(s)), floor); - if (new_size != imgsz) { - printf("WARNING ⚠️ --img-size %d must be multiple of max stride %d, updating to %d\n", imgsz, s, new_size); - } - return new_size; -} - -image_data image_add_border(image_data im, int top, int bottom, - int left, int right, u8x3 color) { - - int W = im.extent[0]; - int H = im.extent[1]; - int channels = 3; - - i32x2 new_extent = { W + left + right, H + top + bottom }; - image_data bordered = image_alloc(new_extent, im.format); - - - for (int y = 0; y < new_extent[1]; ++y) { - for (int x = 0; x < new_extent[0]; ++x) { - int idx = (y * new_extent[0] + x) * channels; - bordered.data[idx+0] = color[0]; - bordered.data[idx+1] = color[1]; - bordered.data[idx+2] = color[2]; - } - } - - - for (int y = 0; y < H; ++y) { - for (int x = 0; x < W; ++x) { - int src = (y * W + x) * channels; - int dst = ((y + top) * new_extent[0] + (x + left)) * channels; - bordered.data[dst+0] = im.data[src+0]; - bordered.data[dst+1] = im.data[src+1]; - bordered.data[dst+2] = im.data[src+2]; - } - } - return bordered; -} - - -void resize_rgb_bilinear( - uint8_t* src_data, - i32x2 src_extent, - uint8_t* dst_data, - i32x2 dst_extent) { - - double inv_scale_x = static_cast(src_extent[0]) / dst_extent[0]; - double inv_scale_y = static_cast(src_extent[1]) / dst_extent[1]; - - for (int y = 0; y < dst_extent[1]; ++y) { - for (int x = 0; x < dst_extent[0]; ++x) { - double src_xf = (x + 0.5) * inv_scale_x - 0.5; - double src_yf = (y + 0.5) * inv_scale_y - 0.5; - - src_xf = std::max(0.0, src_xf); - src_yf = std::max(0.0, src_yf); - - int x0 = static_cast(src_xf); - int y0 = static_cast(src_yf); - int x1 = std::min(x0 + 1, src_extent[0] - 1); - int y1 = std::min(y0 + 1, src_extent[1] - 1); - - double wx = src_xf - x0; - double wy = src_yf - y0; - double w00 = (1 - wx) * (1 - wy); - double w01 = wx * (1 - wy); - double w10 = (1 - wx) * wy; - double w11 = wx * wy; - - - for (int c = 0; c < 3; ++c) { - double value = - src_data[(y0 * src_extent[0] + x0) * 3 + c] * w00 + - src_data[(y0 * src_extent[0] + x1) * 3 + c] * w01 + - src_data[(y1 * src_extent[0] + x0) * 3 + c] * w10 + - src_data[(y1 * src_extent[0] + x1) * 3 + c] * w11; - - dst_data[(y * dst_extent[0] + x) * 3 + c] = - static_cast(std::min(255.0, std::max(0.0, value + 0.5))); - } - } - } -} - - -image_data linear_image_resize(image_data im, i32x2 new_shape) { - image_data resized = image_alloc(new_shape, im.format); - resize_rgb_bilinear(im.data.get(), im.extent, resized.data.get(), new_shape); - return resized; -} - - -LetterboxResult letterbox(image_data im, i32x2 new_shape, u8x3 color, - bool _auto, bool scaleFill, bool scaleup, int stride) -{ - i32x2 shape = im.extent; - int orig_w = shape[0]; - int orig_h = shape[1]; - - if (new_shape[0] == 0) - new_shape = {new_shape[1], new_shape[1]}; - - - - float r = std::min( - (float)new_shape[1] / (float)orig_h, - (float)new_shape[0] / (float)orig_w - ); - - if (!scaleup) - r = std::min(r, 1.0f); - - - int new_unpad_w = std::round(orig_w * r); - int new_unpad_h = std::round(orig_h * r); - - float dw = (float)new_shape[0] - new_unpad_w; - float dh = (float)new_shape[1] - new_unpad_h; - - if (_auto) { - dw = std::fmod(dw, stride); - dh = std::fmod(dh, stride); - } else if (scaleFill) { - dw = 0.0f; - dh = 0.0f; - new_unpad_w = new_shape[0]; - new_unpad_h = new_shape[1]; - } - - dw /= 2.0f; - dh /= 2.0f; - - im = image_scale(im, {new_unpad_w, new_unpad_h}); - - int left = std::round(dw - 0.1f); - int right = std::round(dw + 0.1f); - int top = std::round(dh - 0.1f); - int bottom = std::round(dh + 0.1f); - im = image_add_border(std::move(im), top, bottom, left, right, color); - - - LetterboxResult out; - out.img = std::move(im); - out.gain = r; - out.pad_w = dw; - out.pad_h = dh; - return out; -} - - -inline void xywh2xyxy(float* boxes, int n) { - - for (int i = 0; i < n; ++i) { - float x = boxes[i * 4 + 0]; - float y = boxes[i * 4 + 1]; - float w = boxes[i * 4 + 2]; - float h = boxes[i * 4 + 3]; - boxes[i * 4 + 0] = x - w / 2.0f; - boxes[i * 4 + 1] = y - h / 2.0f; - boxes[i * 4 + 2] = x + w / 2.0f; - boxes[i * 4 + 3] = y + h / 2.0f; - } -} - - -inline float box_iou(const float* b1, const float* b2) { - - float ix1 = std::max(b1[0], b2[0]); - float iy1 = std::max(b1[1], b2[1]); - float ix2 = std::min(b1[2], b2[2]); - float iy2 = std::min(b1[3], b2[3]); - float iw = std::max(0.0f, ix2 - ix1); - float ih = std::max(0.0f, iy2 - iy1); - float inter = iw * ih; - float area1 = (b1[2] - b1[0]) * (b1[3] - b1[1]); - float area2 = (b2[2] - b2[0]) * (b2[3] - b2[1]); - float union_ = area1 + area2 - inter; - return union_ > 0.f ? (inter / union_) : 0.f; -} - - -std::vector nms(const std::vector>& boxes, const std::vector& scores, const std::vector& class_ids, float iou_thres, bool agnostic, int max_wh) { - std::vector order(boxes.size()); - std::iota(order.begin(), order.end(), 0); - std::sort(order.begin(), order.end(), [&](int a, int b) { - return scores[a] > scores[b]; - }); - std::vector keep(boxes.size(), true); - std::vector kept; - for (size_t i = 0; i < order.size(); ++i) { - int idx_i = order[i]; - if (!keep[idx_i]) continue; - kept.push_back(idx_i); - const float* b1 = boxes[idx_i].data(); - int ci = agnostic ? 0 : class_ids[idx_i] * max_wh; - for (size_t j = i+1; j < order.size(); ++j) { - int idx_j = order[j]; - if (!keep[idx_j]) continue; - int cj = agnostic ? 0 : class_ids[idx_j] * max_wh; - float b2[4] = {boxes[idx_j][0]+cj, boxes[idx_j][1], boxes[idx_j][2]+cj, boxes[idx_j][3]}; - float bb1[4] = {b1[0]+ci, b1[1], b1[2]+ci, b1[3]}; - if (box_iou(bb1, b2) > iou_thres) { - keep[idx_j] = false; - } - } - } - return kept; -} - -std::vector non_max_suppression( - DetectOutput const& outputs, - float conf_thres, - float iou_thres, - int max_det, - int max_nms, - int max_wh -) { - if (!outputs.predictions_cls || !outputs.predictions_bbox) { - return {}; - } - - if (!(0.0f <= conf_thres && conf_thres <= 1.0f)) throw std::runtime_error("Invalid Confidence threshold"); - if (!(0.0f <= iou_thres && iou_thres <= 1.0f)) throw std::runtime_error("Invalid IoU threshold"); - - tensor_data td_cls = transfer_from_backend(outputs.predictions_cls); - tensor_data td_box = transfer_from_backend(outputs.predictions_bbox); - const float* cls_ptr = td_cls.as_f32().data(); - const float* box_ptr = td_box.as_f32().data(); - const int64_t nc = outputs.predictions_cls->ne[0]; - const int64_t na = outputs.predictions_cls->ne[1]; - float max_conf = -1.0f; - float min_conf = 2.0f; - int max_conf_anchor = -1; - int max_conf_class = -1; - - for (int64_t j = 0; j < na; ++j) { - for (int64_t c = 0; c < nc; ++c) { - float conf = cls_ptr[c + j*nc]; - if (conf > max_conf) { - max_conf = conf; - max_conf_anchor = j; - max_conf_class = c; - } - min_conf = std::min(min_conf, conf); - } - } - - if (max_conf_anchor >= 0) { - float x1 = box_ptr[0 + max_conf_anchor*4]; - float y1 = box_ptr[1 + max_conf_anchor*4]; - float x2 = box_ptr[2 + max_conf_anchor*4]; - float y2 = box_ptr[3 + max_conf_anchor*4]; - } - std::vector> boxes; - std::vector scores; - std::vector class_ids; - boxes.reserve(na); - scores.reserve(na); - class_ids.reserve(na); - - for (int64_t j = 0; j < na; ++j) { - float x1 = box_ptr[0 + j*4]; - float y1 = box_ptr[1 + j*4]; - float x2 = box_ptr[2 + j*4]; - float y2 = box_ptr[3 + j*4]; - for (int64_t c = 0; c < nc; ++c) { - float conf = cls_ptr[c + j*nc]; - if (conf < conf_thres) continue; - boxes.push_back({x1, y1, x2, y2}); - scores.push_back(conf); - class_ids.push_back((int)c); - } - } - if (boxes.empty()) { - return {}; - } - if ((int)boxes.size() > max_nms) { - std::vector order(boxes.size()); - std::iota(order.begin(), order.end(), 0); - std::partial_sort(order.begin(), order.begin() + max_nms, order.end(), - [&](int a, int b){ return scores[a] > scores[b]; }); - order.resize(max_nms); - std::vector> b2; - std::vector s2; - std::vector c2; - b2.reserve(order.size()); s2.reserve(order.size()); c2.reserve(order.size()); - for (int idx : order) { - b2.push_back(boxes[idx]); - s2.push_back(scores[idx]); - c2.push_back(class_ids[idx]); - } - boxes.swap(b2); - scores.swap(s2); - class_ids.swap(c2); - } - if (!scores.empty()) { - float max_score = *std::max_element(scores.begin(), scores.end()); - } - if (boxes.empty()) { - return {}; - } - std::vector keep = nms(boxes, scores, class_ids, iou_thres, /*agnostic=*/false, max_wh); - - if ((int)keep.size() > max_det) keep.resize(max_det); - - std::vector result; - result.reserve(keep.size()); - for (int idx : keep) { - detected_obj o; - o.x1 = boxes[idx][0]; o.y1 = boxes[idx][1]; - o.x2 = boxes[idx][2]; o.y2 = boxes[idx][3]; - o.confidence = scores[idx]; - o.class_id = class_ids[idx]; - o.class_confidence = scores[idx]; - result.push_back(o); - } - return result; -} - -void scale_boxes( - std::vector& detections, - i32x2 model_shape, - i32x2 img_shape, - float gain, - float pad_w, - float pad_h) -{ - if (gain == 0.0f) { - float ratio_w = (float)model_shape[0] / (float)img_shape[0]; - float ratio_h = (float)model_shape[1] / (float)img_shape[1]; - gain = std::min(ratio_w, ratio_h); - } - - for (auto& det : detections) { - det.x1 = (det.x1 - pad_w) / gain; - det.x2 = (det.x2 - pad_w) / gain; - det.y1 = (det.y1 - pad_h) / gain; - det.y2 = (det.y2 - pad_h) / gain; - - det.x1 = std::clamp(det.x1, 0.0f, (float)img_shape[0]); - det.x2 = std::clamp(det.x2, 0.0f, (float)img_shape[0]); - det.y1 = std::clamp(det.y1, 0.0f, (float)img_shape[1]); - det.y2 = std::clamp(det.y2, 0.0f, (float)img_shape[1]); - } - -} - -void draw_line(uint8_t* img, int width, int height, int channels, - int x1, int y1, int x2, int y2, uint8_t r, uint8_t g, uint8_t b, int thickness = 2) { - -int dx = abs(x2 - x1); -int dy = abs(y2 - y1); -int sx = (x1 < x2) ? 1 : -1; -int sy = (y1 < y2) ? 1 : -1; -int err = dx - dy; - -while (true) { - -for (int ty = -thickness/2; ty <= thickness/2; ty++) { - for (int tx = -thickness/2; tx <= thickness/2; tx++) { - int px = x1 + tx; - int py = y1 + ty; - if (px >= 0 && px < width && py >= 0 && py < height) { - int idx = (py * width + px) * channels; - img[idx + 0] = r; - img[idx + 1] = g; - img[idx + 2] = b; - } - } -} - -if (x1 == x2 && y1 == y2) break; -int e2 = 2 * err; -if (e2 > -dy) { - err -= dy; - x1 += sx; -} -if (e2 < dx) { - err += dx; - y1 += sy; -} -} -} - - -void draw_rectangle(uint8_t* img, int width, int height, int channels, - int x1, int y1, int x2, int y2, - uint8_t r, uint8_t g, uint8_t b, int thickness = 2) { - -x1 = std::max(0, std::min(x1, width - 1)); -y1 = std::max(0, std::min(y1, height - 1)); -x2 = std::max(0, std::min(x2, width - 1)); -y2 = std::max(0, std::min(y2, height - 1)); - - -draw_line(img, width, height, channels, x1, y1, x2, y1, r, g, b, thickness); -draw_line(img, width, height, channels, x2, y1, x2, y2, r, g, b, thickness); -draw_line(img, width, height, channels, x2, y2, x1, y2, r, g, b, thickness); -draw_line(img, width, height, channels, x1, y1, x1, y2, r, g, b, thickness); -} - - -void draw_filled_rectangle(uint8_t* img, int width, int height, int channels, - int x1, int y1, int x2, int y2, - uint8_t r, uint8_t g, uint8_t b) { -x1 = std::max(0, std::min(x1, width - 1)); -y1 = std::max(0, std::min(y1, height - 1)); -x2 = std::max(0, std::min(x2, width - 1)); -y2 = std::max(0, std::min(y2, height - 1)); - -for (int y = y1; y <= y2; y++) { -for (int x = x1; x <= x2; x++) { - int idx = (y * width + x) * channels; - img[idx + 0] = r; - img[idx + 1] = g; - img[idx + 2] = b; -} -} -} - - -void draw_char(uint8_t* img, int width, int height, int channels, - int x, int y, char c, uint8_t r, uint8_t g, uint8_t b) { - -static const uint8_t font_5x7[][7] = { - -{0x0E, 0x11, 0x13, 0x15, 0x19, 0x11, 0x0E}, - -{0x04, 0x0C, 0x04, 0x04, 0x04, 0x04, 0x0E}, - -{0x0E, 0x11, 0x01, 0x02, 0x04, 0x08, 0x1F}, - -{0x1F, 0x02, 0x04, 0x02, 0x01, 0x11, 0x0E}, - -{0x02, 0x06, 0x0A, 0x12, 0x1F, 0x02, 0x02}, - -{0x1F, 0x10, 0x1E, 0x01, 0x01, 0x11, 0x0E}, - -{0x06, 0x08, 0x10, 0x1E, 0x11, 0x11, 0x0E}, - -{0x1F, 0x01, 0x02, 0x04, 0x08, 0x08, 0x08}, - -{0x0E, 0x11, 0x11, 0x0E, 0x11, 0x11, 0x0E}, - -{0x0E, 0x11, 0x11, 0x0F, 0x01, 0x02, 0x0C}, - -{0x18, 0x19, 0x02, 0x04, 0x08, 0x13, 0x03} -}; - -int idx = -1; -if (c >= '0' && c <= '9') idx = c - '0'; -else if (c == '%') idx = 10; -else return; - -for (int row = 0; row < 7; row++) { -uint8_t pattern = font_5x7[idx][row]; -for (int col = 0; col < 5; col++) { - if (pattern & (1 << (4 - col))) { - int px = x + col; - int py = y + row; - if (px >= 0 && px < width && py >= 0 && py < height) { - int pidx = (py * width + px) * channels; - img[pidx + 0] = r; - img[pidx + 1] = g; - img[pidx + 2] = b; - } - } -} -} -} - - -void draw_text(uint8_t* img, int width, int height, int channels, - int x, int y, const char* text, - uint8_t r, uint8_t g, uint8_t b) { -int offset = 0; -while (*text) { -draw_char(img, width, height, channels, x + offset, y, *text, r, g, b); -offset += 6; -text++; -} -} - - -void get_class_color(int class_id, uint8_t& r, uint8_t& g, uint8_t& b) { - -int h = (class_id * 137) % 360; -float s = 0.8f; -float v = 0.95f; - - -float c = v * s; -float x = c * (1 - std::fabs(std::fmod(h / 60.0f, 2) - 1)); -float m = v - c; - -float r1, g1, b1; -if (h < 60) { r1 = c; g1 = x; b1 = 0; } -else if (h < 120) { r1 = x; g1 = c; b1 = 0; } -else if (h < 180) { r1 = 0; g1 = c; b1 = x; } -else if (h < 240) { r1 = 0; g1 = x; b1 = c; } -else if (h < 300) { r1 = x; g1 = 0; b1 = c; } -else { r1 = c; g1 = 0; b1 = x; } - -r = (uint8_t)((r1 + m) * 255); -g = (uint8_t)((g1 + m) * 255); -b = (uint8_t)((b1 + m) * 255); -} - -void draw_detections( - image_data& img, - std::vector const& detections, - std::vector const& class_names) { - - - int width = img.extent[0]; - int height = img.extent[1]; - int channels = 3; - - - uint8_t* img_data = nullptr; - - - if (img.format == image_format::rgb_u8 || img.format == image_format::rgba_u8) { - img_data = img.data.get(); - channels = (img.format == image_format::rgba_u8) ? 4 : 3; - } else if (img.format == image_format::rgb_f32 || img.format == image_format::rgba_f32) { - - - channels = (img.format == image_format::rgba_f32) ? 4 : 3; - - - image_data temp_u8 = image_f32_to_u8(img, - channels == 4 ? image_format::rgba_u8 : image_format::rgb_u8); - - - img_data = temp_u8.data.get(); - - - for (auto const& det : detections) { - int x1 = (int)det.x1; - int y1 = (int)det.y1; - int x2 = (int)det.x2; - int y2 = (int)det.y2; - - uint8_t r, g, b; - get_class_color(det.class_id, r, g, b); - draw_rectangle(img_data, width, height, channels, x1, y1, x2, y2, r, g, b, 3); - - const char* class_name = (det.class_id >= 0 && (size_t)det.class_id < class_names.size()) - ? class_names[det.class_id].c_str() - : "unknown"; - - char conf_str[16]; - snprintf(conf_str, sizeof(conf_str), "%d%%", (int)(det.class_confidence * 100)); - - printf("Detection: %s %s at [%d, %d, %d, %d] (obj_conf: %.2f, cls_conf: %.2f)\n", - class_name, conf_str, x1, y1, x2, y2, - det.confidence, det.class_confidence); - - int text_width = strlen(conf_str) * 6; - int text_height = 10; - int label_y = std::max(0, y1 - text_height - 2); - - draw_filled_rectangle(img_data, width, height, channels, - x1, label_y, x1 + text_width, label_y + text_height, - r, g, b); - draw_text(img_data, width, height, channels, - x1 + 2, label_y + 2, conf_str, 255, 255, 255); - } - - - img = std::move(temp_u8); - return; - } else { - printf("Error: Unsupported image format for drawing\n"); - return; - } - - - for (auto const& det : detections) { - int x1 = (int)det.x1; - int y1 = (int)det.y1; - int x2 = (int)det.x2; - int y2 = (int)det.y2; - - uint8_t r, g, b; - get_class_color(det.class_id, r, g, b); - draw_rectangle(img_data, width, height, channels, x1, y1, x2, y2, r, g, b, 3); - - const char* class_name = (det.class_id >= 0 && (size_t)det.class_id < class_names.size()) - ? class_names[det.class_id].c_str() - : "unknown"; - - char conf_str[16]; - snprintf(conf_str, sizeof(conf_str), "%d%%", (int)(det.class_confidence * 100)); - - printf("Detection: %s %s at [%d, %d, %d, %d] (obj_conf: %.2f, cls_conf: %.2f)\n", - class_name, conf_str, x1, y1, x2, y2, - det.confidence, det.class_confidence); - - int text_width = strlen(conf_str) * 6; - int text_height = 10; - int label_y = std::max(0, y1 - text_height - 2); - - draw_filled_rectangle(img_data, width, height, channels, - x1, label_y, x1 + text_width, label_y + text_height, - r, g, b); - draw_text(img_data, width, height, channels, - x1 + 2, label_y + 2, conf_str, 255, 255, 255); - } -} -} - -namespace visp::yolov9t { - - float resize_longest_side(i32x2 extent, int target_longest_side) { - int longest_side = std::max(extent[0], extent[1]); - return float(target_longest_side) / float(longest_side); -} -int scale_coord(int coord, float scale) { - return int(coord * scale + 0.5f); -} -i32x2 scale_extent(i32x2 extent, float scale) { - return i32x2{scale_coord(extent[0], scale), scale_coord(extent[1], scale)}; -} -image_data yolov9t_process_input2(image_view image, yolov9t_params const& p) { - - image_data resized; - if (image.extent[0] != p.input_size || image.extent[1] != p.input_size) { - resized = image_scale(image, {p.input_size, p.input_size}); - image = image_view(resized); - - - } - - return image_u8_to_f32(image, image_format::rgb_f32, p.offset, 1.f / p.scale); -} - -LetterboxResult yolov9t_process_input(image_data image, yolov9t_params const& p) { - - - LetterboxResult lb = letterbox( - std::move(image), - {p.input_size, p.input_size}, - u8x3{114,114,114}, - /*_auto=*/false, /*scaleFill=*/false, /*scaleup=*/true, - yolov9t_params::stride); - - image_data result; - - if (lb.img.format == image_format::rgb_u8) { - result = image_alloc({p.input_size, p.input_size}, image_format::rgb_f32); - image_u8_to_f32(lb.img, result, f32x4(p.offset), f32x4(p.scale)); - } else { - result = std::move(lb.img); - } - - LetterboxResult out; - out.img = std::move(result); - out.gain = lb.gain; - out.pad_w = lb.pad_w; - out.pad_h = lb.pad_h; - - - return out; -} - -static void write_tensor_txt(tensor t, char const* filename) { - if (!t) { - return; - } - FILE* f = fopen(filename, "w"); - if (!f) { - printf("Failed to open %s for writing\n", filename); - return; - } - - fprintf( - f, - "# %ld,%ld,%ld,%ld\n# %s\n", - (long)t->ne[0], (long)t->ne[1], (long)t->ne[2], (long)t->ne[3], ggml_type_name(t->type)); - - - tensor_data td = transfer_from_backend(t); - size_t n = ggml_nelements(t); - - if (t->type == GGML_TYPE_F32) { - auto data = td.as_f32(); - for (size_t i = 0; i < data.size(); ++i) { - fprintf(f, i + 1 == data.size() ? "%.4f\n" : "%.4f ", (double)data[i]); - } - } else if (t->type == GGML_TYPE_F16) { - const ggml_fp16_t* h = reinterpret_cast(td.data.get()); - for (size_t i = 0; i < n; ++i) { - float v = ggml_fp16_to_fp32(h[i]); - fprintf(f, i + 1 == n ? "%.4f\n" : "%.4f ", (double)v); - } - } else if (t->type == GGML_TYPE_I32) { - auto data = td.as_i32(); - for (size_t i = 0; i < data.size(); ++i) { - fprintf(f, i + 1 == data.size() ? "%d\n" : "%d ", data[i]); - } - } else { - - const unsigned char* bytes = reinterpret_cast(td.data.get()); - size_t nb = ggml_nbytes(t); - for (size_t i = 0; i < nb; ++i) { - fprintf(f, i + 1 == nb ? "%02X\n" : "%02X ", bytes[i]); - } - } - fclose(f); -} - -void save_features_to_txt(DetectOutput const& out, char const* base_path, std::vector const& keys) { - if (!base_path) { - base_path = "features"; - } - - if (!keys.empty()) { - for (int k : keys) { - auto it = out.features_map.find(k); - if (it == out.features_map.end()) continue; - std::string fname = std::string(base_path) + "_layer_" + std::to_string(k) + ".txt"; - write_tensor_txt(it->second, fname.c_str()); - printf("-> feature layer %d saved to %s\n", k, fname.c_str()); - } - return; - } - for (auto const& [k, t] : out.features_map) { - std::string fname = std::string(base_path) + "_layer_" + std::to_string(k) + ".txt"; - write_tensor_txt(t, fname.c_str()); - printf("-> feature layer %d saved to %s\n", k, fname.c_str()); - } -} - -} - -namespace visp::yolov9t { - -void save_input_to_txt(tensor input, char const* filepath) { - if (!input || !filepath) return; - FILE* f = fopen(filepath, "w"); - if (!f) { - printf("Failed to open %s for writing\n", filepath); - return; - } - fprintf(f, "# input shape C,H,W,N = %ld,%ld,%ld,%ld\n# type = %s\n", - (long)input->ne[0], (long)input->ne[1], (long)input->ne[2], (long)input->ne[3], ggml_type_name(input->type)); - - tensor_data td = transfer_from_backend(input); - if (input->type == GGML_TYPE_F32) { - auto data = td.as_f32(); - for (size_t i = 0; i < data.size(); ++i) { - fprintf(f, i + 1 == data.size() ? "%g\n" : "%g ", (double)data[i]); - } - } else if (input->type == GGML_TYPE_F16) { - const ggml_fp16_t* h = reinterpret_cast(td.data.get()); - size_t n = ggml_nelements(input); - for (size_t i = 0; i < n; ++i) { - float v = ggml_fp16_to_fp32(h[i]); - fprintf(f, i + 1 == n ? "%g\n" : "%g ", (double)v); - } - } else if (input->type == GGML_TYPE_I32) { - auto data = td.as_f32(); - for (size_t i = 0; i < data.size(); ++i) { - fprintf(f, i + 1 == data.size() ? "%g\n" : "%g ", data[i]); - } - } else { - const unsigned char* bytes = reinterpret_cast(td.data.get()); - size_t nb = ggml_nbytes(input); - for (size_t i = 0; i < nb; ++i) { - fprintf(f, i + 1 == nb ? "%02X\n" : "%02X ", bytes[i]); - } - } - fclose(f); -} - -} - -#pragma GCC diagnostic pop diff --git a/src/visp/arch/yolov9t.h b/src/visp/arch/yolov9t.h deleted file mode 100644 index f1a6564..0000000 --- a/src/visp/arch/yolov9t.h +++ /dev/null @@ -1,264 +0,0 @@ -#pragma once - -#include "visp/image.h" -#include "visp/ml.h" -#include "visp/util.h" -#include "visp/vision.h" - -#include -#include -#include - -namespace visp::yolov9t { -struct LetterboxResult { - image_data img; - float gain; - float pad_w; - float pad_h; -}; - -struct yolov9t_params { - float scale = 1.0f; - float offset = 0.0f; - int num_classes = 80; - int input_size = 640; - std::string variant = "tiny"; - - static constexpr int stride = 32; -}; - -struct detected_obj { - float x1, y1, x2, y2; - float confidence; - int class_id; - float class_confidence; -}; - -struct DetectOutput { - std::vector raw_outputs; - std::vector features; - std::map features_map; - tensor predictions_cls; - tensor predictions_bbox; - tensor anchor_points; - tensor strides_points; - tensor dfl_proj; - int reg_max = 0; - std::vector anchor_host_data; - std::vector stride_host_data; - std::vector dfl_proj_host_data; - std::vector dbg_reg_logits_1x1; - std::vector dbg_cls_logits_1x1; - std::vector dbg_reg_mid; - std::vector dbg_cls_mid; - tensor dbg_cls_logits_cat; - tensor dbg_dfl_softmax; - tensor dbg_box_view; - tensor debug_cls_logits = nullptr; -}; - -struct PreprocessResult { - tensor input_tensor; - float scale; - int pad_w; - int pad_h; - int orig_w; - int orig_h; -}; -struct NMSParams { - float conf_thres = 0.25f; - float iou_thres = 0.45f; - int max_det = 1000; - int max_nms = 30000; - int max_wh = 7680; -}; - - - -float resize_longest_side(i32x2 extent, int target_longest_side); - -LetterboxResult yolov9t_process_input(image_data image, yolov9t_params const& p); - -image_data yolov9t_process_input2(image_view image, yolov9t_params const& p); -void sync_detect_outputs(DetectOutput& outputs, backend_device const& backend); - -yolov9t_params yolov9t_detect_params(model_file const& file); - - -tensor Conv( - model_ref m, - tensor x, - std::string const& name, - int c1, - int c2, - int k = 1, - int s = 1, - int p = -1, - bool act = true, - bool debug = false, - bool bn = false); -tensor ELAN1( - model_ref m, - tensor x, - std::string const& name, - int c1, - int c2, - int c3, - int c4, - bool debug = false); -tensor AConv(model_ref m, tensor x, std::string const& name, int c1, int c2, bool debug = false); -tensor RepNCSPELAN4( - model_ref m, - tensor x, - std::string const& name, - int c1, - int c2, - int c3, - int c4, - int n = 1, - bool debug = false); -tensor SPPELAN( - model_ref m, - tensor x, - std::string const& name, - int c1, - int c2, - int c3, - int k = 5, - bool debug = false); -tensor RepBottleneck( - model_ref m, - tensor x, - std::string const& name, - int c1, - int c2, - bool shortcut = true, - int g = 1, - int k = 3, - float e = 0.5, - bool debug = false); -tensor RepCSP( - model_ref m, - tensor x, - std::string const& name, - int c1, - int c2, - int n, - bool shortcut = true, - int g = 1, - float e = 0.5, - bool debug = false); -tensor Upsample(model_ref m, tensor x, int scale_factor, bool debug = false); -tensor Concat(model_ref m, tensor a, tensor b, int axis = 2, bool debug = false); -tensor C3( - model_ref m, - tensor x, - std::string const& name, - int c1, - int c2, - int n = 1, - bool shortcut = true, - int g = 1, - float e = 0.5, - bool debug = false); -tensor RepConv( - model_ref m, - tensor x, - std::string const& name, - int c1, - int c2, - int k = 3, - int s = 1, - int p = 1, - int g = 1, - int d = 1, - bool act = true, - bool bn = false, - bool deploy = false, - bool debug = false); - -tensor Bottleneck( - model_ref m, - tensor x, - std::string const& name, - int c1, - int c2, - bool shortcut = true, - int g = 1, - float e = 0.5, - bool debug = false); - - -std::map yolov9t_backbone(model_ref m, tensor x); - - -tensor dfl_forward(model_ref m, tensor weight, tensor x, int reg_max, bool debug=false); - -std::pair make_anchors( - model_ref m, - DetectOutput const& out, - std::vector& anchor_host, - std::vector& stride_host, - std::vector const& strides, - float grid_cell_offset = 0.5f); -tensor dist2bbox(model_ref m, tensor dists, tensor anchors, bool xywh); - -DetectOutput detect_forward( - model_ref m, std::vector features, std::vector ch, int nc, bool training); - -DetectOutput yolov9t_forward(model_ref m, tensor x); - - -std::vector process_outputs( - detected_obj const& outputs, - yolov9t_params const& params, - i32x2 input_size, - float conf_threshold = 0.25f); - - -std::vector non_max_suppression( - DetectOutput const& outputs, - float conf_thres = 0.25f, - float iou_thres = 0.45f, - int max_det = 300, - int max_nms = 30000, - int max_wh = 7680); - -image_data hwc_to_chw_f32(image_data const& hwc); - - -std::vector const& get_coco_class_names(); - -int check_img_size(int imgsz, int s = 32, int floor = 0); - -LetterboxResult letterbox(image_data im, i32x2 new_shape, u8x3 color, - bool _auto, bool scaleFill, bool scaleup, int stride); - -image_data linear_image_resize(image_data im, i32x2 new_shape); -image_data image_add_border(image_data im, int top, int bottom, int left, int right, u8x3 color); - -void scale_boxes(std::vector& boxes, - i32x2 from_shape, - i32x2 to_shape, - float gain, - float pad_w, - float pad_h); - -std::vector nms( - std::vector const& detections, - float iou_threshold, - bool agnostic, - int max_wh = 7680); -void xywh2xyxy(float* boxes, int n); -void draw_detections( - image_data& img, - std::vector const& detections, - std::vector const& class_names); - - -void save_features_to_txt( - DetectOutput const& out, char const* base_path, std::vector const& keys = {}); - - -void save_input_to_txt(tensor input, char const* filepath); -} diff --git a/src/visp/arch_registry.cpp b/src/visp/arch_registry.cpp index 5fd7c78..c414439 100644 --- a/src/visp/arch_registry.cpp +++ b/src/visp/arch_registry.cpp @@ -20,13 +20,20 @@ arch_registrar::arch_registrar(std::string_view name, arch_forward_fn forward, a registry().push_back(arch_entry{name, forward, std::move(task)}); } +void register_runtime_graph(); +void register_mmlab_config(); + arch_entry const* arch_find(std::string_view name) { + register_runtime_graph(); + register_mmlab_config(); auto& r = registry(); auto it = std::find_if(r.begin(), r.end(), [&](arch_entry const& e) { return e.name == name; }); return it == r.end() ? nullptr : &*it; } std::span arch_all() { + register_runtime_graph(); + register_mmlab_config(); return std::span(registry()); } diff --git a/src/visp/arch_registry.h b/src/visp/arch_registry.h index 29e7fd8..76fe4bf 100644 --- a/src/visp/arch_registry.h +++ b/src/visp/arch_registry.h @@ -1,8 +1,8 @@ // arch_registry.h — g2c 가 생성한 arch 를 **이름으로** 찾아 실행한다. // // 왜 있나: 생성물을 CLI 에 붙일 때마다 enum·도움말·인자파싱·전방선언·switch·본체를 -// 손으로 고쳐야 했다. `inference_yolov9t.cpp` 가 그렇게 7곳을 고쳤는데 CMake 한 줄을 -// 빠뜨려 **빌드에서 통째로 빠진 채 죽은 코드**가 됐다(아무도 모르고 있었다). +// 손으로 고쳐야 했다. 한 번은 그렇게 7곳을 고쳤는데 CMake 한 줄을 빠뜨려 +// **빌드에서 통째로 빠진 채 죽은 코드**가 됐다(아무도 모르고 있었다). // 생성물은 시그니처가 균일하므로 등록으로 대신할 수 있다: // // tensor _forward(model_ref, tensor, _params const&); @@ -16,6 +16,7 @@ // static 으로 바꾸려면 `--whole-archive` 가 필요하다 → `DECISIONS.md`. #pragma once +#include "visp/mmdet_head.h" #include "visp/ml.h" #include @@ -32,30 +33,39 @@ using arch_forward_fn = tensor (*)(model_ref, tensor, model_file const&); // 그래프 뒤에 무엇을 해야 하는지. **gguf 에 이 정보가 없다** — g2c 가 안 쓴다. // g2c 를 고치지 않기로 했으므로 `tools/install_arch.py` 가 모델을 보고 등록 TU 에 박는다. -enum class arch_kind { raw, detect_yolo }; +enum class arch_kind { raw, detect_mmdet }; struct arch_task { arch_kind kind = arch_kind::raw; - // detect_yolo 전용. 그래프가 내는 것은 **격자 단위 ltrb + 클래스 로짓**이고, - // 격자점 곱·시그모이드·top-k 는 호스트가 한다(g2c 가 그 앞에서 끊는다). int num_classes = 80; - std::vector strides{8, 16, 32}; - bool nms_free = true; // YOLO26/v10: one2one 이라 NMS 없이 top-k - float score_thr = 0.25f; - float nms_thr = 0.7f; // nms_free 면 안 쓴다 - int max_det = 300; - int input_size = 640; + int input_size = 640; // 정사각 모델의 한 변 + // ⚠️ **비정사각 모델은 여기에 실어야 한다.** 0 이면 `input_size` 를 정사각으로 쓴다. + // 안 실으면 러너가 정사각으로 리사이즈해 **조용히 틀린 출력**을 낸다 — + // mmseg cityscapes(512×1024)가 512×512 로 줄어 출력이 절반만 나왔다(2026-09-01). + // mmpose 는 전부 256×192 라 한 계열도 못 돌았다. + int input_w = 0; + int input_h = 0; - // 전처리 정규화 `(v - mean) / std`. 기본값은 0~1 (YOLO 규약). + // 전처리 정규화 `(v - mean) / std`. 기본값은 0~1. // ⚠️ **모델마다 다르다.** torchvision 분류기는 ImageNet 통계를 쓰는데, 0~1 로 돌리면 // 크래시 없이 값만 틀린다(ResNet-18 실측 상대 L1 5.1e-02, argmax 는 우연히 일치). std::array mean{0.0f, 0.0f, 0.0f}; std::array stdv{255.0f, 255.0f, 255.0f}; - // 그래프 출력 중 몇 번이 박스/점수인지. 생성 모델마다 다르다(one2many 분기가 앞에 올 수 있다). - int box_out = -1; // -1 = 자동(뒤에서 두 번째 4채널 출력) - int score_out = -1; + // 채널 순서를 뒤집는다. mmdet 설정의 `to_rgb` 가 그대로 온다 — 안 뒤집으면 크래시 없이 + // 값만 틀린다. + bool to_rgb = false; + // MMDetection 시험 파이프라인의 `Resize(keep_ratio=True)` + 오른쪽·아래 패딩. + // 끄면 가로·세로를 따로 늘린다 — 분할·자세는 그게 맞지만, 검출기에 쓰면 비정사각 + // 이미지에서 점수가 벌어지고 중복 박스가 나온다(YOLO 실측 2026-09-02, bowl 0.70→0.93). + bool keep_ratio = false; + std::array pad_value{0.0f, 0.0f, 0.0f}; // 패딩 픽셀 값(0~255, 정규화 전) std::vector class_names; // 비면 "class N" + + // detect_mmdet 전용. head 조립과 디코드에 필요한 값 전부가 여기 들어간다. + // ⚠️ 옛 경로는 이 값을 `.postproc.h` 로 러너에 컴파일해 넣었고, 모델마다 실행 + // 파일이 갈렸다. 등록 단위로 옮겨 vision-cli 하나가 여러 검출기를 돌린다. + mmdet_cfg mmdet; }; struct arch_entry { diff --git a/src/visp/ml.cpp b/src/visp/ml.cpp index 4c0906e..38073e7 100644 --- a/src/visp/ml.cpp +++ b/src/visp/ml.cpp @@ -8,6 +8,10 @@ #include #include +#ifdef GGML_USE_GTX +# include "ggml-gtx.h" +#endif + namespace visp { // @@ -24,6 +28,10 @@ std::string_view to_string(backend_type type) { bool load_ggml_backends() { static const bool loaded = []() { +#ifdef GGML_USE_GTX + // depend/ggml-gtx 는 ggml 밖에서 빌드되므로 ggml 의 정적 등록 목록에 없다. + ggml_backend_register(ggml_backend_gtx_reg()); +#endif if (ggml_backend_reg_count() > 0) { return true; // already loaded } diff --git a/tools/detect/head.cpp b/src/visp/mmdet_head.cpp similarity index 99% rename from tools/detect/head.cpp rename to src/visp/mmdet_head.cpp index 5e49c4a..cc847cf 100755 --- a/tools/detect/head.cpp +++ b/src/visp/mmdet_head.cpp @@ -1,4 +1,4 @@ -#include "head.h" +#include "visp/mmdet_head.h" #include diff --git a/tools/detect/head.h b/src/visp/mmdet_head.h similarity index 93% rename from tools/detect/head.h rename to src/visp/mmdet_head.h index 6c6b417..e9fbb7b 100755 --- a/tools/detect/head.h +++ b/src/visp/mmdet_head.h @@ -1,7 +1,7 @@ -// mmdet 검출 head 를 **손코딩 C++ 부품**으로 조립한다. g2c 는 backbone/neck 만 그래프 -// 데이터(gguf graph.nodes)로 내보내고, head(공유 conv 타워 + 최종 cls/reg conv)는 여기서 -// 조립한다 — 교수님 지시("backbone 만 g2c, head 는 vision.cpp 부품"). decode+NMS 는 -// postproc.cpp(detect_anchor)가 담당하므로, 이 부품은 raw cls_score/bbox_pred 까지만 낸다. +// mmdet 검출 head 를 손코딩 C++ 로 조립한다. g2c 는 backbone/neck 만 그래프 데이터(gguf +// graph.nodes)로 내보내고, head(공유 conv 타워 + 최종 cls/reg conv)는 여기서 조립한다. +// 디코드와 NMS 는 postproc.cpp 가 맡으므로 이 파일은 raw cls_score/bbox_pred 까지만 낸다. +// 전체 실행 순서는 mmdet_run.cpp 가 엮는다. #pragma once #include "visp/ml.h" @@ -12,7 +12,7 @@ namespace visp { -// 어느 조립기를 쓸지. 계열이 늘어도 러너는 안 바뀐다 — mmdet_head_forward 가 갈라준다. +// 어느 조립기를 쓸지. 계열이 늘어도 부르는 쪽은 안 바뀐다 — mmdet_head_forward 가 갈라준다. enum class head_kind { anchor, // RetinaNet · ATSS · PAA — cls/reg(+centerness) 타워, Delta 디코드 // 타워 조립은 anchor 와 같다(레벨별 타워 + obj 갈래). **디코드가 다르다** — 코더가 @@ -43,7 +43,8 @@ enum class head_kind { ddq, // + 층마다 NMS 로 중복 query 를 가린다 → **한 그래프로 못 돈다** }; -// head-conv 구조. 값은 mmdet_to_pt.py 가 .postproc.h 에 굽는다(런타임 파일 없음). +// head-conv 구조. 값은 mmdet_to_pt.py 가 .postproc.h 에 C++ 상수로 굽고, +// install_arch.py 가 그 헤더를 등록 단위에 실어 `arch_task::mmdet` 로 넣는다. // // 계열별로 함수를 따로 쓰지 않고 **플래그로 가른다** — RetinaNet/ATSS/PAA/FCOS/GFL 은 // "타워 2개 + 출력 conv 몇 개"라는 뼈대가 같고 곁가지만 다르기 때문이다. @@ -87,7 +88,7 @@ struct anchor_head_cfg { bool corner_emb = false; // CentripetalNet: emb 대신 guiding/centripetal shift 갈래 + DCNv1 feature adaption. bool corner_centripetal = false; - // 코너 디코드 파라미터(test_cfg). 조립기는 안 쓰고 러너가 디코더에 넘긴다. + // 코너 디코드 파라미터(test_cfg). 조립기는 안 쓰고 디코더로 넘어간다. int corner_topk = 100; int local_max_kernel = 3; float corner_distance_thr = 0.5f; @@ -134,10 +135,8 @@ struct head_outputs { std::vector>> extra; }; -// Everything the runner needs to run one detector. -// Once an architecture is fixed these are constants, so mmdet_to_pt.py emits them as -// mmdet_params() in .postproc.h and they are compiled into the runner. Nothing is read -// at run time. +// 검출기 한 벌을 돌리는 데 필요한 값 전부. 아키텍처가 정해지면 상수이므로 mmdet_to_pt.py 가 +// .postproc.h 에 굽고 실행 파일에 컴파일해 넣는다. 실행 시점에 읽는 파일은 없다. struct mmdet_cfg { anchor_head_cfg head; det_params det; @@ -155,8 +154,8 @@ void anchor_head_forward(model_ref m, std::vector const& feats, std::vector& cls_out, std::vector& box_out); // VFNet head(distance + star-shaped deformable conv). anchor head 와 달리 offset 을 -// bbox_pred 로부터 손코딩으로 만들어 conv_2d_deform 에 넣는다("head 는 ggml 로 안 바뀐다" -// → offset 생성이 자동변환 불가한 부분). 값은 .vfhead.json 에서 읽는다. +// bbox_pred 로부터 손코딩으로 만들어 conv_2d_deform 에 넣는다 — offset 생성이 자동 변환으로 +// 넘어가지 않는 부분이다. struct vfnet_head_cfg { int stacked_convs = 3; int feat_channels = 256; @@ -168,7 +167,7 @@ struct vfnet_head_cfg { }; // FPN features(레벨별, cwhn) → 레벨별 raw cls_score / bbox_pred_refine(cwhn, inference 모드). -// · dcn_base : {18,1,1,1} cwhn — dcn_base_offset(고정 3x3 그리드), 런타임이 값을 채워 넘김. +// · dcn_base : {18,1,1,1} cwhn — dcn_base_offset(고정 3x3 그리드). 부르는 쪽이 값을 채운다. // · cls_out[l] : ne={num_classes, w, h, 1}, box_out[l] : ne={4, w, h, 1} // 가중치는 모든 레벨 공유. offset = star_dcn_offset(bbox_pred) - dcn_base 를 그래프로 조립. void vfnet_head_forward(model_ref m, std::vector const& feats, @@ -176,7 +175,7 @@ void vfnet_head_forward(model_ref m, std::vector const& feats, std::vector& cls_out, std::vector& box_out); // RepPoints. 점 9개를 예측하고 그 집합을 bbox 로 바꾼다(transform_method='moment'). -// · dcn_base : {18,1,1,1} — dcn_base_offset(고정 3x3 그리드). 런타임이 값을 채운다. +// · dcn_base : {18,1,1,1} — dcn_base_offset(고정 3x3 그리드). 부르는 쪽이 값을 채운다. // 두 단계다: init 로 점을 놓고, 그 점을 offset 삼아 deform conv 로 refine 한다. void reppoints_head_forward(model_ref m, std::vector const& feats, anchor_head_cfg const& c, tensor dcn_base, @@ -210,7 +209,7 @@ void dino_head_forward(model_ref m, std::vector const& feats, // **이 계열만 한 그래프로 못 돈다.** decoder 층마다 `batched_nms` 로 중복 query 를 골라내 // 다음 층의 self-attention 마스크를 만드는데(추론 경로에서도), NMS 는 반복 횟수와 분기가 // 실행 중 값에 따라 정해지는 탐욕 알고리즘이라 정적 그래프로 표현할 수 없다. -// → 조립기를 **토막**으로 나누고 러너가 패스 사이에서 호스트 NMS 를 돌린다. +// → 조립기를 **토막**으로 나누고 mmdet_run 이 패스 사이에서 호스트 NMS 를 돌린다. // 패스 0 : ddq_encode_forward → 호스트 NMS → query 900개 // 패스 i : ddq_layer_forward → 호스트 NMS → 다음 층 마스크 struct ddq_stage { diff --git a/src/visp/mmdet_run.cpp b/src/visp/mmdet_run.cpp new file mode 100644 index 0000000..ccf3ac6 --- /dev/null +++ b/src/visp/mmdet_run.cpp @@ -0,0 +1,503 @@ +#include "visp/mmdet_run.h" + +#include "util/string.h" +#include "visp/ml.h" + +#include + +#include +#include +#include +#include +#include + +namespace visp { +namespace { + +std::vector to_vec(tensor t) { + int64_t ne = ggml_nelements(t); + std::vector d((size_t)ne); + transfer_from_backend(t, std::span(d.data(), d.size())); + return d; +} + +void dump_bin(std::string const& path, std::vector const& v) { + if (FILE* f = fopen(path.c_str(), "wb")) { + fwrite(v.data(), sizeof(float), v.size(), f); + fclose(f); + } +} + +float sigm(float v) { return 1.0f / (1.0f + std::exp(-v)); } + +// DDQ 만 그래프가 여러 벌이다. decoder 층마다 호스트 NMS 로 중복 query 를 가려 다음 층의 +// self-attention 마스크를 만드는데, NMS 는 실행 중 값에 따라 반복이 정해지는 탐욕 +// 알고리즘이라 정적 그래프로 표현할 수 없다(mmdet_head.h 의 DDQ 절). +std::vector run_ddq(backend_device& backend, model_weights& weights, + compute_graph& graph, model_ref m, tensor input, + std::vector const& feats, mmdet_cfg const& cfg, + std::span in_data) { + anchor_head_cfg const& hc = cfg.head; + det_params const& dp = cfg.det; + + std::vector lv; + ddq_levels(feats, lv); + ddq_stage s0; + ddq_encode_forward(m, feats, hc, s0); + for (tensor x : {s0.mem, s0.om, s0.ecls, s0.ecoord, s0.qmap}) { + ggml_build_forward_expand(graph, x); + } + compute_graph_allocate(graph, backend); + transfer_to_backend(input, in_data); + compute(graph, backend); + + const int64_t T = s0.mem->ne[1], E = s0.mem->ne[0], NC = s0.ecls->ne[0]; + const int64_t NQ = hc.num_queries; + std::vector mem_v = to_vec(s0.mem), qm_v = to_vec(s0.qmap); + std::vector ec_v = to_vec(s0.ecls), eb_v = to_vec(s0.ecoord); + + // 호스트: 점수 = sigmoid(클래스 최댓값), 박스 = cxcywh→xyxy(sigmoid(coord)). + std::vector dets((size_t)T); + for (int64_t i = 0; i < T; ++i) { + float best = -1e30f; + for (int64_t k = 0; k < NC; ++k) best = std::max(best, ec_v[i * NC + k]); + float cx = sigm(eb_v[i * 4 + 0]), cy = sigm(eb_v[i * 4 + 1]); + float w = sigm(eb_v[i * 4 + 2]), h = sigm(eb_v[i * 4 + 3]); + dets[i] = {cx - w * 0.5f, cy - h * 0.5f, cx + w * 0.5f, cy + h * 0.5f, sigm(best), 0}; + } + std::vector keep = nms(dets, hc.ddq_iou_thr); + if ((int64_t)keep.size() > NQ) keep.resize((size_t)NQ); + if ((int64_t)keep.size() < NQ) { + throw except("ddq: NMS 후 {} 개 < num_queries {} — 그래프 모양이 안 맞는다", + keep.size(), (long long)NQ); + } + // DDQ 는 query 내용을 학습 embedding 이 아니라 **선택된 위치의 feature** 로 채운다. + std::vector q_v((size_t)E * NQ), r_v((size_t)4 * NQ); + for (int64_t i = 0; i < NQ; ++i) { + const int64_t t0 = keep[(size_t)i]; + std::copy_n(&qm_v[(size_t)t0 * E], E, &q_v[(size_t)i * E]); + for (int k = 0; k < 4; ++k) r_v[(size_t)i * 4 + k] = sigm(eb_v[(size_t)t0 * 4 + k]); + } + std::vector mask_v((size_t)NQ * NQ, 0.0f); // 가산 마스크(0 = 통과) + std::vector alive((size_t)NQ, 1); // 0층은 전부 살아있다 + + std::vector> cls_out, box_out; + for (int li = 0; li < hc.dec_layers; ++li) { + compute_graph g = compute_graph_init(65536); + model_ref mi(weights, g); + tensor memT = compute_graph_input(mi, GGML_TYPE_F32, {E, T, 1, 1}, "mem"); + tensor qT = compute_graph_input(mi, GGML_TYPE_F32, {E, NQ, 1, 1}, "q"); + tensor rT = compute_graph_input(mi, GGML_TYPE_F32, {4, NQ, 1, 1}, "ref"); + tensor kT = compute_graph_input(mi, GGML_TYPE_F32, {NQ, NQ, 1, 1}, "mask"); + for (tensor x : {memT, qT, rT, kT}) ggml_build_forward_expand(g, x); + ddq_stage si; + ddq_layer_forward(mi, lv, hc, li, memT, qT, rT, kT, si); + for (tensor x : {si.query, si.ref, si.cls, si.box}) ggml_build_forward_expand(g, x); + compute_graph_allocate(g, backend); + transfer_to_backend(memT, std::span(mem_v.data(), mem_v.size())); + transfer_to_backend(qT, std::span(q_v.data(), q_v.size())); + transfer_to_backend(rT, std::span(r_v.data(), r_v.size())); + transfer_to_backend(kT, std::span(mask_v.data(), mask_v.size())); + compute(g, backend); + + cls_out.push_back(to_vec(si.cls)); + box_out.push_back(to_vec(si.box)); + q_v = to_vec(si.query); + r_v = to_vec(si.ref); + if (li + 1 < hc.dec_layers) { + // 다음 층 마스크. mmdet 규약 두 가지를 그대로 따른다: + // ① NMS 는 **지금까지 살아남은 query 끼리만** 돈다(집합이 단조 감소한다). + // ② 셀 (i,j) 는 **행이나 열 중 하나라도** 살아있으면 통과, 아니면 가린다. + std::vector ori; + for (int64_t i = 0; i < NQ; ++i) if (alive[(size_t)i]) ori.push_back((int)i); + std::vector dq(ori.size()); + for (size_t n = 0; n < ori.size(); ++n) { + const int64_t i = ori[n]; + float best = -1e30f; + for (int64_t k = 0; k < NC; ++k) + best = std::max(best, cls_out.back()[(size_t)i * NC + k]); + float cx = r_v[(size_t)i * 4 + 0], cy = r_v[(size_t)i * 4 + 1]; + float w = r_v[(size_t)i * 4 + 2], h = r_v[(size_t)i * 4 + 3]; + dq[n] = {cx - w * 0.5f, cy - h * 0.5f, cx + w * 0.5f, cy + h * 0.5f, + sigm(best), 0}; + } + std::vector next((size_t)NQ, 0); + for (int k : nms(dq, hc.ddq_iou_thr)) next[(size_t)ori[(size_t)k]] = 1; + alive.swap(next); + const float NEG = -1e30f; + for (int64_t i = 0; i < NQ; ++i) + for (int64_t j = 0; j < NQ; ++j) + mask_v[(size_t)i * NQ + j] = + (alive[(size_t)i] || alive[(size_t)j]) ? 0.0f : NEG; + } + } + if (const char* pre = std::getenv("MMDET_DUMP_HEAD")) { + for (size_t l = 0; l < cls_out.size(); ++l) + dump_bin(std::string(pre) + ".cls." + std::to_string(l) + ".bin", cls_out[l]); + for (size_t l = 0; l < box_out.size(); ++l) + dump_bin(std::string(pre) + ".box." + std::to_string(l) + ".bin", box_out[l]); + } + + // 최종 디코드 — mmdet `DDQDETRHead.predict_by_feat`. 마지막 층 출력에서 살아남은 + // query 만 골라 DETR top-k 를 탄다. 층 사이 NMS 가 이미 중복을 걸렀으므로 최종 NMS 는 + // 없다. + // ⚠️ 박스는 마지막 층 hidden 에서 낸 sigmoid cxcywh 다. `s.ref` 는 **다음 층 참조점**이라 + // 미묘하게 다르고, 섞으면 값만 조용히 틀린다. + std::vector fcls, fbox; + std::vector const& lc = cls_out.back(); + std::vector const& lb = box_out.back(); + int n_alive = 0; + for (int64_t i = 0; i < NQ; ++i) { + if (!alive[(size_t)i]) continue; + fcls.insert(fcls.end(), lc.begin() + i * NC, lc.begin() + (i + 1) * NC); + fbox.insert(fbox.end(), lb.begin() + i * 4, lb.begin() + (i + 1) * 4); + ++n_alive; + } + detr_params qp; + qp.num_queries = n_alive; + qp.num_classes = dp.num_classes; // 배경 제외 수 (프론트엔드 규약) + qp.use_sigmoid = true; // deformable 계열 — 항상 sigmoid+focal + qp.max_per_img = dp.max_per_img; + qp.input_w = dp.input_w; + qp.input_h = dp.input_h; + return detect_detr(fcls.data(), fbox.data(), qp); +} + +} // namespace + +std::vector mmdet_run(backend_device& backend, model_weights& weights, + model_file const& file, mmdet_backbone_fn backbone, + mmdet_cfg cfg, std::span in_data, + int input_w, int input_h) { + anchor_head_cfg& hc = cfg.head; + det_params& dp = cfg.det; + dp.input_w = input_w; + dp.input_h = input_h; + const int L = (int)dp.strides.size(); + if (L == 0) { + throw except("이 설정의 head 는 내보내기에서 인식되지 않았다 — 백본만 있다"); + } + if (in_data.size() != (size_t)3 * input_w * input_h) { + throw except("입력 크기 {} != {} (전처리·해상도 확인)", in_data.size(), + (size_t)3 * input_w * input_h); + } + + compute_graph graph = compute_graph_init(131072); + model_ref m(weights, graph); + tensor input = compute_graph_input(m, GGML_TYPE_F32, {3, input_w, input_h, 1}, "x"); + ggml_build_forward_expand(graph, input); + ggml_build_forward_expand(graph, backbone(m, input, file)); + + // 백본이 낸 FPN feature 를 이름으로 잡는다. + std::vector feats; + for (int l = 0; l < L; ++l) { + tensor f = ggml_graph_get_tensor(graph.graph, ("out_" + std::to_string(l)).c_str()); + if (!f) throw except("백본 출력 out_{} 이 없다 (생성물의 출력 규약 확인)", l); + feats.push_back(f); + } + // deformable 을 쓰는 계열(vfnet·reppoints)은 고정 3×3 격자가 필요하다. mmdet 의 + // dcn_base_offset 과 같은 값 — [y,x] 가 번갈아 놓인 18개다. + // ⚠️ 안 쓰는 계열에서 만들면 그래프에 안 실려 버퍼가 없고 add_buffer 가 죽는다. + tensor dcn_base = nullptr; + if (hc.kind == head_kind::vfnet || hc.kind == head_kind::reppoints) { + dcn_base = compute_graph_input(m, GGML_TYPE_F32, {18, 1, 1, 1}, "dcn_base"); + tensor_data d = tensor_alloc(dcn_base); + std::span v = d.as_f32(); + for (int i = 0; i < 9; ++i) { + v[2 * i] = float(i / 3 - 1); // y: -1,-1,-1, 0,0,0, 1,1,1 + v[2 * i + 1] = float(i % 3 - 1); // x: -1,0,1 반복 + } + m.add_buffer(std::move(d)); + } + + if (hc.kind == head_kind::ddq) { + return run_ddq(backend, weights, graph, m, input, feats, cfg, in_data); + } + + std::vector dets; + head_outputs ho; + mmdet_head_forward(m, feats, hc, dcn_base, ho); + std::vector&cls_t = ho.cls, &box_t = ho.box; + for (tensor t : cls_t) ggml_build_forward_expand(graph, t); + for (tensor t : box_t) ggml_build_forward_expand(graph, t); + for (tensor t : ho.ctr) ggml_build_forward_expand(graph, t); + // ⚠️ extra 갈래도 **그래프에 넣어야** 버퍼가 생긴다. 안 넣고 읽으면 + // `GGML_ASSERT(buf != NULL && "tensor buffer not set")` 로 죽는다. + for (auto const& e : ho.extra) + for (tensor t : e.second) ggml_build_forward_expand(graph, t); + + compute_graph_allocate(graph, backend); + transfer_to_backend(input, in_data); + compute(graph, backend); + + // 검증용 원시 덤프. **디코드 전** 값을 그대로 내보내 torch 의 bbox_head 출력과 직접 + // 대조한다 — NMS 를 거치면 어디가 틀렸는지 짚을 수 없다. + if (const char* pre = std::getenv("MMDET_DUMP_HEAD")) { + auto dump = [&](const char* kind, size_t l, tensor t) { + dump_bin(std::string(pre) + "." + kind + "." + std::to_string(l) + ".bin", to_vec(t)); + }; + // ⚠️ **head 입력(neck 출력)도 같이 낸다.** head 출력만 보면 「조립이 틀렸다」와 + // 「백본·neck 이 이미 틀렸다」를 가를 수 없다. + for (size_t l = 0; l < feats.size(); ++l) dump("feat", l, feats[l]); + for (size_t l = 0; l < cls_t.size(); ++l) dump("cls", l, cls_t[l]); + for (size_t l = 0; l < box_t.size(); ++l) dump("box", l, box_t[l]); + for (size_t l = 0; l < ho.ctr.size(); ++l) dump("ctr", l, ho.ctr[l]); + for (auto const& e : ho.extra) + for (size_t l = 0; l < e.second.size(); ++l) dump(e.first.c_str(), l, e.second[l]); + } + + // ── CornerNet / CentripetalNet ───────────────────────────────────────── + // 앵커가 없다 — 코너 heatmap 두 장에서 top-k 를 뽑아 쌍을 만든다. 갈래가 6~8개라 + // out.extra 를 쓰고, **마지막 레벨만** 쓴다(mmdet 도 `tl_heats[-1]` 이다). + bool corner_done = false; + if (hc.kind == head_kind::cornernet) { + if (ho.cls.empty() || ho.box.empty() || ho.ctr.empty() || ho.extra.empty()) { + throw except("corner head 출력이 비었다"); + } + auto last = [](std::vector const& v) { return v.back(); }; + auto named = [&](const char* tag) -> std::vector { + for (auto const& e : ho.extra) + if (e.first == tag && !e.second.empty()) return to_vec(e.second.back()); + return {}; + }; + tensor t_cls = last(ho.cls); + const int cfh = (int)t_cls->ne[2], cfw = (int)t_cls->ne[1]; + corner_params cp; + cp.num_classes = dp.num_classes; + cp.topk = hc.corner_topk; + cp.local_max_kernel = hc.local_max_kernel; + cp.distance_threshold = hc.corner_distance_thr; + cp.score_thr = dp.score_thr; + cp.nms_thr = dp.nms_thr; + cp.max_per_img = dp.max_per_img; + cp.input_w = dp.input_w; + cp.input_h = dp.input_h; + cp.centripetal = hc.corner_centripetal; + dets = detect_corner(to_vec(t_cls), to_vec(last(ho.box)), + to_vec(last(ho.ctr)), named("brof"), + named("tlemb"), named("bremb"), + named("tlcs"), named("brcs"), + cfh, cfw, cp); + corner_done = true; + } + + // ── CenterNet ────────────────────────────────────────────────────────── + // 앵커가 없고 레벨도 하나다. heatmap 의 국소 최대점이 중심이고, 같은 자리의 wh/offset 이 + // 상자를 만든다. heat 는 head 안에서 이미 sigmoid 를 거쳤다. + if (hc.kind == head_kind::centernet) { + if (ho.cls.empty() || ho.box.empty() || ho.ctr.empty()) { + throw except("centernet head 출력이 비었다(heatmap·wh·offset 필요)"); + } + tensor t = ho.cls.back(); + centernet_params cp; + cp.num_classes = dp.num_classes; + cp.topk = hc.corner_topk; // test_cfg.topk (헤더가 같은 칸을 쓴다) + cp.local_max_kernel = hc.local_max_kernel; + cp.input_w = dp.input_w; + cp.input_h = dp.input_h; + dets = detect_centernet(to_vec(t), to_vec(ho.box.back()), to_vec(ho.ctr.back()), + (int)t->ne[2], (int)t->ne[1], cp); + corner_done = true; + } + + // DETR 계열은 출력 인덱스가 **decoder 층**이라 레벨 수와 무관하다. + const bool is_detr = hc.kind == head_kind::detr || + hc.kind == head_kind::conditional_detr || + hc.kind == head_kind::dab_detr || + hc.kind == head_kind::deformable_detr || + hc.kind == head_kind::dino; + + // YOLOv3 는 레벨당 **한 갈래**다(na×(5+nc) 한 텐서에 box·obj·cls 가 다 들어 있다). + const bool single_branch = hc.kind == head_kind::yolo; + + if (!corner_done && !is_detr && !single_branch && + ((int)cls_t.size() < L || (int)box_t.size() < L)) { + throw except("head 출력이 {} 레벨에 못 미친다 (cls {}, box {})", + L, cls_t.size(), box_t.size()); + } + + const int NL = (is_detr || corner_done) ? 0 : L; + std::vector> cls_v(NL), box_v(NL), ctr_v; + std::vector> feat_hw(NL); + for (int l = 0; l < NL; ++l) { + feat_hw[l] = { (int)cls_t[l]->ne[2], (int)cls_t[l]->ne[1] }; // (fh, fw) + cls_v[l] = to_vec(cls_t[l]); + if (!single_branch) box_v[l] = to_vec(box_t[l]); + } + // centerness 갈래가 있으면 score factor 로 넘긴다(ATSS·PAA·DDOD·FCOS…). mmdet 은 이걸 + // top-k 뒤에 곱한다 — 안 넘기면 **박스는 맞고 점수만** 높게 나온다(실측 Δ0.071). + // ⚠️ YOLACT 의 coeff 갈래는 점수가 아니다 — `ctr_tanh` 로 가른다. + if (!corner_done && (int)ho.ctr.size() >= L && !hc.ctr_tanh) { + ctr_v.resize(L); + for (int l = 0; l < L; ++l) ctr_v[l] = to_vec(ho.ctr[l]); + } + + // 디코드는 **조립과 다른 축**이다. 조립은 타워 모양(head_kind)으로 갈리지만, 디코드는 + // 박스가 무엇이냐로 갈린다 — 앵커 대비 delta 냐, 격자점 대비 거리냐. + const bool distance_box = hc.kind == head_kind::fcos || hc.kind == head_kind::gfl || + hc.kind == head_kind::vfnet || hc.kind == head_kind::reppoints || + hc.bbox_mul_stride; + + if (corner_done) { + // 위에서 채웠다. + } else if (is_detr) { + // ⚠️ out.cls/out.box 의 인덱스가 **FPN 레벨이 아니라 decoder 층**이다. mmdet 도 + // `all_layers_*[-1]` 만 쓴다 — 앞 층을 쓰면 shape 가 맞아 안 죽고 더 거친 박스가 + // 나온다. + if (ho.cls.empty() || ho.box.empty()) throw except("DETR head 출력이 비었다"); + std::vector cls_last = to_vec(ho.cls.back()); + std::vector box_last = to_vec(ho.box.back()); + detr_params qp; + qp.num_queries = hc.num_queries; + qp.num_classes = dp.num_classes; // 배경을 뺀 수 (프론트엔드가 그렇게 싣는다) + qp.use_sigmoid = dp.use_sigmoid; // 고전 DETR=softmax, Deformable/DINO=sigmoid + qp.max_per_img = dp.max_per_img; + qp.input_w = dp.input_w; + qp.input_h = dp.input_h; + const int q_actual = (int)ho.box.back()->ne[1]; + if (q_actual > 0 && q_actual != qp.num_queries) { + qp.num_queries = q_actual; // 설정보다 텐서를 따른다 + } + dets = detect_detr(cls_last.data(), box_last.data(), qp); + } else if (hc.kind == head_kind::tood) { + // 조립기가 이미 `distance2bbox` 까지 그래프로 펴서 **격자 좌표계 xyxy** 를 낸다 — + // stride 만 곱하면 픽셀이다. + // ⚠️ cls 는 `sqrt(σ(logits)·σ(align))` 라 **이미 확률**이다. 시그모이드를 또 걸면 + // 박스는 그대로인데 점수만 낮아진다. + std::vector cand; + for (int l = 0; l < L; ++l) { + const int fh = feat_hw[l].first, fw = feat_hw[l].second, npos = fh * fw; + const float stride = dp.strides[l]; + float const* cs = cls_v[l].data(); + float const* bx = box_v[l].data(); + for (int pos = 0; pos < npos; ++pos) { + float const* c1 = cs + (size_t)pos * dp.num_classes; + int best = 0; + for (int j = 1; j < dp.num_classes; ++j) + if (c1[j] > c1[best]) best = j; + if (c1[best] <= dp.score_thr) continue; + float const* b1 = bx + (size_t)pos * 4; + detection d{b1[0] * stride, b1[1] * stride, b1[2] * stride, b1[3] * stride, + c1[best], best}; + if (dp.input_w > 0) { + d.x1 = std::min(std::max(d.x1, 0.0f), (float)dp.input_w); + d.x2 = std::min(std::max(d.x2, 0.0f), (float)dp.input_w); + d.y1 = std::min(std::max(d.y1, 0.0f), (float)dp.input_h); + d.y2 = std::min(std::max(d.y2, 0.0f), (float)dp.input_h); + } + cand.push_back(d); + } + } + int maxlabel = 0; + for (auto const& c : cand) maxlabel = std::max(maxlabel, c.label); + for (int lab = 0; lab <= maxlabel; ++lab) { + std::vector per; + for (auto const& c : cand) if (c.label == lab) per.push_back(c); + if (per.empty()) continue; + for (int k : nms(per, dp.nms_thr)) dets.push_back(per[k]); + } + std::sort(dets.begin(), dets.end(), + [](detection const& a, detection const& b) { return a.score > b.score; }); + if ((int)dets.size() > dp.max_per_img) dets.resize(dp.max_per_img); + } else if (hc.kind == head_kind::rpn) { + // 제안 생성. mmdet 은 클래스별이 아니라 **레벨별**로 NMS 를 걸고(level_ids), pre-NMS + // 점수 임계값이 없다 — `detect_anchor` 로 두면 둘 다 어긋난다. + rpn_params rp; + rp.strides = dp.strides; + rp.octave_base_scale = dp.octave_base_scale; + rp.octave_scales = dp.octave_scales; + rp.ratios = dp.ratios; + for (int k = 0; k < 4; ++k) { rp.means[k] = dp.means[k]; rp.stds[k] = dp.stds[k]; } + rp.nms_pre = dp.nms_pre; + rp.nms_thr = dp.nms_thr; + rp.max_per_img = dp.max_per_img; + rp.input_w = dp.input_w; + rp.input_h = dp.input_h; + dets = detect_rpn(cls_v, box_v, feat_hw, rp); + // `label` 은 레벨 번호로 돌아온다. 밖에서는 클래스 자리라 0 으로 둔다. + for (auto& d : dets) d.label = 0; + } else if (hc.kind == head_kind::sabl) { + // 변마다 버킷 분류 + 오프셋. 두 번째 box 갈래는 `extra[0]`("bcls") 로 온다. + sabl_params sp; + sp.strides = dp.strides; + sp.anchor_scale = dp.anchor_scale; + sp.num_buckets = dp.num_buckets; + sp.bucket_scale = dp.bucket_scale; + sp.num_classes = dp.num_classes; + sp.score_thr = dp.score_thr; + sp.nms_thr = dp.nms_thr; + sp.nms_pre = dp.nms_pre; + sp.max_per_img = dp.max_per_img; + sp.input_w = dp.input_w; + sp.input_h = dp.input_h; + std::vector> bcls_v(L); + if (!ho.extra.empty() && (int)ho.extra[0].second.size() >= L) + for (int l = 0; l < L; ++l) bcls_v[l] = to_vec(ho.extra[0].second[l]); + dets = detect_sabl(cls_v, bcls_v, box_v, feat_hw, sp); + } else if (hc.kind == head_kind::yolo) { + // YOLOv3: 레벨당 **한 갈래**(na×(5+nc))라 조립기가 `out.cls` 에만 담는다. 앵커가 + // (w,h) 쌍이고 objectness 로 먼저 거른다 — 전용 디코더로 보낸다. + yolov3_params yp; + yp.strides = dp.strides; + yp.base_sizes = dp.base_sizes; + yp.num_classes = dp.num_classes; + yp.conf_thr = dp.conf_thr; + yp.score_thr = dp.score_thr; + yp.nms_thr = dp.nms_thr; + yp.nms_pre = dp.nms_pre; + yp.max_per_img = dp.max_per_img; + yp.input_w = dp.input_w; + yp.input_h = dp.input_h; + dets = detect_yolov3(cls_v, feat_hw, yp); + } else if (hc.kind == head_kind::yolox) { + // 격자 단위 (dx,dy,log w,log h) + 별도 objectness. 코더가 없어 앵커 파라미터가 안 + // 실리고, 그대로 두면 detect_anchor 가 후보 0개를 낸다. + // ⚠️ obj 는 `out.ctr` 로 온다. 비면 점수가 cls 만 남아 **박스는 맞고 점수만** 높다. + yolox_params yp; + yp.strides = dp.strides; + yp.prior_offset = hc.center_offset; // YOLOX 의 MlvlPointGenerator 는 offset=0 + yp.num_classes = dp.num_classes; + yp.score_thr = dp.score_thr; + yp.nms_thr = dp.nms_thr; + yp.max_per_img = dp.max_per_img; + yp.input_w = dp.input_w; + yp.input_h = dp.input_h; + if ((int)ctr_v.size() < L) { + throw except("YOLOX 인데 objectness 갈래가 {} 레벨뿐이다 (필요 {})", + ctr_v.size(), L); + } + dets = detect_yolox(cls_v, box_v, ctr_v, feat_hw, yp); + } else if (distance_box) { + fcos_params fp; + fp.strides = dp.strides; + fp.num_classes = dp.num_classes; + fp.score_thr = dp.score_thr; + fp.nms_thr = dp.nms_thr; + fp.nms_pre = dp.nms_pre; + fp.max_per_img = dp.max_per_img; + fp.input_w = dp.input_w; + fp.input_h = dp.input_h; + fp.base_edge = dp.base_edge; // FoveaBox 만 채워져 온다 + // ⚠️ 격자 오프셋을 kind 로 박으면 안 된다. 같은 `fcos` kind 라도 FCOS·NAS-FCOS· + // FoveaBox 는 0.5 지만 AutoAssign 은 0 이다. 프론트엔드가 생성기에서 읽어 싣는 + // 값을 그대로 쓴다. 박으면 최대 stride 128 에서 64px 밀린다(실측 64.30px). + fp.point_offset = hc.center_offset; + fp.box_xyxy_offset = hc.kind == head_kind::reppoints; + dets = detect_fcos(cls_v, box_v, ctr_v, feat_hw, fp); + } else if (dp.score_voting) { + // PAA·LAD — 점수가 sqrt(cls×iou) 라 iou 갈래(ctr) 없이는 디코드할 수 없다. + if ((int)ctr_v.size() < L) { + throw except("PAA 인데 iou 갈래가 {} 레벨뿐이다 (필요 {})", ctr_v.size(), L); + } + dets = detect_paa(cls_v, box_v, ctr_v, feat_hw, dp); + } else if (dp.fast_nms) { + // YOLACT — softmax + fast NMS. coeff 갈래(ctr_tanh)는 박스 검증에서 안 쓴다. + dets = detect_yolact(cls_v, box_v, feat_hw, dp); + } else { + dets = detect_anchor(cls_v, box_v, feat_hw, dp, ctr_v.empty() ? nullptr : &ctr_v); + } + + return dets; +} + +} // namespace visp diff --git a/src/visp/mmdet_run.h b/src/visp/mmdet_run.h new file mode 100644 index 0000000..5e8d9dd --- /dev/null +++ b/src/visp/mmdet_run.h @@ -0,0 +1,30 @@ +// mmdet_run.h — mmdet 검출기 한 벌을 백본부터 디코드까지 끝낸다. +// +// 계열마다 조립기(mmdet_head.cpp)와 디코더(postproc.cpp)의 조합이 다르고, DDQ 는 층 사이에 +// 호스트 NMS 가 끼어 그래프가 여러 벌로 갈린다. 그 분기를 한자리에 두어 vision-cli 와 검증 +// 하네스가 같은 코드를 쓴다. +#pragma once + +#include "visp/mmdet_head.h" +#include "visp/ml.h" +#include "visp/postproc.h" + +#include +#include + +namespace visp { + +// 백본 그래프를 세우는 함수. arch_registry 의 `arch_forward_fn` 과 같은 시그니처이며, +// 생성물의 `_forward(m, x, _detect_params(file))` 를 감싼 것이다. +using mmdet_backbone_fn = tensor (*)(model_ref, tensor, model_file const&); + +// 전처리까지 끝난 CWHN 입력을 받아 박스를 낸다. 좌표계는 입력 해상도(`input_w`×`input_h`)이며, +// 원본 이미지로 되돌리는 일은 호출한 쪽이 한다. +// +// `cfg.det.input_w` / `input_h` 는 여기서 인자 값으로 덮어쓴다. +std::vector mmdet_run(backend_device& backend, model_weights& weights, + model_file const& file, mmdet_backbone_fn backbone, + mmdet_cfg cfg, std::span input_cwhn, + int input_w, int input_h); + +} // namespace visp diff --git a/src/visp/mmlab.h b/src/visp/mmlab.h new file mode 100644 index 0000000..d776ffc --- /dev/null +++ b/src/visp/mmlab.h @@ -0,0 +1,8 @@ +#pragma once +#include "visp/arch_registry.h" +namespace visp::mmlab { +tensor config_forward(model_ref, tensor, model_file const&); +arch_task config_task(model_file const&); +i64x4 config_input_shape(model_file const&); +} +namespace visp { void register_mmlab_config(); } diff --git a/src/visp/mmlab/architectures.inc b/src/visp/mmlab/architectures.inc new file mode 100644 index 0000000..a6d55dc --- /dev/null +++ b/src/visp/mmlab/architectures.inc @@ -0,0 +1,62 @@ +// Inference architecture families enumerated from existing porting harness configs. +MMLAB_ARCH("mmdet", "ATSS") +MMLAB_ARCH("mmdet", "AutoAssign") +MMLAB_ARCH("mmdet", "BaseReID") +MMLAB_ARCH("mmdet", "BoxInst") +MMLAB_ARCH("mmdet", "CascadeRCNN") +MMLAB_ARCH("mmdet", "CenterNet") +MMLAB_ARCH("mmdet", "CondInst") +MMLAB_ARCH("mmdet", "ConditionalDETR") +MMLAB_ARCH("mmdet", "CornerNet") +MMLAB_ARCH("mmdet", "CrowdDet") +MMLAB_ARCH("mmdet", "DABDETR") +MMLAB_ARCH("mmdet", "DDOD") +MMLAB_ARCH("mmdet", "DDQDETR") +MMLAB_ARCH("mmdet", "DETR") +MMLAB_ARCH("mmdet", "DINO") +MMLAB_ARCH("mmdet", "DeformableDETR") +MMLAB_ARCH("mmdet", "FCOS") +MMLAB_ARCH("mmdet", "FOVEA") +MMLAB_ARCH("mmdet", "FSAF") +MMLAB_ARCH("mmdet", "FastRCNN") +MMLAB_ARCH("mmdet", "FasterRCNN") +MMLAB_ARCH("mmdet", "GFL") +MMLAB_ARCH("mmdet", "GLIP") +MMLAB_ARCH("mmdet", "GridRCNN") +MMLAB_ARCH("mmdet", "GroundingDINO") +MMLAB_ARCH("mmdet", "HybridTaskCascade") +MMLAB_ARCH("mmdet", "KnowledgeDistillationSingleStageDetector") +MMLAB_ARCH("mmdet", "LAD") +MMLAB_ARCH("mmdet", "Mask2Former") +MMLAB_ARCH("mmdet", "Mask2FormerVideo") +MMLAB_ARCH("mmdet", "MaskFormer") +MMLAB_ARCH("mmdet", "MaskRCNN") +MMLAB_ARCH("mmdet", "MaskScoringRCNN") +MMLAB_ARCH("mmdet", "NASFCOS") +MMLAB_ARCH("mmdet", "PAA") +MMLAB_ARCH("mmdet", "PanopticFPN") +MMLAB_ARCH("mmdet", "PointRend") +MMLAB_ARCH("mmdet", "QueryInst") +MMLAB_ARCH("mmdet", "RPN") +MMLAB_ARCH("mmdet", "RTMDet") +MMLAB_ARCH("mmdet", "RepPointsDetector") +MMLAB_ARCH("mmdet", "RetinaNet") +MMLAB_ARCH("mmdet", "SCNet") +MMLAB_ARCH("mmdet", "SOLO") +MMLAB_ARCH("mmdet", "SOLOv2") +MMLAB_ARCH("mmdet", "SingleStageDetector") +MMLAB_ARCH("mmdet", "SparseRCNN") +MMLAB_ARCH("mmdet", "TOOD") +MMLAB_ARCH("mmdet", "TridentFasterRCNN") +MMLAB_ARCH("mmdet", "VFNet") +MMLAB_ARCH("mmdet", "YOLACT") +MMLAB_ARCH("mmdet", "YOLOF") +MMLAB_ARCH("mmdet", "YOLOV3") +MMLAB_ARCH("mmdet", "YOLOX") +MMLAB_ARCH("mmseg", "CascadeEncoderDecoder") +MMLAB_ARCH("mmseg", "DepthEstimator") +MMLAB_ARCH("mmseg", "EncoderDecoder") +MMLAB_ARCH("mmseg", "MultimodalEncoderDecoder") +MMLAB_ARCH("mmpose", "BottomupPoseEstimator") +MMLAB_ARCH("mmpose", "PoseLifter") +MMLAB_ARCH("mmpose", "TopdownPoseEstimator") diff --git a/src/visp/mmlab/backbones.cpp b/src/visp/mmlab/backbones.cpp new file mode 100644 index 0000000..2edb515 --- /dev/null +++ b/src/visp/mmlab/backbones.cpp @@ -0,0 +1,621 @@ +#include "config.h" +#include + +namespace visp::mmlab { +Tensors backbones(model_ref m, Json const& n, Tensors const& inputs) { + auto type=n.at("type").get(); + auto const& a=n.at("attrs"); + auto const& c=n.at("children"); + tensor x=inputs.at(0); + if(type=="StemConv")return run(m,child(n,"proj"),{x}); + if(type=="OverlapPatchEmbed")return {layer(m,n,"norm",layer(m,n,"proj",x))}; + if(type=="MSCAAttention") { + auto base=layer(m,n,"conv0",x),attention=base; + for(int i=0;i<3;++i) { + auto key="conv"+std::to_string(i);attention=ggml_add(m,attention,layer(m,n,key+"_2",layer(m,n,key+"_1",base))); + } + return {ggml_mul(m,x,layer(m,n,"conv3",attention))}; + } + if(type=="MSCASpatialAttention")return {ggml_add(m,x,layer(m,n,"proj_2",layer(m,n,"spatial_gating_unit",layer(m,n,"activation",layer(m,n,"proj_1",x)))))}; + if(type=="MSCABlock") { + auto scale=[&](tensor y,char const* key){auto g=parameter(m,n,key);return ggml_mul(m,y,ggml_reshape_4d(m,g,1,1,g->ne[0],1));}; + x=ggml_add(m,x,scale(layer(m,n,"attn",layer(m,n,"norm1",x)),"layer_scale_1")); + return {ggml_add(m,x,scale(layer(m,n,"mlp",layer(m,n,"norm2",x)),"layer_scale_2"))}; + } + if(type=="MSCAN") { + Tensors out; + for(int i=1;i<=a.at("num_stages").get();++i) { + auto key=std::to_string(i);x=layer(m,n,"block"+key,layer(m,n,"patch_embed"+key,x)); + auto t=ggml_reshape_3d(m,ggml_cont(m,ggml_permute(m,x,1,2,0,3)),x->ne[2],x->ne[0]*x->ne[1],x->ne[3]); + t=layer(m,n,"norm"+key,t); + x=ggml_cont(m,ggml_permute(m,ggml_reshape_4d(m,t,t->ne[0],x->ne[0],x->ne[1],x->ne[3]),2,0,1,3));out.push_back(x); + }return out; + } + if(type=="EfficientNet") { + Tensors out;auto indices=a.at("out_indices").get>();auto const& blocks=child(n,"layers");int i=0; + for(auto const& key:blocks.at("order")) {x=layer(m,blocks,key.get(),x);if(std::find(indices.begin(),indices.end(),i)!=indices.end())out.push_back(x);++i;}return out; + } + if(type=="EdgeResidual") { + auto y=layer(m,n,"conv1",x);if(a.value("with_se",false))y=layer(m,n,"se",y);y=layer(m,n,"conv2",y); + return {a.value("with_residual",false)?ggml_add(m,x,y):y}; + } + if(type=="DDRNet") { + auto width=x->ne[0]/8,height=x->ne[1]/8;bool align=a.value("align_corners",false);x=layer(m,n,"stem",x); + auto const& context=child(n,"context_branch_layers"),spatial=child(n,"spatial_branch_layers");tensor low=x,high=x; + for(int i=0;i<2;++i) { + auto key=std::to_string(i),suffix=std::to_string(i+1); + low=layer(m,context,key,i?ggml_relu(m,low):low);high=layer(m,spatial,key,i?ggml_relu(m,high):high); + auto compressed=layer(m,n,"compression_"+suffix,ggml_relu(m,low)); + low=ggml_add(m,low,layer(m,n,"down_"+suffix,ggml_relu(m,high))); + high=ggml_add(m,high,resize(m,compressed,width,height,true,align)); + } + high=layer(m,spatial,"2",ggml_relu(m,high));low=layer(m,n,"spp",layer(m,context,"2",ggml_relu(m,low))); + return {ggml_add(m,high,resize(m,low,width,height,true,align))}; + } + if(type=="PagFM") { + auto high=x,low=inputs.at(1);if(a.value("after_relu",false)){high=ggml_relu(m,high);low=ggml_relu(m,low);} + bool bilinear=a.value("upsample_mode",std::string{"bilinear"})=="bilinear"; + auto product=ggml_mul(m,layer(m,n,"f_p",high),resize(m,layer(m,n,"f_i",low),high->ne[0],high->ne[1],bilinear));tensor sigma; + if(a.value("with_channel",false))sigma=layer(m,n,"up",product); + else {auto t=ggml_cont(m,ggml_permute(m,product,1,2,0,3));sigma=ggml_cont(m,ggml_permute(m,ggml_sum_rows(m,t),2,0,1,3));} + sigma=ggml_sigmoid(m,sigma);low=resize(m,low,high->ne[0],high->ne[1],bilinear); + return {ggml_add(m,ggml_mul(m,low,sigma),ggml_mul(m,high,ggml_scale_bias(m,sigma,-1,1)))}; + } + if(type=="LightBag"||type=="Bag") { + auto high=x,low=inputs.at(1),sigma=ggml_sigmoid(m,inputs.at(2)); + if(type=="Bag")return {layer(m,n,"conv",ggml_add(m,ggml_mul(m,high,sigma),ggml_mul(m,low,ggml_scale_bias(m,sigma,-1,1))))}; + return {ggml_add(m,layer(m,n,"f_p",ggml_add(m,high,ggml_mul(m,low,ggml_scale_bias(m,sigma,-1,1)))),layer(m,n,"f_i",ggml_add(m,low,ggml_mul(m,high,sigma))))}; + } + if(type=="PIDNet") { + auto width=x->ne[0]/8,height=x->ne[1]/8;bool align=a.value("align_corners",false);x=layer(m,n,"stem",x); + auto const& ib=child(n,"i_branch_layers"),pb=child(n,"p_branch_layers"),db=child(n,"d_branch_layers");tensor low=x,high=x,detail=x; + for(int i=0;i<2;++i) { + auto key=std::to_string(i),suffix=std::to_string(i+1);low=ggml_relu(m,layer(m,ib,key,low)); + high=layer(m,pb,key,i?ggml_relu(m,high):high);detail=layer(m,db,key,i?ggml_relu(m,detail):detail); + high=run(m,child(n,"pag_"+suffix),{high,layer(m,n,"compression_"+suffix,low)}).at(0); + detail=ggml_add(m,detail,resize(m,layer(m,n,"diff_"+suffix,low),width,height,true,align)); + } + high=layer(m,pb,"2",ggml_relu(m,high));detail=layer(m,db,"2",ggml_relu(m,detail));low=layer(m,n,"spp",layer(m,ib,"2",low)); + return run(m,child(n,"dfm"),{high,resize(m,low,width,height,true,align),detail}); + } + if(type=="ICNet") { + Tensors out{layer(m,n,"conv_sub1",x)};bool align=a.value("align_corners",false);auto const& backbone=child(n,"backbone"); + auto half=[&](tensor t) { + auto width=t->ne[0]/2,height=t->ne[1]/2; + if(!align&&(t->ne[0]%2||t->ne[1]%2))t=ggml_cont(m,ggml_view_4d(m,t,2*width,2*height,t->ne[2],t->ne[3],t->nb[1],t->nb[2],t->nb[3],0)); + return resize(m,t,width,height,true,align); + }; + x=half(x); + x=layer(m,backbone,"layer2",layer(m,backbone,"layer1",layer(m,backbone,"maxpool",layer(m,backbone,"stem",x))));out.push_back(layer(m,n,"conv_sub2",x)); + x=half(x);x=layer(m,backbone,"layer4",layer(m,backbone,"layer3",x)); + auto pooled=run(m,child(n,"psp_modules"),{x});pooled.push_back(x); + out.push_back(layer(m,n,"conv_sub4",layer(m,n,"psp_bottleneck",cat(m,pooled))));return out; + } + if(type=="DetailBranch")return run(m,child(n,"detail_branch"),{x}); + if(type=="StemBlock") {x=layer(m,n,"conv_first",x);return {layer(m,n,"fuse_last",cat(m,{layer(m,n,"convs",x),layer(m,n,"pool",x)}))};} + if(type=="GELayer")return {layer(m,n,"act",ggml_add(m,layer(m,n,"conv2",layer(m,n,"dwconv",layer(m,n,"conv1",x))),c.contains("shortcut")?layer(m,n,"shortcut",x):x))}; + if(type=="CEBlock")return {layer(m,n,"conv_last",ggml_add(m,x,layer(m,n,"conv_gap",layer(m,n,"gap",x))))}; + if(type=="SemanticBranch") {Tensors out;for(auto const& name:a.at("semantic_stages")){x=layer(m,n,name.get(),x);out.push_back(x);}return out;} + if(type=="BGALayer") { + auto detail=layer(m,n,"detail_dwconv",x),down=layer(m,n,"detail_down",x); + auto semantic=layer(m,n,"semantic_conv",inputs.at(1)),sd=layer(m,n,"semantic_dwconv",inputs.at(1));bool align=a.value("align_corners",false); + auto first=ggml_mul(m,detail,ggml_sigmoid(m,resize(m,semantic,detail->ne[0],detail->ne[1],true,align))); + auto second=ggml_mul(m,down,ggml_sigmoid(m,sd)); + return {layer(m,n,"conv",ggml_add(m,first,resize(m,second,first->ne[0],first->ne[1],true,align)))}; + } + if(type=="BiSeNetV2") { + auto xs=run(m,child(n,"semantic"),{x});auto y=run(m,child(n,"bga"),{layer(m,n,"detail",x),xs.back()}).at(0); + Tensors all{y},out;all.insert(all.end(),xs.begin(),xs.end()-1); + for(auto const& index:a.at("out_indices"))out.push_back(all.at(index.get()));return out; + } + if(type=="STDCModule") { + auto original=x;auto const& blocks=child(n,"layers");Tensors out;bool add=a.at("fusion_type")=="add",down=a.value("with_downsample",false);int i=0; + for(auto const& key:blocks.at("order")) { + if(i==1&&down&&!add)x=layer(m,n,"downsample",x); + x=layer(m,blocks,key.get(),x);out.push_back(x);++i; + } + if(down&&!add)out[0]=layer(m,n,"skip",out[0]); + auto y=cat(m,out);if(add)y=ggml_add(m,y,down?layer(m,n,"skip",original):original);return {y}; + } + if(type=="STDCNet") { + Tensors out;auto const& stages=child(n,"stages"); + for(auto const& key:stages.at("order")){x=layer(m,stages,key.get(),x);out.push_back(x);} + if(a.value("with_final_conv",false))out.back()=layer(m,n,"final_conv",out.back()); + out.erase(out.begin(),out.begin()+a.at("num_shallow_features").get());return out; + } + if(type=="STDCContextPathNet") { + auto xs=run(m,child(n,"backbone"),{x});auto up=layer(m,n,"conv_avg",adaptive_pool(m,xs.back(),1,1));Tensors arms; + auto const& blocks=child(n,"arms");bool bilinear=a.value("upsample_mode",std::string{"nearest"})=="bilinear",align=a.contains("align_corners")&&!a.at("align_corners").is_null()&&a.at("align_corners").get();int i=0; + for(auto const& key:blocks.at("order")) { + auto source=xs.at(xs.size()-1-i),target=xs.at(xs.size()-2-i); + up=resize(m,up,source->ne[0],source->ne[1],bilinear,align); + up=ggml_add(m,layer(m,blocks,key.get(),source),up); + up=layer(m,child(n,"convs"),key.get(),resize(m,up,target->ne[0],target->ne[1],bilinear,align));arms.push_back(up);++i; + } + auto fused=run(m,child(n,"ffm"),{xs[0],arms.at(1)}).at(0);Tensors out{xs[0]};out.insert(out.end(),arms.begin(),arms.end());out.push_back(fused);return out; + } + if(type=="LayerNorm2d") { + auto y=ggml_cont(m,ggml_permute(m,x,1,2,0,3)); + y=ggml_norm(m,y,a.value("eps",1e-6f));auto wm=weights(m,n); + if(auto weight=wm.find("weight"))y=ggml_mul(m,y,weight); + if(auto bias=wm.find("bias"))y=ggml_add(m,y,bias); + return {ggml_cont(m,ggml_permute(m,y,2,0,1,3))}; + } + if(type=="ConvNeXtBlock") { + auto y=layer(m,n,"depthwise_conv",x);bool linear=a.value("linear_pw_conv",true); + if(linear) { + y=ggml_cont(m,ggml_permute(m,y,1,2,0,3));auto wm=weights(m,child(n,"norm")); + y=ggml_norm(m,y,child(n,"norm").at("attrs").value("eps",1e-6f)); + if(auto weight=wm.find("weight"))y=ggml_mul(m,y,weight); + if(auto bias=wm.find("bias"))y=ggml_add(m,y,bias); + } else y=layer(m,n,"norm",y); + y=layer(m,n,"act",layer(m,n,"pointwise_conv1",y)); + if(c.contains("grn")&&child(n,"grn").at("type")!="Identity")throw std::runtime_error("ConvNeXt GRN not implemented"); + y=layer(m,n,"pointwise_conv2",y); + if(linear)y=ggml_cont(m,ggml_permute(m,y,2,0,1,3)); + if(auto gamma=weights(m,n).find("gamma"))y=ggml_mul(m,y,ggml_reshape_4d(m,gamma,1,1,gamma->ne[0],1)); + return {ggml_add(m,x,y)}; + } + if(type=="ConvNeXt") { + Tensors out;auto indices=a.at("out_indices").get>(); + auto const& stages=child(n,"stages");int i=0; + for(auto const& key:stages.at("order")) { + auto name=key.get();x=layer(m,stages,name,layer(m,child(n,"downsample_layers"),name,x)); + if(std::find(indices.begin(),indices.end(),i)!=indices.end()) { + auto y=a.value("gap_before_final_norm",false)?adaptive_pool(m,x,1,1):x; + y=layer(m,n,"norm"+std::to_string(i),y); + if(a.value("gap_before_final_norm",false))y=ggml_reshape_2d(m,y,y->ne[2],y->ne[3]);out.push_back(y); + }++i; + }return out; + } + if(type=="Pooling")return {ggml_sub(m,layer(m,n,"pool",x),x)}; + if(type=="Mlp") { + x=layer(m,n,"fc1",x);if(c.contains("dwconv"))x=layer(m,n,"dwconv",x); + return {layer(m,n,"drop",layer(m,n,"fc2",layer(m,n,"drop",layer(m,n,"act",x))))}; + } + if(type=="PoolFormerBlock") { + auto wm=weights(m,n);auto scale=[&](tensor y,char const* key){auto g=wm.weights(key);return ggml_mul(m,y,ggml_reshape_4d(m,g,1,1,g->ne[0],1));}; + x=ggml_add(m,x,scale(layer(m,n,"token_mixer",layer(m,n,"norm1",x)),"layer_scale_1")); + return {ggml_add(m,x,scale(layer(m,n,"mlp",layer(m,n,"norm2",x)),"layer_scale_2"))}; + } + if(type=="PoolFormer") { + x=layer(m,n,"patch_embed",x);Tensors out;auto const& network=child(n,"network"); + auto indices=a.at("out_indices").get>();int i=0; + for(auto const& key:network.at("order")) {x=layer(m,network,key.get(),x); + if(std::find(indices.begin(),indices.end(),i)!=indices.end())out.push_back(layer(m,n,"norm"+std::to_string(i),x));++i;} + return out; + } + if(type=="AdaptivePadding") { + auto k=a.at("kernel_size"),s=a.at("stride"),d=a.at("dilation"); + auto ph=std::max(0,((x->ne[1]+s.at(0).get()-1)/s.at(0).get()-1)*s.at(0).get()+(k.at(0).get()-1)*d.at(0).get()+1-x->ne[1]); + auto pw=std::max(0,((x->ne[0]+s.at(1).get()-1)/s.at(1).get()-1)*s.at(1).get()+(k.at(1).get()-1)*d.at(1).get()+1-x->ne[0]); + bool same=a.at("padding")=="same";int lw=same?int(pw/2):0,lh=same?int(ph/2):0; + if(pw||ph)x=ggml_pad_ext(m,x,lw,int(pw)-lw,lh,int(ph)-lh,0,0,0,0);return {x}; + } + if(type=="PatchEmbed") { + if(c.contains("proj"))return {layer(m,n,"norm",layer(m,n,"proj",x))}; + if(c.contains("adap_padding"))x=layer(m,n,"adap_padding",x); + x=layer(m,n,"projection",x); + if(c.contains("norm")) { + auto w=x->ne[0],h=x->ne[1],channels=x->ne[2],batch=x->ne[3]; + auto tokens=ggml_reshape_3d(m,ggml_cont(m,ggml_permute(m,x,1,2,0,3)),channels,w*h,batch); + tokens=layer(m,n,"norm",tokens); + x=ggml_cont(m,ggml_permute(m,ggml_reshape_4d(m,tokens,channels,w,h,batch),2,0,1,3)); + } + return {x}; + } + if(type=="TransformerEncoderLayer") { + if(child(n,"attn").at("type")=="EfficientMultiheadAttention")return transformers(m,n,inputs); + auto normalized=layer(m,n,a.at("norm1_name").get(),x); + x=run(m,child(n,"attn"),{normalized,nullptr,x}).at(0); + normalized=layer(m,n,a.at("norm2_name").get(),x); + return run(m,child(n,"ffn"),{normalized,x}); + } + if(type=="VisionTransformer") { + auto features=layer(m,n,"patch_embed",x);auto w=features->ne[0],h=features->ne[1],channels=features->ne[2]; + auto tokens=ggml_reshape_3d(m,ggml_cont(m,ggml_permute(m,features,1,2,0,3)),channels,w*h,features->ne[3]); + auto wm=weights(m,n);tokens=ggml_concat(m,wm.weights("cls_token"),tokens,1); + auto position=wm.weights("pos_embed"); + if(position->ne[1]!=tokens->ne[1]) { + int ph=a.at("img_size").at(0).get()/a.at("patch_size").get(); + int pw=a.at("img_size").at(1).get()/a.at("patch_size").get(); + auto cls=ggml_cont(m,ggml_view_2d(m,position,channels,1,position->nb[1],0)); + auto grid=ggml_cont(m,ggml_view_2d(m,position,channels,pw*ph,position->nb[1],position->nb[1])); + grid=ggml_cont(m,ggml_permute(m,ggml_reshape_4d(m,grid,channels,pw,ph,1),2,0,1,3)); + auto mode=a.value("interpolate_mode",std::string{"bicubic"}); + int interpolation=mode=="bicubic"?GGML_SCALE_MODE_BICUBIC:mode=="bilinear"?GGML_SCALE_MODE_BILINEAR:GGML_SCALE_MODE_NEAREST; + grid=interpolate(m,grid,{w,h},interpolation); + grid=ggml_reshape_3d(m,ggml_cont(m,ggml_permute(m,grid,1,2,0,3)),channels,w*h,1); + position=ggml_concat(m,cls,grid,1); + } + tokens=ggml_add(m,tokens,position);bool cls=a.value("with_cls_token",true); + auto strip=[&](tensor t){return ggml_cont(m,ggml_view_3d(m,t,channels,w*h,t->ne[2],t->nb[1],t->nb[2],cls?t->nb[1]:0));}; + if(!cls)tokens=ggml_cont(m,ggml_view_3d(m,tokens,channels,w*h,tokens->ne[2],tokens->nb[1],tokens->nb[2],tokens->nb[1])); + if(a.value("pre_norm",false))tokens=layer(m,n,a.at("pre_ln_name").get(),tokens); + Tensors out; + auto append=[&](tensor t) { + auto spatial=ggml_cont(m,ggml_transpose(m,strip(t))); + out.push_back(ggml_reshape_4d(m,spatial,w,h,channels,t->ne[2])); + if(a.value("output_cls_token",false))out.push_back(ggml_cont(m,ggml_view_2d(m,t,channels,1,t->nb[1],0))); + }; + if(a.value("out_origin",false))append(tokens); + auto indices=a.at("out_indices").get>();auto const& layers=child(n,"layers");int i=0; + for(auto const& key:layers.at("order")) { + tokens=layer(m,layers,key.get(),tokens); + if(i+1==int(layers.at("order").size())&&a.value("final_norm",false))tokens=layer(m,n,a.at("norm1_name").get(),tokens); + if(std::find(indices.begin(),indices.end(),i++)!=indices.end())append(tokens); + } + return out; + } + if(type=="SpatialPath") { + for(auto const& name:a.at("layers"))x=layer(m,n,name.get(),x);return {x}; + } + if(type=="AttentionRefinementModule") { + x=layer(m,n,"conv_layer",x);return {ggml_mul(m,x,layer(m,n,"atten_conv_layer",x))}; + } + if(type=="ContextPath") { + auto features=run(m,child(n,"backbone"),{x}); + auto gap=layer(m,n,"gap_conv",features.at(3)); + auto low=ggml_add(m,layer(m,n,"arm32",features.at(3)),gap); + low=layer(m,n,"conv_head32",resize(m,low,features[2]->ne[0],features[2]->ne[1],false)); + auto high=ggml_add(m,layer(m,n,"arm16",features.at(2)),low); + high=layer(m,n,"conv_head16",resize(m,high,features[1]->ne[0],features[1]->ne[1],false)); + return {high,low}; + } + if(type=="BiSeNetV1") { + auto context=run(m,child(n,"context_path"),{x});auto spatial=layer(m,n,"spatial_path",x); + auto fused=run(m,child(n,"ffm"),{spatial,context.at(0)}).at(0); + Tensors all{fused,context.at(0),context.at(1)},out; + for(auto const& index:a.at("out_indices"))out.push_back(all.at(index.get()));return out; + } + if(type=="HourglassModule") { + auto up=layer(m,n,"up1",x),low=layer(m,n,"low3",layer(m,n,"low2",layer(m,n,"low1",x))); + auto cfg=a.at("upsample_cfg");auto w=up->ne[0],h=up->ne[1]; + if(cfg.contains("scale_factor")){float scale=cfg.at("scale_factor").get();w=int64_t(low->ne[0]*scale);h=int64_t(low->ne[1]*scale);} + return {ggml_add(m,up,resize(m,low,w,h,cfg.value("mode",std::string{"nearest"})=="bilinear",cfg.value("align_corners",false)))}; + } + if(type=="HourglassNet") { + x=layer(m,n,"stem",x);Tensors out;int stacks=a.at("num_stacks").get(); + for(int i=0;i(),groups=a.at("groups").get();auto count=x->ne[2]; + if(radix==1)return {ggml_reshape_2d(m,ggml_sigmoid(m,x),count,x->ne[3])}; + x=ggml_reshape_4d(m,ggml_cont(m,x),count/(radix*groups),radix,groups,x->ne[3]); + x=ggml_soft_max(m,ggml_cont(m,ggml_permute(m,x,1,0,2,3))); + x=ggml_cont(m,ggml_permute(m,x,2,0,1,3)); + return {ggml_reshape_2d(m,x,count,x->ne[3])}; + } + if(type=="SplitAttentionConv2d") { + x=layer(m,n,"relu",layer(m,n,a.at("norm0_name").get(),layer(m,n,"conv",x))); + int radix=a.at("radix").get();auto channels=x->ne[2]/radix; + auto part=[&](int i){return ggml_cont(m,ggml_view_4d(m,x,x->ne[0],x->ne[1],channels,x->ne[3],x->nb[1],x->nb[2],x->nb[3],size_t(i*channels)*x->nb[2]));}; + auto gap=part(0);for(int i=1;i(),layer(m,n,"fc1",gap))); + auto attention=layer(m,n,"rsoftmax",layer(m,n,"fc2",gap)); + tensor sum=nullptr; + for(int i=0;inb[0]),1,1,channels,1); + auto y=ggml_mul(m,part(i),weights);sum=sum?ggml_add(m,sum,y):y; + } + return {sum}; + } + if(type=="SSDVGG") { + Tensors out;auto indices=a.at("out_feature_indices").get>();auto const& features=child(n,"features");int i=0; + for(auto const& key:features.at("order")) { + x=layer(m,features,key.get(),x); + if(std::find(indices.begin(),indices.end(),i++)!=indices.end())out.push_back(x); + } + return out; + } + if(type=="ERFNet")return {layer(m,n,"decoder",layer(m,n,"encoder",x))}; + if(type=="NonBottleneck1d")return {layer(m,n,"act",ggml_add(m,x,layer(m,n,"convs_layers",x)))}; + if(type=="UpsamplerBlock")return {layer(m,n,"act",layer(m,n,"bn",layer(m,n,"conv",x)))}; + if(type=="DownsamplerBlock") { + auto conv=layer(m,n,"conv",x),pool=layer(m,n,"pool",x); + pool=resize(m,pool,conv->ne[0],conv->ne[1],true); + return {layer(m,n,"act",layer(m,n,"bn",cat(m,{conv,pool})))}; + } + if(type=="Bottle2neck") { + auto identity=c.contains("downsample")?layer(m,n,"downsample",x):x; + auto plugin=[&](tensor t,int i){auto key="after_conv"+std::to_string(i)+"_plugin_names";if(a.contains(key))for(auto const& k:a.at(key))t=layer(m,n,k.get(),t);return t;}; + x=plugin(layer(m,n,"relu",layer(m,n,a.at("norm1_name").get(),layer(m,n,"conv1",x))),1); + int width=a.at("width").get(),scales=a.at("scales").get(); + auto piece=[&](int i){return ggml_cont(m,ggml_view_4d(m,x,x->ne[0],x->ne[1],width,x->ne[3],x->nb[1],x->nb[2],x->nb[3],size_t(i*width)*x->nb[2]));}; + auto const& convs=child(n,"convs"),bns=child(n,"bns"); + tensor previous=nullptr;Tensors parts; + for(int i=0;i(),layer(m,n,"conv3",x)),3); + return {layer(m,n,"relu",ggml_add(m,x,identity))}; + } + if(type=="BasicConvBlock")return {layer(m,n,"convs",x)}; + if(type=="InterpConv")return {layer(m,n,"interp_upsample",x)}; + if(type=="DeconvModule")return {layer(m,n,"deconv_upsamping",x)}; + if(type=="UpConvBlock")return {layer(m,n,"conv_block",cat(m,{x,layer(m,n,"upsample",inputs.at(1))}))}; + if(type=="UNet") { + Tensors encoded;auto const& encoder=child(n,"encoder"); + for(auto const& key:encoder.at("order")) {x=layer(m,encoder,key.get(),x);encoded.push_back(x);} + Tensors out{x};auto const& decoder=child(n,"decoder"); + for(int i=int(decoder.at("order").size())-1;i>=0;--i) { + x=run(m,child(decoder,std::to_string(i)),{encoded[i],x}).at(0);out.push_back(x); + } + return out; + } + if(type=="LearningToDownsample")return {layer(m,n,"dsconv2",layer(m,n,"dsconv1",layer(m,n,"conv",x)))}; + if(type=="GlobalFeatureExtractor") { + x=layer(m,n,"bottleneck3",layer(m,n,"bottleneck2",layer(m,n,"bottleneck1",x))); + auto pooled=run(m,child(n,"ppm"),{x});pooled.insert(pooled.begin(),x); + return {layer(m,n,"out",cat(m,pooled))}; + } + if(type=="FeatureFusionModule") { + if(c.contains("conv0")) {auto y=layer(m,n,"conv0",cat(m,inputs));return {ggml_add(m,y,ggml_mul(m,y,layer(m,n,"attention",y)))};} + if(c.contains("conv1")) { + auto y=layer(m,n,"conv1",cat(m,inputs));auto atten=layer(m,n,"conv_atten",layer(m,n,"gap",y)); + return {ggml_add(m,y,ggml_mul(m,y,atten))}; + } + if(!c.contains("dwconv"))throw std::runtime_error("unknown feature fusion protocol"); + auto low=inputs.at(1);low=resize(m,low,x->ne[0],x->ne[1],true,a.value("align_corners",false)); + low=layer(m,n,"conv_lower_res",layer(m,n,"dwconv",low)); + return {layer(m,n,"relu",ggml_add(m,layer(m,n,"conv_higher_res",x),low))}; + } + if(type=="FastSCNN") { + auto high=layer(m,n,"learning_to_downsample",x),low=layer(m,n,"global_feature_extractor",high); + Tensors all{high,low,run(m,child(n,"feature_fusion"),{high,low}).at(0)},out; + for(auto const& index:a.at("out_indices"))out.push_back(all.at(index.get()));return out; + } + if(type=="ResBlock")return {ggml_add(m,x,layer(m,n,"conv2",layer(m,n,"conv1",x)))}; + if(type=="Darknet") { + Tensors out;auto indices=a.at("out_indices").get>();int i=0; + for(auto const& name:a.at("cr_blocks")) { + x=layer(m,n,name.get(),x); + if(std::find(indices.begin(),indices.end(),i++)!=indices.end())out.push_back(x); + } + return out; + } + if(type=="InvertedResidual") { + if(c.contains("depthwise_conv")) {auto variant=n;variant["type"]="InvertedResidualV3";return backbones(m,variant,inputs);} + auto y=layer(m,n,"conv",x); + return {a.value("use_res_connect",false)?ggml_add(m,x,y):y}; + } + if(type=="InvertedResidualV3") { + auto y=x; + if(a.value("with_expand_conv",false))y=layer(m,n,"expand_conv",y); + y=layer(m,n,"depthwise_conv",y); + if(a.value("with_se",false))y=layer(m,n,"se",y); + y=layer(m,n,"linear_conv",y); + return {a.value("with_res_shortcut",false)?ggml_add(m,x,y):y}; + } + if(type=="SELayer") { + auto y=layer(m,n,"conv2",layer(m,n,"conv1",layer(m,n,"global_avgpool",x))); + return {ggml_mul(m,x,y)}; + } + if(type=="MobileNetV2" || type=="MobileNetV3") { + if(type=="MobileNetV2")x=layer(m,n,"conv1",x); + Tensors out;auto indices=a.at("out_indices").get>();int i=0; + for(auto const& name:a.at("layers")) { + x=layer(m,n,name.get(),x); + if(std::find(indices.begin(),indices.end(),i++)!=indices.end())out.push_back(x); + } + return out; + } + if(type=="CpmBlock") return {layer(m,n,"model",x)}; + if(type=="CPM") { + Tensors out{layer(m,n,"stem",x)}; + auto middle=layer(m,n,"middle",x); + auto const& stages=child(n,"cpm_stages"), convs=child(n,"out_convs"), mids=child(n,"middle_conv"); + for(auto const& k:stages.at("order")) { + auto key=k.get(); + auto y=cat(m,{out.back(),layer(m,mids,key,middle)}); + out.push_back(layer(m,convs,key,layer(m,stages,key,y))); + } + return out; + } + if(type=="HRModule") { + auto const& branches=child(n,"branches"); + Tensors xs; + for(size_t i=0;i(),inputs.at(i))); + if(xs.size()==1)return xs; + Tensors out; + auto const& fuse=child(n,"fuse_layers"); + for(size_t i=0;i()); + tensor y=nullptr; + for(size_t j=0;j>(); + int i=0; + for(auto const& name:a.at("layers")) { + x=layer(m,n,name.get(),x); + if(std::find(indices.begin(),indices.end(),i)!=indices.end())out.push_back(x); + ++i; + } + return out; + } + if(type=="Focus") { + // Original order: top-left, bottom-left, top-right, bottom-right. + x=cat(m,{space_to_depth_quad(m,x,0,0),space_to_depth_quad(m,x,0,1), + space_to_depth_quad(m,x,1,0),space_to_depth_quad(m,x,1,1)}); + return {layer(m,n,"conv",x)}; + } + if(type=="CSPNeXtBlock" || type=="DarknetBottleneck") { + auto y=layer(m,n,"conv2",layer(m,n,"conv1",x)); + return {a.value("add_identity",false)?ggml_add(m,x,y):y}; + } + if(type=="CSPLayer") { + auto shortcut=layer(m,n,"short_conv",x); + x=layer(m,n,"blocks",layer(m,n,"main_conv",x)); + x=cat(m,{x,shortcut}); + if(a.value("channel_attention",false))x=layer(m,n,"attention",x); + return {layer(m,n,"final_conv",x)}; + } + if(type=="SPPBottleneck") { + x=layer(m,n,"conv1",x); + Tensors xs{x}; + auto const& pools=child(n,"poolings"); + for(auto const& k:pools.at("order")) xs.push_back(layer(m,pools,k.get(),x)); + return {layer(m,n,"conv2",cat(m,xs))}; + } + if(type=="ChannelAttention") { + auto y=layer(m,n,"act",layer(m,n,"fc",layer(m,n,"global_avgpool",x))); + return {ggml_mul(m,x,y)}; + } + if(type=="TCN") { + x=layer(m,n,"expand_conv",x); + Tensors out; + auto const& blocks=child(n,"tcn_blocks"); + for(auto const& k:blocks.at("order")) { + x=layer(m,blocks,k.get(),x);out.push_back(x); + } + return out; + } + if(type=="BasicTemporalBlock") { + auto y=layer(m,n,"conv2",layer(m,n,"conv1",x)); + if(a.value("residual",true)) { + int64_t start=a.value("use_stride_conv",false) + ? a.value("causal_shift",0)+a.at("kernel_size").get()/2 + : a.at("pad").get()+a.value("causal_shift",0); + int64_t stride=a.value("use_stride_conv",false)?a.at("kernel_size").get():1; + // ne0 strides require gathering rows after transpose (views cannot + // represent a strided innermost axis). + auto indices=ggml_cast(m,ggml_arange(m,float(start),float(start+y->ne[0]*stride),float(stride)),GGML_TYPE_I32); + auto r=ggml_cont(m,ggml_transpose(m,ggml_get_rows(m,ggml_cont(m,ggml_transpose(m,x)),indices))); + if(c.contains("short_cut")) r=layer(m,n,"short_cut",r); + y=ggml_add(m,y,r); + } + return {y}; + } + if(type=="BasicBlock" || type=="Bottleneck") { + // YOLOF's residual Bottleneck has ConvModules and no post-add ReLU. + if(child(n,"conv1").at("type")=="ConvModule") { + auto y=layer(m,n,"conv2",layer(m,n,"conv1",x));if(type=="Bottleneck")y=layer(m,n,"conv3",y); + y=ggml_add(m,y,c.contains("downsample")?layer(m,n,"downsample",x):x); + return {c.contains("act")?layer(m,n,"act",y):y}; + } + auto plugins=[&](tensor t,int conv) { + auto key="after_conv"+std::to_string(conv)+"_plugin_names"; + if(a.contains(key))for(auto const& name:a.at(key))t=layer(m,n,name.get(),t); + return t; + }; + tensor identity=c.contains("downsample")?layer(m,n,"downsample",x):x; + x=layer(m,n,"conv1",x); + x=layer(m,n,a.value("norm1_name",std::string{"bn1"}),x); + x=layer(m,n,"relu",x); + x=plugins(x,1); + x=layer(m,n,"conv2",x); + bool split=child(n,"conv2").at("type")=="SplitAttentionConv2d"; + if(split) {if(a.value("avg_down_stride",false))x=layer(m,n,"avd_layer",x);} + else x=layer(m,n,a.value("norm2_name",std::string{"bn2"}),x); + if(type=="Bottleneck") { + if(!split)x=layer(m,n,"relu",x); + x=plugins(x,2); + x=layer(m,n,"conv3",x); + x=layer(m,n,a.value("norm3_name",std::string{"bn3"}),x); + x=plugins(x,3); + } + return {layer(m,n,"relu",ggml_add(m,x,identity))}; + } + if(type=="TridentBottleneck") { + auto residual=c.contains("downsample")?layer(m,n,"downsample",x):x; + x=layer(m,n,"relu",layer(m,n,a.at("norm1_name").get(),layer(m,n,"conv1",x))); + x=layer(m,n,"relu",layer(m,n,a.at("norm2_name").get(),layer(m,n,"conv2",x))); + x=layer(m,n,a.at("norm3_name").get(),layer(m,n,"conv3",x)); + return {layer(m,n,"relu",ggml_add(m,x,residual))}; + } + if(type=="ResNet" || type=="ResNetV1c" || type=="ResNetV1d" || type=="Res2Net" || type=="RegNet" || type=="ResNeSt"||type=="TridentResNet"||type=="DetectoRS_ResNet") { + if(a.value("deep_stem",false)) x=layer(m,n,"stem",x); + else { + x=layer(m,n,"conv1",x); + x=layer(m,n,a.value("norm1_name",std::string{"bn1"}),x); + x=layer(m,n,"relu",x); + } + if(type!="RegNet")x=layer(m,n,"maxpool",x); + Tensors out; + if(type=="DetectoRS_ResNet"&&a.value("output_img",false))out.push_back(inputs.at(0)); + auto indices=a.at("out_indices").get>(); + int i=0; + for(auto const& stage:a.at("res_layers")) { + x=layer(m,n,stage.get(),x); + if(std::find(indices.begin(),indices.end(),i)!=indices.end()) out.push_back(x); + ++i; + } + return out; + } + if(type=="InputInjection") return {layer(m,n,"pool",x)}; + if(type=="GlobalContextExtractor") { + tensor y=layer(m,n,"avg_pool",x); + y=ggml_reshape_2d(m,ggml_cont(m,y),x->ne[2],x->ne[3]); + y=layer(m,n,"fc",y); + y=ggml_reshape_4d(m,y,1,1,x->ne[2],x->ne[3]); + return {ggml_mul(m,x,y)}; + } + if(type=="ContextGuidedBlock") { + tensor orig=x; + x=layer(m,n,"conv1x1",x); + x=cat(m,{layer(m,n,"f_loc",x),layer(m,n,"f_sur",x)}); + x=layer(m,n,"activate",layer(m,n,"bn",x)); + if(a.value("downsample",false)) x=layer(m,n,"bottleneck",x); + x=layer(m,n,"f_glo",x); + if(a.value("skip_connect",false)) x=ggml_add(m,orig,x); + return {x}; + } + if(type=="CGNet") { + auto p2=layer(m,n,"inject_2x",x), p4=layer(m,n,"inject_4x",x); + x=layer(m,n,"stem",x); + x=layer(m,n,"norm_prelu_0",cat(m,{x,p2})); + Tensors out{x}; + for(int level=1;level<=2;++level) { + auto const& stage=child(n,"level"+std::to_string(level)); + tensor down=nullptr; + for(auto const& k:stage.at("order")) { + x=layer(m,stage,k.get(),x); + if(!down) down=x; + } + if(!down) throw std::runtime_error("CGNet stage must contain blocks"); + x=cat(m,level==1?Tensors{x,down,p4}:Tensors{down,x}); + x=layer(m,n,"norm_prelu_"+std::to_string(level),x); + out.push_back(x); + } + return out; + } + throw std::runtime_error("missing backbone forward"); +} +} // namespace visp::mmlab diff --git a/src/visp/mmlab/config.cpp b/src/visp/mmlab/config.cpp new file mode 100644 index 0000000..278028f --- /dev/null +++ b/src/visp/mmlab/config.cpp @@ -0,0 +1,147 @@ +#include "config.h" +#include +#include + +namespace visp::mmlab { +// MMDetection CocoDataset.METAINFO["classes"] 순서. +static std::vector const& coco_class_names() { + static std::vector const names = { + "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", + "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", + "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", + "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", + "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", + "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", + "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", + "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", + "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", + "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", + "toothbrush"}; + return names; +} +Json read_config(model_file const& f) { + auto text = f.get_string("mmlab.config"); + if (text.empty() || text.size() > 16*1024*1024) + throw std::runtime_error("mmlab-config: missing or oversized configuration"); + auto j = Json::parse(text); + if (j.at("version") != 1 || j.contains("instructions")) + throw std::runtime_error("mmlab-config: unsupported schema"); + return j; +} +Json const& child(Json const& n, std::string const& key) { return n.at("children").at(key); } +model_ref weights(model_ref m, Json const& n) { + return m.with_prefix(tensor_name(n.at("prefix").get().c_str())); +} +tensor parameter(model_ref m,Json const& n,char const* name) { + if(n.contains("tensors")&&n.at("tensors").contains(name))return m.weights(n.at("tensors").at(name).get().c_str()); + return weights(m,n).weights(name); +} +Tensors run(model_ref m, Json const& n, Tensors const& x) { + static std::unordered_map const builders = { +#define MMLAB_TYPE(name, group) {name, &group}, +#include "types.inc" +#undef MMLAB_TYPE + }; + auto type = n.at("type").get(); + auto it = builders.find(type); + if (it == builders.end()) throw std::runtime_error("mmlab-config: unimplemented module " + type); + try { return it->second(m, n, x); } + catch (std::exception const& e) { + throw std::runtime_error(type + "(" + n.at("prefix").get() + "): " + e.what()); + } +} +tensor one(model_ref m, Json const& n, tensor x) { + auto out = run(m,n,{x}); + if (out.size()!=1) throw std::runtime_error("expected a single tensor"); + return out[0]; +} +tensor layer(model_ref m, Json const& n, std::string const& key, tensor x) { + return one(m,child(n,key),x); +} +tensor cat(model_ref m, Tensors const& xs, int axis) { + if(xs.empty()) throw std::runtime_error("empty concatenation"); + tensor y=xs[0]; + for(size_t i=1;ine[0],weight->ne[1])); + result=result?ggml_add(m,result,sampled):sampled; + }return result; +} +i64x4 config_input_shape(model_file const& f) { + auto s=read_config(f).at("input_shape").get>(); + if(s.size()<2 || s.size()>4 || s[0]!=1) throw std::runtime_error("mmlab-config: batch 1, rank 2..4 required"); + for(auto d:s) if(d<=0 || d>INT32_MAX) throw std::runtime_error("mmlab-config: invalid input shape"); + if(s.size()==4) return {s[1],s[3],s[2],s[0]}; + i64x4 out{1,1,1,1}; + for(size_t i=0;i>(); + if(cfg.contains("std")) task.stdv=cfg.at("std").get>(); + task.to_rgb=cfg.value("to_rgb",false); + // 이 키가 없는 옛 GGUF 는 예전처럼 가로·세로를 따로 늘린다. + task.keep_ratio=cfg.value("keep_ratio",false); + if(cfg.contains("pad_value")) task.pad_value=cfg.at("pad_value").get>(); + if(cfg.value("ecosystem",std::string{})=="mmdet" && cfg.value("mmdet_head_type",std::string{"raw"})!="raw") { + auto const& j=cfg.at("mmdet"); + auto& c=task.mmdet; + static std::unordered_map const kinds={ + {"anchor",head_kind::anchor}, + {"yolox",head_kind::yolox}, + {"rpn",head_kind::rpn}, + {"fcos",head_kind::fcos}, + {"gfl",head_kind::gfl}, + {"vfnet",head_kind::vfnet}, + {"reppoints",head_kind::reppoints}, + {"sabl",head_kind::sabl}, + {"tood",head_kind::tood}, + {"centernet",head_kind::centernet}, + {"yolof",head_kind::yolof}, + {"yolo",head_kind::yolo}, + {"cornernet",head_kind::cornernet}, + {"detr",head_kind::detr}, + {"conditional_detr",head_kind::conditional_detr}, + {"dab_detr",head_kind::dab_detr}, + {"deformable_detr",head_kind::deformable_detr}, + {"dino",head_kind::dino}, + {"ddq",head_kind::ddq}, + }; + c.head.kind=kinds.at(j.at("head").at("kind").get()); +#include "../runtime_mmdet.inc" + c.det.input_w=task.input_w;c.det.input_h=task.input_h; + std::copy(task.mean.begin(),task.mean.end(),c.img_mean); + std::copy(task.stdv.begin(),task.stdv.end(),c.img_std); + c.to_rgb=task.to_rgb; + task.kind=arch_kind::detect_mmdet; + // 이름이 없으면 결과 그림이 "class N" 으로 나온다. config 에 classes 가 있으면 그것을, + // 없고 80 클래스면 COCO 이름을 붙인다. + if(cfg.contains("classes")) task.class_names=cfg.at("classes").get>(); + else if(c.det.num_classes==80) task.class_names=coco_class_names(); + } + return task; +} +} // namespace visp::mmlab diff --git a/src/visp/mmlab/config.h b/src/visp/mmlab/config.h new file mode 100644 index 0000000..8cbd6bc --- /dev/null +++ b/src/visp/mmlab/config.h @@ -0,0 +1,35 @@ +#pragma once +#include "visp/mmlab.h" +#include "visp/nn.h" +#include "nlohmann/json.hpp" +#include + +namespace visp::mmlab { +using Json = nlohmann::json; +using Tensors = std::vector; +using Builder = Tensors (*)(model_ref, Json const&, Tensors const&); +Json read_config(model_file const&); +arch_task task_from_metadata(Json const&, i64x4 shape); +Tensors run(model_ref, Json const&, Tensors const&); +tensor one(model_ref, Json const&, tensor); +Json const& child(Json const&, std::string const&); +tensor layer(model_ref, Json const&, std::string const&, tensor); +model_ref weights(model_ref, Json const&); +tensor parameter(model_ref, Json const&, char const*); +tensor cat(model_ref, Tensors const&, int axis = 2); +tensor resize(model_ref, tensor, int64_t w, int64_t h, bool bilinear, bool align = false); +tensor adaptive_pool(model_ref, tensor, int w, int h, bool maximum = false); +tensor sample_tokens(model_ref, tensor, int64_t width, int64_t height, tensor x, tensor y); +Tensors modules(model_ref, Json const&, Tensors const&); +Tensors backbones(model_ref, Json const&, Tensors const&); +Tensors heads(model_ref, Json const&, Tensors const&); +Tensors transformers(model_ref, Json const&, Tensors const&); +Tensors text_fusion(model_ref, Json const&, Tensors const&); +Tensors models(model_ref, Json const&, Tensors const&); +Tensors build_architecture(model_ref, std::string const&, Json const&, Tensors const&); +tensor config_forward(model_ref, tensor, model_file const&); +arch_task config_task(model_file const&); +i64x4 config_input_shape(model_file const&); +} // namespace visp::mmlab + +namespace visp { void register_mmlab_config(); } diff --git a/src/visp/mmlab/heads.cpp b/src/visp/mmlab/heads.cpp new file mode 100644 index 0000000..f8941fc --- /dev/null +++ b/src/visp/mmlab/heads.cpp @@ -0,0 +1,1313 @@ +#include "config.h" +#include +#include + +namespace visp::mmlab { +namespace { +tensor select(Tensors const& inputs, Json const& value) { + int64_t index=value.get(); + if(index<0)index+=int64_t(inputs.size()); + if(index<0||index>=int64_t(inputs.size()))throw std::runtime_error("feature index out of range"); + return inputs.at(size_t(index)); +} +Tensors transform(model_ref m, Json const& a, Tensors const& inputs) { + auto index=a.at("in_index"); + if(!index.is_array()) return {select(inputs,index)}; + Tensors out; + for(auto const& i:index) out.push_back(select(inputs,i)); + auto mode=a.value("input_transform",Json{}); + if(mode=="resize_concat") { + for(size_t i=1;ine[0],out[0]->ne[1],true,a.value("align_corners",false)); + return {cat(m,out)}; + } + if(mode!="multiple_select") throw std::runtime_error("unknown input transform"); + return out; +} +} +Tensors heads(model_ref m, Json const& n, Tensors const& inputs) { + auto type=n.at("type").get(); + auto const& a=n.at("attrs"); auto const& c=n.at("children"); + if(type=="CLIPTextEncoder")return {parameter(m,n,"native_class_embeddings")}; + if(type=="MLP") { + auto x=inputs.at(0);auto const& layers=child(n,"layers");int i=0; + for(auto const& key:layers.at("order")) {x=layer(m,layers,key.get(),x);if(++i1?inputs[1]:nullptr}).at(0); + return run(m,child(child(n,"ffns"),"0"),{layer(m,child(n,"norms"),"1",x),x}); + } + if(type=="MLPMaskDecoder") { + auto query=layer(m,n,"query_mlp",inputs.at(0)),pixel=layer(m,n,"pix_mlp",inputs.at(1)); + auto tokens=[&](tensor x){return ggml_reshape_2d(m,ggml_cont(m,ggml_permute(m,x,1,2,0,3)),x->ne[2],x->ne[0]*x->ne[1]);}; + auto mask=ggml_cont(m,ggml_transpose(m,ggml_mul_mat(m,query,tokens(pixel)))); + Tensors out{ggml_reshape_4d(m,mask,pixel->ne[0],pixel->ne[1],query->ne[1],1)}; + auto attention=layer(m,n,"attn_mlp",inputs.at(1));int heads=a.at("total_heads").get(),layers=a.at("total_layers").get(); + for(int l=0;lne[0],attention->ne[1],query->ne[0],1,attention->nb[1],attention->nb[2],attention->nb[3],size_t((l*heads+h)*query->ne[0])*attention->nb[2])); + biases.push_back(ggml_cont(m,ggml_transpose(m,ggml_mul_mat(m,query,tokens(values))))); + } + auto bias=cat(m,biases,2);auto shape=i64x4{bias->ne[0],bias->ne[1],bias->ne[2],bias->ne[3]}; + bias=layer(m,n,"bias_scaling",ggml_reshape_2d(m,bias,1,ggml_nelements(bias))); + bias=ggml_reshape_4d(m,bias,pixel->ne[0],pixel->ne[1],shape[1],shape[2]);out.push_back(bias); + }return out; + } + if(type=="SideAdapterNetwork") { + auto image=inputs.at(0),feature=layer(m,n,"patch_embed",image);auto width=feature->ne[0],height=feature->ne[1],channels=feature->ne[2]; + auto tokens=[&](tensor x){return ggml_reshape_2d(m,ggml_cont(m,ggml_permute(m,x,1,2,0,3)),x->ne[2],x->ne[0]*x->ne[1]);}; + auto position=parameter(m,n,"pos_embed");auto size=child(n,"patch_embed").at("attrs").at("init_out_size"); + if(position->ne[1]!=width*height) { + position=ggml_cont(m,ggml_permute(m,ggml_reshape_4d(m,position,channels,size.at(1).get(),size.at(0).get(),1),2,0,1,3)); + position=tokens(interpolate(m,position,{width,height},GGML_SCALE_MODE_BICUBIC)); + } + position=ggml_concat(m,parameter(m,n,"query_pos_embed"),position,1); + auto x=ggml_add(m,ggml_concat(m,parameter(m,n,"query_embed"),tokens(feature),1),position); + auto count=parameter(m,n,"query_embed")->ne[1];int fusion=0;auto indices=a.at("fusion_index").get>(); + auto fuse=[&]() { + auto clip=layer(m,child(n,"conv_clips"),std::to_string(fusion),inputs.at(1+2*fusion)); + clip=tokens(resize(m,clip,width,height,true)); + auto queries=ggml_cont(m,ggml_view_2d(m,x,channels,count,x->nb[1],0)); + auto pixels=ggml_cont(m,ggml_view_2d(m,x,channels,width*height,x->nb[1],size_t(count)*x->nb[1])); + x=ggml_concat(m,queries,ggml_add(m,pixels,clip),1);++fusion; + }; + if(indices.at(0)==0)fuse();auto const& layers=child(n,"encode_layers");int i=0; + for(auto const& key:layers.at("order")) { + x=layer(m,layers,key.get(),x);++i; + if(inb[1],0)); + auto pixel=ggml_cont(m,ggml_view_2d(m,x,channels,width*height,x->nb[1],size_t(count)*x->nb[1])); + pixel=ggml_cont(m,ggml_permute(m,ggml_reshape_4d(m,pixel,channels,width,height,1),2,0,1,3)); + return run(m,child(n,"mask_decoder"),{query,pixel}); + } + if(type=="RecWithAttnbias") { + if(a.value("cross_attn",false)||a.at("sos_token_format")!="cls_token")throw std::runtime_error("SAN recognition requires self-attention and cls tokens"); + auto feature=inputs.at(0),cls=inputs.at(1);int count=a.at("sos_token_num").get(),heads=a.at("num_heads").get(); + auto width=feature->ne[0],height=feature->ne[1],spatial=width*height,channels=feature->ne[2]; + auto tokens=ggml_reshape_2d(m,ggml_cont(m,ggml_permute(m,feature,1,2,0,3)),channels,spatial); + auto sos=ggml_repeat(m,cls,ggml_new_tensor_2d(m,GGML_TYPE_F32,channels,count)); + auto x=ggml_concat(m,sos,ggml_concat(m,cls,tokens,1),1);int i=0;auto const& layers=child(n,"layers"); + for(auto const& key:layers.at("order")) { + auto bias=adaptive_pool(m,inputs.at(inputs.size()==3?2:2+i),int(width),int(height),true); + bias=ggml_reshape_3d(m,bias,spatial,count,bias->ne[3]); + if(bias->ne[2]==1)bias=ggml_repeat(m,bias,ggml_new_tensor_3d(m,GGML_TYPE_F32,spatial,count,heads)); + auto positions=ggml_arange(m,0,float(count),1),left=ggml_reshape_2d(m,positions,count,1),right=ggml_reshape_2d(m,positions,1,count); + auto square=ggml_new_tensor_2d(m,GGML_TYPE_F32,count,count); + auto diagonal=ggml_scale(m,ggml_step(m,ggml_abs(m,ggml_sub(m,ggml_repeat(m,left,square),right))),-100); + auto cls_column=ggml_scale_bias(m,ggml_sum_rows(m,diagonal),0,-100); + auto top=ggml_concat(m,diagonal,cls_column,0);top=ggml_repeat(m,top,ggml_new_tensor_3d(m,GGML_TYPE_F32,count+1,count,heads)); + top=ggml_concat(m,top,bias,0); + auto zero=ggml_scale(m,ggml_arange(m,0,float(count+1+spatial),1),0); + auto lower=ggml_repeat(m,zero,ggml_new_tensor_3d(m,GGML_TYPE_F32,count+1+spatial,spatial+1,heads)); + auto blocked=ggml_scale_bias(m,ggml_repeat(m,ggml_reshape_2d(m,ggml_step(m,ggml_scale_bias(m,ggml_arange(m,0,float(count+1+spatial),1),-1,float(count)-.5f)),count+1+spatial,1),lower),-100,0); + auto mask=ggml_concat(m,top,blocked,1); + x=run(m,child(layers,key.get()),{x,mask}).at(0);++i; + } + x=ggml_cont(m,ggml_view_2d(m,x,channels,count,x->nb[1],0));x=layer(m,n,"proj",layer(m,n,"ln_post",x)); + if(a.value("final_norm",true))x=ggml_div(m,x,ggml_sqrt(m,ggml_sum_rows(m,ggml_sqr(m,x))));return {x}; + } + if(type=="SideAdapterCLIPHead") { + auto features=inputs;auto classes=features.back();features.pop_back();auto image=features.front();features.erase(features.begin()); + auto masks_input=features;masks_input.insert(masks_input.begin(),image);auto masks=run(m,child(n,"side_adapter_network"),masks_input); + Tensors recognition{features.at(features.size()-2),features.back()};recognition.insert(recognition.end(),masks.begin()+1,masks.end()); + auto embedding=run(m,child(n,"rec_with_attnbias"),recognition).at(0); + auto scores=ggml_soft_max(m,ggml_mul_mat(m,classes,embedding));int64_t count=a.at("num_classes").get(); + scores=ggml_cont(m,ggml_view_2d(m,scores,count,scores->ne[1],scores->nb[1],0)); + auto mask=ggml_sigmoid(m,resize(m,masks.front(),image->ne[0],image->ne[1],true)); + auto spatial=mask->ne[0]*mask->ne[1];mask=ggml_reshape_2d(m,mask,spatial,mask->ne[2]); + auto result=ggml_mul_mat(m,ggml_cont(m,ggml_transpose(m,mask)),ggml_cont(m,ggml_transpose(m,scores))); + return {ggml_reshape_4d(m,result,image->ne[0],image->ne[1],count,1)}; + } + if(type=="MultiScaleDeformableAttention") { + auto query=inputs.at(0),position=inputs.at(1),refx=inputs.at(2),refy=inputs.at(3); + int heads=a.at("num_heads").get(),levels=a.at("num_levels").get(),points=a.at("num_points").get(); + auto shapes=a.at("native_shapes");auto value=layer(m,n,"value_proj",query);auto channels=value->ne[0],count=query->ne[1],dim=channels/heads; + value=ggml_cont(m,ggml_permute(m,ggml_reshape_3d(m,value,dim,heads,count),0,2,1,3)); + auto q=ggml_add(m,query,position),offsets=layer(m,n,"sampling_offsets",q),attention=layer(m,n,"attention_weights",q); + attention=ggml_soft_max(m,ggml_reshape_3d(m,attention,points*levels,heads,count)); + auto extract=[&](tensor t,int64_t index,int64_t stride,int64_t row_stride) { + auto v=ggml_view_3d(m,t,1,heads,count,stride,row_stride,size_t(index)*t->nb[0]); + return ggml_reshape_2d(m,ggml_cont(m,ggml_permute(m,v,0,2,1,3)),count,heads); + }; + tensor total=nullptr;int64_t begin=0; + for(int l=0;l(),height=shapes.at(l).at(1).get(); + auto features=ggml_cont(m,ggml_view_3d(m,value,dim,width*height,heads,value->nb[1],value->nb[2],size_t(begin)*value->nb[1]));begin+=width*height; + for(int p=0;pnb[0],offsets->nb[1]),oy=extract(offsets,2*index+1,2*levels*points*offsets->nb[0],offsets->nb[1]); + auto target=ggml_new_tensor_2d(m,GGML_TYPE_F32,count,heads); + auto xp=ggml_add(m,ox,ggml_repeat(m,ggml_scale_bias(m,refx,float(width),-.5f),target)); + auto yp=ggml_add(m,oy,ggml_repeat(m,ggml_scale_bias(m,refy,float(height),-.5f),target)); + auto sampled=sample_tokens(m,features,width,height,xp,yp); + auto weight=extract(attention,index,attention->nb[1],attention->nb[2]); + sampled=ggml_mul(m,sampled,ggml_reshape_3d(m,weight,1,count,heads));total=total?ggml_add(m,total,sampled):sampled; + } + } + total=ggml_reshape_2d(m,ggml_cont(m,ggml_permute(m,total,0,2,1,3)),channels,count); + return {ggml_add(m,query,layer(m,n,"output_proj",total))}; + } + if(type=="Mask2FormerTransformerEncoder") { + auto x=inputs.at(0);auto const& layers=child(n,"layers"); + for(auto const& key:layers.at("order")) { + auto node=child(layers,key.get());node["attrs"]["native_shapes"]=a.at("native_shapes"); + x=run(m,node,{x,inputs.at(1),inputs.at(2),inputs.at(3)}).at(0); + }return {x}; + } + if(type=="DeformableDetrTransformerEncoderLayer") { + auto attention=child(n,"self_attn");attention["attrs"]["native_shapes"]=a.at("native_shapes"); + auto x=run(m,attention,inputs).at(0);auto const& norms=child(n,"norms");x=layer(m,norms,"0",x); + return {layer(m,norms,"1",layer(m,n,"ffn",x))}; + } + if(type=="MSDeformAttnPixelDecoder") { + int levels=a.at("num_encoder_levels").get();Tensors projected,positions,rx,ry;Json shapes=Json::array(); + auto encoding=parameter(m,child(n,"level_encoding"),"weight"); + for(int i=0;ine[0],height=x->ne[1],channels=x->ne[2]; + auto position=layer(m,n,"postional_encoding",x),level=ggml_view_1d(m,encoding,channels,size_t(i)*encoding->nb[1]); + positions.push_back(ggml_add(m,position,level));shapes.push_back({width,height}); + projected.push_back(ggml_reshape_2d(m,ggml_cont(m,ggml_permute(m,x,1,2,0,3)),channels,width*height)); + auto nx=ggml_reshape_2d(m,ggml_scale_bias(m,ggml_arange(m,0,float(width),1),1.f/width,.5f/width),width,1); + auto ny=ggml_reshape_2d(m,ggml_scale_bias(m,ggml_arange(m,0,float(height),1),1.f/height,.5f/height),1,height); + auto shape=ggml_new_tensor_2d(m,GGML_TYPE_F32,width,height); + rx.push_back(ggml_reshape_1d(m,ggml_repeat(m,nx,shape),width*height));ry.push_back(ggml_reshape_1d(m,ggml_repeat(m,ny,shape),width*height)); + } + auto encoder=child(n,"encoder");encoder["attrs"]["native_shapes"]=shapes; + auto memory=run(m,encoder,{cat(m,projected,1),cat(m,positions,1),cat(m,rx,0),cat(m,ry,0)}).at(0);Tensors out;int64_t begin=0; + for(auto const& shape:shapes) { + int64_t width=shape.at(0).get(),height=shape.at(1).get(); + auto t=ggml_cont(m,ggml_view_2d(m,memory,memory->ne[0],width*height,memory->nb[1],size_t(begin)*memory->nb[1]));begin+=width*height; + out.push_back(ggml_cont(m,ggml_permute(m,ggml_reshape_4d(m,t,t->ne[0],width,height,1),2,0,1,3))); + } + for(int i=int(inputs.size())-levels-1;i>=0;--i) { + auto lateral=layer(m,child(n,"lateral_convs"),std::to_string(int(inputs.size())-levels-1-i),inputs.at(i)); + out.push_back(layer(m,child(n,"output_convs"),std::to_string(int(inputs.size())-levels-1-i),ggml_add(m,lateral,resize(m,out.back(),lateral->ne[0],lateral->ne[1],true)))); + } + Tensors result{layer(m,n,"mask_feature",out.back())};for(int i=0;i();++i)result.push_back(out.at(i));return result; + } + if(type=="PointHead") { + auto features=inputs;auto previous=features.back();features.pop_back();features=transform(m,a,features); + auto refined=previous;auto const& cfg=a.at("native_test_cfg");bool align=a.value("align_corners",false); + if(align)throw std::runtime_error("PointRend requires align_corners=False"); + auto tokens=[&](tensor x){return ggml_reshape_2d(m,ggml_cont(m,ggml_permute(m,x,1,2,0,3)),x->ne[2],x->ne[0]*x->ne[1]);}; + for(int iteration=0;iteration();++iteration) { + int scale=cfg.at("scale_factor").get();refined=resize(m,refined,refined->ne[0]*scale,refined->ne[1]*scale,true,align); + auto logits=tokens(refined);auto spatial=refined->ne[0]*refined->ne[1]; + auto top=ggml_cont(m,ggml_argsort_top_k(m,logits,2)); + auto offset=ggml_reshape_2d(m,ggml_scale(m,ggml_arange(m,0,float(spatial),1),float(logits->ne[0])),1,spatial); + auto indices=ggml_cast(m,ggml_add(m,ggml_cast(m,top,GGML_TYPE_F32),offset),GGML_TYPE_I32); + auto scores=ggml_get_rows(m,ggml_reshape_2d(m,logits,1,ggml_nelements(logits)),ggml_reshape_1d(m,indices,2*spatial)); + scores=ggml_reshape_2d(m,scores,2,spatial); + auto first=ggml_view_2d(m,scores,1,spatial,scores->nb[1],0),second=ggml_view_2d(m,scores,1,spatial,scores->nb[1],scores->nb[0]); + auto uncertainty=ggml_reshape_1d(m,ggml_cont(m,ggml_sub(m,second,first)),spatial); + int count=int(std::min(spatial,cfg.at("subdivision_num_points").get())); + auto points=ggml_cont(m,ggml_argsort_top_k(m,uncertainty,count)); + auto linear=ggml_cast(m,points,GGML_TYPE_F32),row=ggml_floor(m,ggml_scale(m,linear,1.f/refined->ne[0])); + auto col=ggml_sub(m,linear,ggml_scale(m,row,float(refined->ne[0]))); + auto nx=ggml_scale_bias(m,col,1.f/refined->ne[0],.5f/refined->ne[0]),ny=ggml_scale_bias(m,row,1.f/refined->ne[1],.5f/refined->ne[1]); + auto sample=[&](tensor x) {return sample_tokens(m,tokens(x),x->ne[0],x->ne[1],ggml_scale_bias(m,nx,float(x->ne[0]),-.5f),ggml_scale_bias(m,ny,float(x->ne[1]),-.5f));}; + Tensors fine;for(auto x:features)fine.push_back(sample(x));auto coarse=sample(previous); + auto x=ggml_cont(m,ggml_transpose(m,cat(m,fine,0)));coarse=ggml_cont(m,ggml_transpose(m,coarse));x=ggml_concat(m,x,coarse,1); + auto const& layers=child(n,"fcs"); + for(auto const& key:layers.at("order")) {x=layer(m,layers,key.get(),x);if(a.value("coarse_pred_each_layer",true))x=ggml_concat(m,x,coarse,1);} + x=ggml_cont(m,ggml_transpose(m,layer(m,n,"fc_seg",x))); + auto output=ggml_set_rows(m,logits,x,points); + refined=ggml_cont(m,ggml_permute(m,ggml_reshape_4d(m,output,output->ne[0],refined->ne[0],refined->ne[1],1),2,0,1,3)); + } + return {resize(m,refined,a.at("native_input_w").get(),a.at("native_input_h").get(),true,align)}; + } + if(type=="PixelDecoder"||type=="TransformerEncoderPixelDecoder") { + auto memory=inputs.back(),x=memory; + if(type=="PixelDecoder")x=layer(m,n,"last_feat_conv",x); + else { + x=layer(m,n,"encoder_in_proj",x);auto width=x->ne[0],height=x->ne[1],channels=x->ne[2]; + auto position=layer(m,n,"positional_encoding",x); + x=ggml_reshape_2d(m,ggml_cont(m,ggml_permute(m,x,1,2,0,3)),channels,width*height); + x=run(m,child(n,"encoder"),{x,position}).at(0); + memory=ggml_cont(m,ggml_permute(m,ggml_reshape_4d(m,x,channels,width,height,1),2,0,1,3)); + x=layer(m,n,"encoder_out_proj",memory); + } + for(int i=int(inputs.size())-2;i>=0;--i) { + auto lateral=layer(m,child(n,"lateral_convs"),std::to_string(i),inputs.at(i)); + x=layer(m,child(n,"output_convs"),std::to_string(i),ggml_add(m,lateral,resize(m,x,lateral->ne[0],lateral->ne[1],false))); + }return {layer(m,n,"mask_feature",x),memory}; + } + if(type=="DetrTransformerDecoderLayer"||type=="Mask2FormerTransformerDecoderLayer") { + auto x=inputs.at(0);auto const& norms=child(n,"norms"); + auto self=[&](tensor t){return run(m,child(n,"self_attn"),{t,inputs.at(1),t}).at(0);}; + auto cross=[&](tensor t){return run(m,child(n,"cross_attn"),{t,inputs.at(1),t,inputs.at(2),inputs.at(2),inputs.at(3),inputs.size()>4?inputs[4]:nullptr}).at(0);}; + x=layer(m,norms,"0",type=="DetrTransformerDecoderLayer"?self(x):cross(x)); + x=layer(m,norms,"1",type=="DetrTransformerDecoderLayer"?cross(x):self(x)); + return {layer(m,norms,"2",layer(m,n,"ffn",x))}; + } + if(type=="DetrTransformerDecoder") { + auto x=inputs.at(0);auto const& layers=child(n,"layers"); + for(auto const& key:layers.at("order"))x=run(m,child(layers,key.get()),{x,inputs.at(1),inputs.at(2),inputs.at(3)}).at(0); + return {layer(m,n,"post_norm",x)}; + } + if(type=="MaskFormerHead"||type=="Mask2FormerHead") { + auto decoded=run(m,child(n,"pixel_decoder"),inputs);auto feature=decoded.at(0); + auto query=parameter(m,child(n,"query_embed"),"weight");tensor output; + if(type=="MaskFormerHead") { + auto memory=layer(m,n,"decoder_input_proj",decoded.at(1)),position=layer(m,n,"decoder_pe",decoded.at(1)); + memory=ggml_reshape_2d(m,ggml_cont(m,ggml_permute(m,memory,1,2,0,3)),memory->ne[2],memory->ne[0]*memory->ne[1]); + output=run(m,child(n,"transformer_decoder"),{ggml_scale(m,query,0),query,memory,position}).at(0); + } else { + auto const& decoder=child(n,"transformer_decoder"),layers=child(decoder,"layers"); + Tensors memories,positions;auto encoding=parameter(m,child(n,"level_embed"),"weight"); + int levels=a.at("num_transformer_feat_level").get(); + for(int i=0;ine[2],x->ne[0]*x->ne[1]); + memories.push_back(ggml_add(m,memory,ggml_view_1d(m,encoding,encoding->ne[0],size_t(i)*encoding->nb[1]))); + positions.push_back(layer(m,n,"decoder_positional_encoding",decoded.at(i+1))); + } + auto state=parameter(m,child(n,"query_feat"),"weight");int i=0; + auto pixels=ggml_reshape_2d(m,ggml_cont(m,ggml_permute(m,feature,1,2,0,3)),feature->ne[2],feature->ne[0]*feature->ne[1]); + for(auto const& key:layers.at("order")) { + int level=i++%levels;auto embed=layer(m,n,"mask_embed",layer(m,decoder,"post_norm",state)); + auto masks=ggml_cont(m,ggml_transpose(m,ggml_mul_mat(m,embed,pixels))); + masks=ggml_reshape_4d(m,masks,feature->ne[0],feature->ne[1],embed->ne[1],1); + auto target=decoded.at(level+1);masks=resize(m,masks,target->ne[0],target->ne[1],true); + auto blocked=ggml_step(m,ggml_scale(m,ggml_reshape_2d(m,masks,target->ne[0]*target->ne[1],embed->ne[1]),-1)); + auto available=ggml_step(m,ggml_scale_bias(m,ggml_sum_rows(m,blocked),-1,float(blocked->ne[0])-.5f)); + auto full=ggml_new_tensor_2d(m,GGML_TYPE_F32,blocked->ne[0],blocked->ne[1]); + auto mask=ggml_scale(m,ggml_mul(m,blocked,ggml_repeat(m,available,full)),-1e9f); + state=run(m,child(layers,key.get()),{state,query,memories[level],positions[level],mask}).at(0); + }output=layer(m,decoder,"post_norm",state); + } + auto classes=ggml_soft_max(m,layer(m,n,"cls_embed",output));int64_t count=a.at("num_classes").get(); + classes=ggml_cont(m,ggml_view_2d(m,classes,count,classes->ne[1],classes->nb[1],0)); + auto embed=layer(m,n,"mask_embed",output); + auto pixels=ggml_reshape_2d(m,ggml_cont(m,ggml_permute(m,feature,1,2,0,3)),feature->ne[2],feature->ne[0]*feature->ne[1]); + auto masks=ggml_mul_mat(m,embed,pixels); + masks=ggml_cont(m,ggml_transpose(m,masks));masks=ggml_reshape_4d(m,masks,feature->ne[0],feature->ne[1],embed->ne[1],1); + masks=ggml_sigmoid(m,resize(m,masks,a.at("native_input_w").get(),a.at("native_input_h").get(),true)); + auto spatial=masks->ne[0]*masks->ne[1];masks=ggml_reshape_2d(m,masks,spatial,masks->ne[2]); + auto scores=ggml_mul_mat(m,ggml_cont(m,ggml_transpose(m,masks)),ggml_cont(m,ggml_transpose(m,classes))); + return {ggml_reshape_4d(m,scores,a.at("native_input_w").get(),a.at("native_input_h").get(),count,1)}; + } + if(type=="KernelUpdator") { + auto update=inputs.at(0),input=inputs.at(1);int width=a.at("in_channels").get(); + auto params=layer(m,n,"dynamic_layer",ggml_reshape_2d(m,update,width,ggml_nelements(update)/width)); + auto features=layer(m,n,"input_layer",ggml_reshape_2d(m,input,width,ggml_nelements(input)/width)); + int64_t channels=a.at("feat_channels").get(); + auto part=[&](tensor t,int index){return ggml_cont(m,ggml_view_3d(m,t,channels,t->ne[1],t->ne[2],t->nb[1],t->nb[2],size_t(index*channels)*t->nb[0]));}; + auto gate=ggml_mul(m,part(features,0),part(params,0)); + if(a.value("gate_norm_act",false))gate=layer(m,n,"activation",layer(m,n,"gate_norm",gate)); + auto input_gate=layer(m,n,"input_norm_in",layer(m,n,"input_gate",gate)); + auto update_gate=layer(m,n,"norm_in",layer(m,n,"update_gate",gate)); + if(a.value("gate_sigmoid",true)){input_gate=ggml_sigmoid(m,input_gate);update_gate=ggml_sigmoid(m,update_gate);} + auto param_out=layer(m,n,"norm_out",part(params,1)),input_out=layer(m,n,"input_norm_out",part(features,1)); + if(a.value("activate_out",false)){param_out=layer(m,n,"activation",param_out);input_out=layer(m,n,"activation",input_out);} + auto y=ggml_add(m,ggml_mul(m,update_gate,param_out),ggml_mul(m,input_gate,input_out)); + y=layer(m,n,"activation",layer(m,n,"fc_norm",layer(m,n,"fc_layer",y))); + return {ggml_reshape_2d(m,y,input->ne[0],input->ne[1])}; + } + if(type=="KernelUpdateHead") { + if(a.at("conv_kernel_size")!=1)throw std::runtime_error("K-Net kernel update requires 1x1 kernels"); + auto x=inputs.at(0),proposal=inputs.at(1),mask=inputs.at(2); + if(child(n,"feat_transform").at("type")!="Identity")x=layer(m,n,"feat_transform",x); + auto gather=resize(m,mask,x->ne[0],x->ne[1],true); + auto logits=ggml_cont(m,ggml_permute(m,gather,1,2,0,3));logits=ggml_soft_max(m,logits); + logits=ggml_reshape_2d(m,ggml_cont(m,ggml_permute(m,logits,2,0,1,3)),x->ne[0]*x->ne[1],mask->ne[2]); + auto features=ggml_mul_mat(m,ggml_reshape_2d(m,x,x->ne[0]*x->ne[1],x->ne[2]),logits); + auto object=run(m,child(n,"kernel_update_conv"),{features,proposal}).at(0); + object=layer(m,n,"attention_norm",layer(m,n,"attention",object)); + if(a.value("with_ffn",true))object=layer(m,n,"ffn_norm",layer(m,n,"ffn",object)); + auto kernel=layer(m,n,"fc_mask",layer(m,n,"mask_fcs",object)); + int scale=a.at("mask_transform_stride").get(); + if(scale==2&&a.at("feat_gather_stride")==1)x=resize(m,x,x->ne[0]/2,x->ne[1]/2,true); + kernel=ggml_cast(m,ggml_reshape_4d(m,kernel,1,1,kernel->ne[0],kernel->ne[1]),GGML_TYPE_F16); + auto prediction=ggml_conv_2d(m,kernel,x,1,1,0,0,1,1); + if(scale==2)prediction=resize(m,prediction,prediction->ne[0]*2,prediction->ne[1]*2,true); + return {prediction,object}; + } + if(type=="IterativeDecodeHead") { + auto head=child(n,"kernel_generate_head");auto classifier=child(head,"conv_seg"); + head["children"]["conv_seg"]={{"type","Identity"},{"prefix",""},{"attrs",Json::object()},{"children",Json::object()}}; + auto feature=run(m,head,inputs).at(0),prediction=one(m,classifier,feature),kernel=parameter(m,classifier,"weight"); + kernel=ggml_reshape_2d(m,kernel,kernel->ne[2],kernel->ne[3]);auto const& updates=child(n,"kernel_update_head"); + for(auto const& key:updates.at("order")) { + auto result=run(m,child(updates,key.get()),{feature,kernel,prediction});prediction=result.at(0);kernel=result.at(1); + }return {prediction}; + } + if(type=="DyHead")return run(m,child(n,"dyhead_blocks"),inputs); + if(type=="DyDCNv2") { + auto y=run(m,child(n,"conv"),inputs).at(0);return {a.value("with_norm",false)?layer(m,n,"norm",y):y}; + } + if(type=="DyHeadBlock") { + Tensors out; + for(size_t i=0;ine[0],packed->ne[1],18,packed->ne[3],packed->nb[1],packed->nb[2],packed->nb[3],0)); + auto mask=ggml_sigmoid(m,ggml_cont(m,ggml_view_4d(m,packed,packed->ne[0],packed->ne[1],9,packed->ne[3],packed->nb[1],packed->nb[2],packed->nb[3],18*packed->nb[2]))); + auto branch=[&](char const* name,tensor input,bool up) { + auto y=run(m,child(n,name),{input,offset,mask}).at(0); + if(up)y=resize(m,y,x->ne[0],x->ne[1],true,true); + return ggml_mul(m,y,layer(m,n,"scale_attn_module",y)); + }; + auto sum=branch("spatial_conv_mid",x,false);int count=1; + if(i>0){sum=ggml_add(m,sum,branch("spatial_conv_low",inputs[i-1],false));++count;} + if(i+1ne[2]; + auto part=[&](int index){return ggml_cont(m,ggml_view_4d(m,coeff,1,1,channels,x->ne[3],coeff->nb[1],coeff->nb[2],coeff->nb[3],size_t(index*channels)*coeff->nb[2]));}; + auto first=ggml_add(m,ggml_mul(m,x,ggml_scale_bias(m,part(0),2,1)),part(1)); + auto second=ggml_add(m,ggml_mul(m,x,ggml_scale(m,part(2),2)),part(3)); + return {ggml_add(m,second,ggml_relu(m,ggml_sub(m,first,second)))}; + } + if(type=="NMF2D") { + auto feature=inputs.at(0);int groups=a.at("S").get(),rank=a.at("R").get(); + if(feature->ne[3]!=1||feature->ne[2]%groups)throw std::runtime_error("NMF requires batch one and divisible channels"); + auto x=ggml_reshape_3d(m,feature,feature->ne[0]*feature->ne[1],feature->ne[2]/groups,groups); + auto bases=parameter(m,n,"visp_bases"); + if(bases->ne[0]!=rank||bases->ne[1]!=x->ne[1]||bases->ne[2]!=groups)throw std::runtime_error("NMF fixed basis shape mismatch"); + auto transpose=[&](tensor t){return ggml_cont(m,ggml_transpose(m,t));}; + auto numerator_coef=[&](tensor b){return transpose(ggml_mul_mat(m,transpose(x),transpose(b)));}; + auto coefficient=ggml_soft_max(m,ggml_scale(m,numerator_coef(bases),a.at("inv_t").get())); + auto update_coef=[&]() { + auto gram=ggml_mul_mat(m,transpose(bases),transpose(bases)); + auto denominator=ggml_mul_mat(m,gram,coefficient); + coefficient=ggml_div(m,ggml_mul(m,coefficient,numerator_coef(bases)),ggml_scale_bias(m,denominator,1,1e-6f)); + }; + for(int i=0;i();++i) { + update_coef(); + auto numerator=transpose(ggml_mul_mat(m,x,transpose(coefficient))); + auto gram=ggml_mul_mat(m,transpose(coefficient),transpose(coefficient)); + auto denominator=ggml_mul_mat(m,gram,bases); + bases=ggml_div(m,ggml_mul(m,bases,numerator),ggml_scale_bias(m,denominator,1,1e-6f)); + } + update_coef();auto y=transpose(ggml_mul_mat(m,bases,coefficient)); + return {ggml_reshape_4d(m,y,feature->ne[0],feature->ne[1],feature->ne[2],1)}; + } + if(type=="Hamburger") { + auto x=inputs.at(0),y=ggml_relu(m,layer(m,n,"ham_in",x)); + return {ggml_relu(m,ggml_add(m,x,layer(m,n,"ham_out",layer(m,n,"ham",y))))}; + } + if(type=="LightHamHead") { + auto features=transform(m,a,inputs); + for(size_t i=1;ine[0],features[0]->ne[1],true,a.value("align_corners",false)); + return {layer(m,n,"conv_seg",layer(m,n,"align",layer(m,n,"hamburger",layer(m,n,"squeeze",cat(m,features)))))}; + } + if(type=="ConcatCell") { + auto x=layer(m,n,"input1_conv",inputs.at(0)),y=layer(m,n,"input2_conv",inputs.at(1)); + auto target=std::make_pair(x->ne[1],x->ne[0])>std::make_pair(y->ne[1],y->ne[0])?x:y; + auto scale=[&](tensor z) { + if(z->ne[0]==target->ne[0]&&z->ne[1]==target->ne[1])return z; + if(std::make_pair(z->ne[1],z->ne[0])ne[1],target->ne[0])) + return resize(m,z,target->ne[0],target->ne[1],a.at("upsample_mode")=="bilinear"); + int kw=int((z->ne[0]+target->ne[0]-1)/target->ne[0]),kh=int((z->ne[1]+target->ne[1]-1)/target->ne[1]); + int pw=int(target->ne[0]*kw-z->ne[0]),ph=int(target->ne[1]*kh-z->ne[1]); + if(pw||ph)z=ggml_pad_ext(m,z,pw/2,pw-pw/2,ph/2,ph-ph/2,0,0,0,0); + return ggml_pool_2d(m,z,GGML_OP_POOL_MAX,kw,kh,kw,kh,0,0); + }; + x=cat(m,{scale(x),scale(y)});return {a.value("with_out_conv",true)?layer(m,n,"out_conv",x):x}; + } + if(type=="CARAFEPack") { + auto x=inputs.at(0);int scale=a.at("scale_factor").get(),kernel=a.at("up_kernel").get(),groups=a.at("up_group").get(); + auto mask=layer(m,n,"content_encoder",layer(m,n,"channel_compressor",x)); + Json shuffle={{"type","PixelShuffle"},{"attrs",{{"upscale_factor",scale}}},{"prefix",""},{"children",Json::object()}}; + mask=one(m,shuffle,mask);auto ow=mask->ne[0],oh=mask->ne[1];int area=kernel*kernel; + mask=ggml_reshape_4d(m,ggml_cont(m,ggml_permute(m,mask,1,2,0,3)),area,groups,ow*oh,x->ne[3]); + mask=ggml_soft_max(m,mask);mask=ggml_cont(m,ggml_permute(m,mask,1,2,0,3)); + mask=ggml_reshape_4d(m,mask,ow,oh,area*groups,x->ne[3]); + if(x->ne[2]%groups)throw std::runtime_error("CARAFE group/channel mismatch"); + int half=kernel/2;auto padded=ggml_pad_ext(m,x,half,half,half,half,0,0,0,0);Tensors out; + for(int g=0;gne[2]/groups; + for(int h=0;hne[0],x->ne[1],channels,x->ne[3],padded->nb[1],padded->nb[2],padded->nb[3],size_t(w)*padded->nb[0]+size_t(h)*padded->nb[1]+size_t(g*channels)*padded->nb[2])); + feature=resize(m,feature,ow,oh,false); + auto weight=ggml_view_4d(m,mask,ow,oh,1,x->ne[3],mask->nb[1],mask->nb[2],mask->nb[3],size_t(g*area+h*kernel+w)*mask->nb[2]); + auto y=ggml_mul(m,feature,weight);total=total?ggml_add(m,total,y):y; + }out.push_back(total); + }return {cat(m,out)}; + } + if(type=="FPN_CARAFE") { + Tensors laterals;auto const& convs=child(n,"lateral_convs");int i=0; + for(auto const& key:convs.at("order")) { + auto x=i<=a.at("backbone_end_level").get()-a.at("start_level").get()?inputs.at(std::min(i+a.at("start_level").get(),int(inputs.size())-1)):laterals.back(); + laterals.push_back(layer(m,convs,key.get(),x));++i; + } + for(int level=int(laterals.size())-1;level>0;--level) { + auto x=laterals[level];if(!a.at("upsample").is_null())x=layer(m,child(n,"upsample_modules"),std::to_string(level-1),x); + auto target=laterals[level-1]; + if(x->ne[0]ne[0]||x->ne[1]ne[1])throw std::runtime_error("CARAFE top-down output too small"); + x=ggml_cont(m,ggml_view_4d(m,x,target->ne[0],target->ne[1],x->ne[2],x->ne[3],x->nb[1],x->nb[2],x->nb[3],0)); + laterals[level-1]=ggml_add(m,target,x); + } + Tensors out;i=0;auto const& outputs=child(n,"fpn_convs"); + for(auto const& key:outputs.at("order"))out.push_back(layer(m,outputs,key.get(),laterals.at(i++))); + return out; + } + if(type=="UpInterpolationConv") { + auto x=inputs.at(0);int scale=a.at("scale_factor").get();auto align=a.value("align_corners",Json{}); + return {layer(m,n,"conv",resize(m,x,x->ne[0]*scale,x->ne[1]*scale,a.at("mode")=="bilinear",align.is_boolean()&&align.get()))}; + } + if(type=="LastConv")return {layer(m,n,"conv_out",inputs.back())}; + if(type=="FPG") { + Tensors feats;auto const& lateral=child(n,"lateral_convs");int level=a.at("start_level").get(); + for(auto const& key:lateral.at("order"))feats.push_back(layer(m,lateral,key.get(),inputs.at(level++))); + auto const& extras=child(n,"extra_downsamples"); + for(auto const& key:extras.at("order"))feats.push_back(layer(m,extras,key.get(),feats.back())); + std::vector outs{feats};int levels=a.at("num_outs").get(); + for(int i=0;i();++i) { + Tensors next;bool reverse=a.at("paths").at(i)=="td"; + auto const& stage=child(child(n,"fpn_transitions"),std::to_string(i)); + for(int j=0;j>(); + if(std::find(skip.begin(),skip.end(),i)!=skip.end()){next.push_back(outs.back().at(j));continue;} + int lvl=reverse?levels-j-1:j;auto const& trans=child(stage,std::to_string(lvl));tensor y=nullptr; + auto add=[&](char const* key,tensor x) { + auto const& node=child(trans,key);if(node.at("type")=="Identity")return; + auto t=one(m,node,x);y=y?ggml_add(m,y,t):t; + }; + auto same=reverse?"same_down":"same_up"; + if(child(trans,same).at("type")!="Identity")add(same,next.at(next.size()-1)); + add("across_lateral",outs.back().at(lvl)); + if(lvl>0)add("across_up",outs.back().at(lvl-1)); + if(lvl(); + for(auto const& key:adapts.at("order"))feats.push_back(layer(m,adapts,key.get(),inputs.at(i++))); + auto const& cells=child(n,"fpn"); + for(auto const& key:cells.at("order")) { + auto name=key.get();feats.push_back(run(m,child(cells,name),{feats.at(name.at(1)-'0'),feats.at(name.at(2)-'0')}).at(0)); + } + Tensors out;int level=1; + for(int index:{9,8,7}) { + auto x=feats.at(index);x=ggml_add(m,x,resize(m,feats.at(5),x->ne[0],x->ne[1],true)); + auto target=inputs.at(level++);out.push_back(resize(m,x,target->ne[0],target->ne[1],true)); + } + auto const& extras=child(n,"extra_downsamples"); + for(auto const& key:extras.at("order"))out.push_back(layer(m,extras,key.get(),out.back())); + return out; + } + if(type=="ISAHead") { + auto x=layer(m,n,"in_conv",transform(m,a,inputs).at(0)),residual=x; + int64_t lw=a.at("down_factor").at(1).get(),lh=a.at("down_factor").at(0).get(),width=x->ne[0],height=x->ne[1],channels=x->ne[2]; + if(x->ne[3]!=1)throw std::runtime_error("ISA requires batch one"); + int64_t gw=(width+lw-1)/lw,gh=(height+lh-1)/lh,pw=gw*lw-width,ph=gh*lh-height; + if(pw||ph)x=ggml_pad_ext(m,x,int(pw/2),int(pw-pw/2),int(ph/2),int(ph-ph/2),0,0,0,0); + x=ggml_reshape_4d(m,x,lw,gw,lh*gh,channels); + x=ggml_cont(m,ggml_permute(m,x,1,0,2,3));x=ggml_reshape_4d(m,x,gw,lw*lh,gh,channels); + x=ggml_cont(m,ggml_permute(m,x,0,3,1,2));x=layer(m,n,"global_relation",x); + x=ggml_reshape_4d(m,x,gw,gh*channels,lw,lh);x=ggml_cont(m,ggml_permute(m,x,2,1,0,3)); + x=ggml_reshape_4d(m,x,lw,gh,channels*gw,lh);x=ggml_cont(m,ggml_permute(m,x,0,3,2,1)); + x=layer(m,n,"local_relation",ggml_reshape_4d(m,x,lw,lh,channels,gw*gh)); + x=ggml_reshape_4d(m,x,lw,lh*channels,gw,gh);x=ggml_cont(m,ggml_permute(m,x,0,2,1,3)); + x=ggml_reshape_4d(m,x,lw*gw,lh,channels,gh);x=ggml_cont(m,ggml_permute(m,x,0,1,3,2)); + x=ggml_reshape_4d(m,x,lw*gw,lh*gh,channels,1); + if(pw||ph)x=ggml_cont(m,ggml_view_4d(m,x,width,height,channels,1,x->nb[1],x->nb[2],x->nb[3],size_t(pw/2)*x->nb[0]+size_t(ph/2)*x->nb[1])); + return {layer(m,n,"conv_seg",layer(m,n,"out_conv",cat(m,{x,residual})))}; + } + if(type=="CrissCrossAttention") { + auto x=inputs.at(0),query=layer(m,n,"query_conv",x),key=layer(m,n,"key_conv",x),value=layer(m,n,"value_conv",x); + auto width=x->ne[0],height=x->ne[1];if(x->ne[3]!=1)throw std::runtime_error("criss-cross attention requires batch one"); + auto vertical=[&](tensor y){return ggml_reshape_3d(m,ggml_cont(m,ggml_permute(m,y,2,1,0,3)),y->ne[2],height,width);}; + auto horizontal=[&](tensor y){return ggml_reshape_3d(m,ggml_cont(m,ggml_permute(m,y,1,2,0,3)),y->ne[2],width,height);}; + auto energy_h=ggml_mul_mat(m,vertical(key),vertical(query)); + auto indices=ggml_arange(m,0,float(height),1),lhs=ggml_reshape_2d(m,indices,height,1),rhs=ggml_reshape_2d(m,indices,1,height); + auto square=ggml_new_tensor_2d(m,GGML_TYPE_F32,height,height); + auto diagonal=ggml_scale_bias(m,ggml_step(m,ggml_abs(m,ggml_sub(m,ggml_repeat(m,lhs,square),rhs))),1e9f,-1e9f); + energy_h=ggml_add(m,energy_h,diagonal);energy_h=ggml_cont(m,ggml_permute(m,energy_h,0,2,1,3)); + auto energy_w=ggml_mul_mat(m,horizontal(key),horizontal(query)); + auto attention=ggml_soft_max(m,cat(m,{energy_h,energy_w},0)); + auto part=[&](int64_t start,int64_t count){return ggml_cont(m,ggml_view_3d(m,attention,count,width,height,attention->nb[1],attention->nb[2],size_t(start)*attention->nb[0]));}; + auto ah=ggml_cont(m,ggml_permute(m,part(0,height),0,2,1,3)),aw=part(height,width); + auto vh=ggml_reshape_3d(m,ggml_cont(m,ggml_permute(m,value,2,0,1,3)),height,value->ne[2],width); + auto vw=ggml_reshape_3d(m,ggml_cont(m,ggml_permute(m,value,0,2,1,3)),width,value->ne[2],height); + auto oh=ggml_cont(m,ggml_permute(m,ggml_mul_mat(m,vh,ah),2,1,0,3)); + auto ow=ggml_cont(m,ggml_permute(m,ggml_mul_mat(m,vw,aw),2,0,1,3)); + return {ggml_add(m,x,layer(m,n,"gamma",ggml_add(m,oh,ow)))}; + } + if(type=="CCHead") { + auto x=transform(m,a,inputs).at(0);auto const& convs=child(n,"convs");auto y=layer(m,convs,"0",x); + for(int i=0;i();++i)y=layer(m,n,"cca",y); + y=layer(m,convs,"1",y);if(a.value("concat_input",false))y=layer(m,n,"conv_cat",cat(m,{x,y}));return {layer(m,n,"conv_seg",y)}; + } + if(type=="SegformerHead") { + auto xs=transform(m,a,inputs);Tensors out;auto const& convs=child(n,"convs"); + bool bilinear=a.value("interpolate_mode",std::string{"bilinear"})=="bilinear"; + for(size_t i=0;ine[0],xs[0]->ne[1],bilinear,a.value("align_corners",false))); + return {layer(m,n,"conv_seg",layer(m,n,"fusion_conv",cat(m,out)))}; + } + if(type=="BasePIDHead")return {layer(m,n,"act",layer(m,n,"norm",layer(m,n,"conv",inputs.at(0))))}; + if(type=="PIDHead")return {layer(m,n,"conv_seg",layer(m,n,"i_head",inputs.at(0)))}; + if(type=="DDRHead")return {layer(m,n,"conv_seg",layer(m,n,"head",inputs.at(0)))}; + if(type=="DAPPM"||type=="PAPPM") { + auto x=inputs.at(0);auto const& scales=child(n,"scales");auto base=layer(m,scales,"0",x);Tensors branches; + bool bilinear=a.value("unsample_mode",std::string{"bilinear"})=="bilinear"; + if(type=="DAPPM")branches.push_back(base); + for(size_t i=1;ine[0],x->ne[1],bilinear); + y=ggml_add(m,y,type=="DAPPM"?branches.back():base); + if(type=="DAPPM")y=layer(m,child(n,"processes"),std::to_string(i-1),y);branches.push_back(y); + } + auto merged=type=="DAPPM"?cat(m,branches):cat(m,{base,layer(m,n,"processes",cat(m,branches))}); + return {ggml_add(m,layer(m,n,"compression",merged),layer(m,n,"shortcut",x))}; + } + if(type=="ReassembleBlocks") { + Tensors out;auto mode=a.at("readout_type").get(); + if(inputs.size()%2)throw std::runtime_error("DPT requires feature/class-token pairs"); + for(size_t i=0;ine[0],height=x->ne[1],channels=x->ne[2];auto key=std::to_string(i); + if(mode!="ignore") { + auto t=ggml_reshape_3d(m,ggml_cont(m,ggml_permute(m,x,1,2,0,3)),channels,width*height,x->ne[3]); + cls=ggml_reshape_3d(m,cls,channels,1,x->ne[3]); + if(mode=="project")t=layer(m,child(n,"readout_projects"),key,cat(m,{t,ggml_repeat(m,cls,t)},0)); + else if(mode=="add")t=ggml_add(m,t,cls);else throw std::runtime_error("DPT readout unsupported"); + x=ggml_cont(m,ggml_permute(m,ggml_reshape_4d(m,t,channels,width,height,x->ne[3]),2,0,1,3)); + } + out.push_back(layer(m,child(n,"resize_layers"),key,layer(m,child(n,"projects"),key,x))); + }return out; + } + if(type=="PreActResidualConvUnit") {auto x=inputs.at(0);return {ggml_add(m,x,layer(m,n,"conv2",layer(m,n,"conv1",x)))};} + if(type=="FeatureFusionBlock") { + auto x=inputs.at(0);if(inputs.size()==2) { + auto residual=inputs[1];if(residual->ne[0]!=x->ne[0]||residual->ne[1]!=x->ne[1])residual=resize(m,residual,x->ne[0],x->ne[1],true); + x=ggml_add(m,x,layer(m,n,"res_conv_unit1",residual)); + } + x=layer(m,n,"res_conv_unit2",x);return {layer(m,n,"project",resize(m,x,2*x->ne[0],2*x->ne[1],true,a.value("align_corners",true)))}; + } + if(type=="DPTHead") { + Tensors pairs;for(auto const& index:a.at("in_index")){auto i=index.get();pairs.push_back(inputs.at(2*i));pairs.push_back(inputs.at(2*i+1));} + auto xs=run(m,child(n,"reassemble_blocks"),pairs);auto const& convs=child(n,"convs"),fusion=child(n,"fusion_blocks"); + for(size_t i=0;ine[0],high->ne[1],true,a.value("align_corners",false)); + low=layer(m,n,"conv_low",low);return {ggml_relu(m,ggml_add(m,low,layer(m,n,"conv_high",high))),low}; + } + if(type=="ICNeck") { + auto middle=run(m,child(n,"cff_24"),{inputs.at(2),inputs.at(1)}); + auto high=run(m,child(n,"cff_12"),{middle.at(0),inputs.at(0)});return {middle.at(1),high.at(1),high.at(0)}; + } + if(type=="SETRUPHead") { + auto x=transform(m,a,inputs).at(0);auto width=x->ne[0],height=x->ne[1],channels=x->ne[2]; + auto t=ggml_reshape_3d(m,ggml_cont(m,ggml_permute(m,x,1,2,0,3)),channels,width*height,x->ne[3]); + t=layer(m,n,"norm",t);x=ggml_cont(m,ggml_permute(m,ggml_reshape_4d(m,t,channels,width,height,x->ne[3]),2,0,1,3)); + x=layer(m,n,"up_convs",x);return {layer(m,n,"conv_seg",x)}; + } + if(type=="SegmenterMaskTransformerHead") { + auto x=transform(m,a,inputs).at(0);auto width=x->ne[0],height=x->ne[1],count=width*height,batch=x->ne[3]; + auto t=ggml_reshape_3d(m,ggml_cont(m,ggml_permute(m,x,1,2,0,3)),x->ne[2],count,batch); + t=layer(m,n,"dec_proj",t);auto cls=weights(m,n).weights("cls_emb");int64_t classes=cls->ne[1]; + t=layer(m,n,"decoder_norm",layer(m,n,"layers",ggml_concat(m,t,cls,1)));auto channels=t->ne[0]; + auto patches=ggml_cont(m,ggml_view_3d(m,t,channels,count,batch,t->nb[1],t->nb[2],0)); + cls=ggml_cont(m,ggml_view_3d(m,t,channels,classes,batch,t->nb[1],t->nb[2],size_t(count)*t->nb[1])); + auto normalize=[&](tensor y){auto norm=ggml_clamp(m,ggml_sqrt(m,ggml_sum_rows(m,ggml_sqr(m,y))),1e-12f,INFINITY);return ggml_div(m,y,norm);}; + patches=normalize(layer(m,n,"patch_proj",patches));cls=normalize(layer(m,n,"classes_proj",cls)); + auto mask=layer(m,n,"mask_norm",ggml_mul_mat(m,cls,patches)); + return {ggml_reshape_4d(m,ggml_cont(m,ggml_transpose(m,mask)),width,height,classes,batch)}; + } + if(type=="Encoding") { + auto x=inputs.at(0);auto channels=x->ne[2],spatial=x->ne[0]*x->ne[1];auto wm=weights(m,n); + auto tokens=ggml_cont(m,ggml_transpose(m,ggml_reshape_3d(m,x,spatial,channels,x->ne[3]))); + auto codes=wm.weights("codewords"),scale=wm.weights("scale"); + auto input_norm=ggml_sum_rows(m,ggml_sqr(m,tokens)); + auto code_norm=ggml_cont(m,ggml_transpose(m,ggml_sum_rows(m,ggml_sqr(m,codes)))); + auto scores=ggml_mul_mat(m,codes,tokens); + scores=ggml_add(m,ggml_add(m,ggml_scale(m,scores,-2),input_norm),code_norm); + auto assignment=ggml_soft_max(m,ggml_mul(m,scores,scale)); + assignment=ggml_cont(m,ggml_transpose(m,assignment)); + auto sums=ggml_sum_rows(m,assignment); + auto aggregate=ggml_mul_mat(m,ggml_cont(m,ggml_transpose(m,tokens)),assignment); + auto weighted=ggml_mul(m,codes,sums); + return {ggml_sub(m,aggregate,weighted)}; + } + if(type=="EncModule") { + auto x=inputs.at(0),encoded=layer(m,n,"encoding_project",x);auto const& blocks=child(n,"encoding"); + for(auto const& key:blocks.at("order")) { + auto const& block=child(blocks,key.get());auto kind=block.at("type").get(); + if(kind=="SyncBatchNorm"||kind=="BatchNorm1d") { + auto wm=weights(m,block);auto gamma=wm.weights("weight"),beta=wm.weights("bias"); + encoded=ggml_add(m,ggml_mul(m,encoded,ggml_reshape_3d(m,gamma,1,gamma->ne[0],1)),ggml_reshape_3d(m,beta,1,beta->ne[0],1)); + } else encoded=one(m,block,encoded); + } + auto feature=ggml_cont(m,ggml_transpose(m,ggml_mean(m,ggml_cont(m,ggml_transpose(m,encoded))))); + feature=ggml_reshape_2d(m,feature,x->ne[2],x->ne[3]); + auto gamma=layer(m,n,"fc",feature);gamma=ggml_reshape_4d(m,gamma,1,1,x->ne[2],x->ne[3]); + return {feature,ggml_relu(m,ggml_add(m,x,ggml_mul(m,x,gamma)))}; + } + if(type=="EncHead") { + auto xs=transform(m,a,inputs);auto x=layer(m,n,"bottleneck",xs.back()); + if(a.value("add_lateral",false)) { + Tensors branches{x};auto const& lateral=child(n,"lateral_convs"); + for(size_t i=0;i+1ne[0],x->ne[1],true,a.value("align_corners",false))); + x=layer(m,n,"fusion",cat(m,branches)); + } + auto encoded=run(m,child(n,"enc_module"),{x});Tensors out{layer(m,n,"conv_seg",encoded.at(1))}; + if(a.value("use_se_loss",false))out.push_back(layer(m,n,"se_layer",encoded.at(0)));return out; + } + if(type=="PPMConcat") { + auto x=inputs.at(0);Tensors out; + for(auto const& key:n.at("order")) {auto y=layer(m,n,key.get(),x); + out.push_back(ggml_reshape_4d(m,ggml_cont(m,y),y->ne[0]*y->ne[1],1,y->ne[2],y->ne[3]));} + return {cat(m,out,0)}; + } + if(type=="AFNB"||type=="APNB") { + auto high=inputs.at(type=="AFNB"?1:0),low=inputs.at(0);tensor context=nullptr; + auto const& stages=child(n,"stages"); + for(auto const& key:stages.at("order")) {auto y=run(m,child(stages,key.get()),{high,low}).at(0);context=context?ggml_add(m,context,y):y;} + return {layer(m,n,"bottleneck",cat(m,{context,high}))}; + } + if(type=="ANNHead") { + auto xs=transform(m,a,inputs);auto x=run(m,child(n,"fusion"),xs).at(0); + return {layer(m,n,"conv_seg",layer(m,n,"context",layer(m,n,"bottleneck",x)))}; + } + if(type=="ACM") { + auto original=inputs.at(0);int scale=a.at("pool_scale").get(); + auto pooled=layer(m,n,"pooled_redu_conv",adaptive_pool(m,original,scale,scale)); + auto x=layer(m,n,"input_redu_conv",original); + auto context=layer(m,n,"global_info",adaptive_pool(m,x,1,1)); + auto affinity=ggml_sigmoid(m,layer(m,n,"gla",ggml_add(m,x,context))); + affinity=ggml_cont(m,ggml_transpose(m,ggml_reshape_3d(m,affinity,x->ne[0]*x->ne[1],scale*scale,x->ne[3]))); + auto values=ggml_reshape_3d(m,ggml_cont(m,pooled),scale*scale,pooled->ne[2],pooled->ne[3]); + auto y=ggml_cont(m,ggml_transpose(m,ggml_mul_mat(m,values,affinity))); + y=ggml_reshape_4d(m,y,x->ne[0],x->ne[1],x->ne[2],x->ne[3]); + y=ggml_relu(m,ggml_add(m,layer(m,n,"residual_conv",y),x)); + if(a.value("fusion",false))y=layer(m,n,"fusion_conv",y);return {y}; + } + if(type=="DCM") { + auto original=inputs.at(0);int size=a.at("filter_size").get(); + auto kernel=layer(m,n,"filter_gen_conv",adaptive_pool(m,original,size,size)); + auto x=layer(m,n,"input_redu_conv",original); + if(x->ne[3]!=1)throw std::runtime_error("dynamic convolution requires batch one"); + kernel=ggml_reshape_4d(m,ggml_cont(m,kernel),size,size,1,x->ne[2]); + int pad=(size-1)/2,extra=(size-1)%2; + x=ggml_pad_ext(m,x,pad+extra,pad,pad+extra,pad,0,0,0,0); + auto y=conv_2d_wt_dw(m,x,kernel,nullptr,1,0,1); + if(c.contains("norm"))y=layer(m,n,"norm",y);y=layer(m,n,"activate",y); + if(a.value("fusion",false))y=layer(m,n,"fusion_conv",y);return {y}; + } + if(type=="APCHead"||type=="DMHead") { + auto x=transform(m,a,inputs).at(0);Tensors branches{x}; + auto const& blocks=child(n,type=="APCHead"?"acm_modules":"dcm_modules"); + for(auto const& key:blocks.at("order"))branches.push_back(layer(m,blocks,key.get(),x)); + return {layer(m,n,"conv_seg",layer(m,n,"bottleneck",cat(m,branches)))}; + } + if(type=="PAM") { + auto base=n;base["type"]="SelfAttentionBlock";auto x=inputs.at(0); + auto y=heads(m,base,{x,x}).at(0);return {ggml_add(m,x,layer(m,n,"gamma",y))}; + } + if(type=="CAM") { + auto x=inputs.at(0);auto features=ggml_reshape_3d(m,ggml_cont(m,x),x->ne[0]*x->ne[1],x->ne[2],x->ne[3]); + auto energy=ggml_mul_mat(m,features,features); + auto maximum=ggml_pool_1d(m,energy,GGML_OP_POOL_MAX,int(energy->ne[0]),int(energy->ne[0]),0); + auto attention=ggml_soft_max(m,ggml_sub(m,ggml_repeat(m,maximum,energy),energy)); + auto y=ggml_mul_mat(m,ggml_cont(m,ggml_transpose(m,features)),attention); + y=ggml_reshape_4d(m,y,x->ne[0],x->ne[1],x->ne[2],x->ne[3]); + return {ggml_add(m,x,layer(m,n,"gamma",y))}; + } + if(type=="DAHead") { + auto x=transform(m,a,inputs).at(0); + auto pam=layer(m,n,"pam_out_conv",layer(m,n,"pam",layer(m,n,"pam_in_conv",x))); + auto cam=layer(m,n,"cam_out_conv",layer(m,n,"cam",layer(m,n,"cam_in_conv",x))); + return {layer(m,n,"conv_seg",ggml_add(m,pam,cam)),layer(m,n,"pam_conv_seg",pam),layer(m,n,"cam_conv_seg",cam)}; + } + if(type=="Feature2Pyramid") { + Tensors out;bool four=c.contains("upsample_4x")&&child(n,"upsample_4x").at("type")!="Identity"; + std::vector ops=four?std::vector{"upsample_4x","upsample_2x","identity","downsample_2x"}: + std::vector{"upsample_2x","identity","downsample_2x","downsample_4x"}; + for(size_t i=0;i();if(xs.size()==1)xs.resize(count,xs[0]);Tensors out; + for(int i=0;i();auto x=xs.at(i); + x=resize(m,x,int64_t(x->ne[0]*scale),int64_t(x->ne[1]*scale),true); + out.push_back(layer(m,child(n,"convs"),std::to_string(i),x)); + } + return out; + } + if(type=="JPU") { + int start=a.at("start_level").get(),end=a.at("backbone_end_level").get();Tensors xs; + auto const& convs=child(n,"conv_layers"); + for(int i=start;ine[0],xs[0]->ne[1],true,a.value("align_corners",false)); + auto x=cat(m,xs);Tensors branches;auto const& dilations=child(n,"dilation_layers"); + for(auto const& key:dilations.at("order"))branches.push_back(layer(m,dilations,key.get(),x)); + Tensors out;for(int i=start;i(),x);out.push_back(x);}return out; + } + if(type=="L2Norm") { + auto x=inputs.at(0),y=ggml_cont(m,ggml_permute(m,x,1,2,0,3)); + auto norm=ggml_scale_bias(m,ggml_sqrt(m,ggml_sum_rows(m,ggml_sqr(m,y))),1,a.value("eps",1e-10f)); + y=ggml_mul(m,ggml_div(m,y,norm),weights(m,n).weights("weight")); + return {ggml_cont(m,ggml_permute(m,y,2,0,1,3))}; + } + if(type=="CTResNetNeck")return {layer(m,n,"deconv_layers",inputs.back())}; + if(type=="BFP") { + int level=a.at("refine_level").get();auto center=inputs.at(level);tensor sum=nullptr; + for(size_t i=0;ine[0]),int(center->ne[1]),true):resize(m,inputs[i],center->ne[0],center->ne[1],false); + sum=sum?ggml_add(m,sum,y):y; + } + auto x=ggml_scale(m,sum,1.f/float(inputs.size())); + if(c.contains("refine"))x=layer(m,n,"refine",x); + Tensors out; + for(size_t i=0;ine[0],inputs[i]->ne[1],false):adaptive_pool(m,x,int(inputs[i]->ne[0]),int(inputs[i]->ne[1]),true); + out.push_back(ggml_add(m,y,inputs[i])); + } + return out; + } + if(type=="ContextBlock") { + auto x=inputs.at(0);tensor context; + if(a.at("pooling_type")=="avg")context=layer(m,n,"avg_pool",x); + else { + auto count=x->ne[0]*x->ne[1];auto mask=layer(m,n,"conv_mask",x); + mask=ggml_soft_max(m,ggml_reshape_3d(m,mask,count,1,x->ne[3])); + auto features=ggml_reshape_3d(m,ggml_cont(m,x),count,x->ne[2],x->ne[3]); + context=ggml_reshape_4d(m,ggml_mul_mat(m,mask,features),1,1,x->ne[2],x->ne[3]); + } + if(c.contains("channel_mul_conv")&&child(n,"channel_mul_conv").at("type")!="Identity")x=ggml_mul(m,x,ggml_sigmoid(m,layer(m,n,"channel_mul_conv",context))); + if(c.contains("channel_add_conv")&&child(n,"channel_add_conv").at("type")!="Identity")x=ggml_add(m,x,layer(m,n,"channel_add_conv",context)); + return {x}; + } + if(type=="NonLocal2d" || type=="DisentangledNonLocal2d") { + auto x=inputs.at(0);auto spatial=x->ne[0]*x->ne[1]; + auto flat=[&](tensor t){return ggml_reshape_3d(m,ggml_cont(m,t),t->ne[0]*t->ne[1],t->ne[2],t->ne[3]);}; + auto mode=a.value("mode",std::string{"embedded_gaussian"}); + if(mode!="gaussian"&&mode!="embedded_gaussian"&&mode!="dot_product")throw std::runtime_error("nonlocal mode unsupported"); + auto q=flat(mode=="gaussian"?x:layer(m,n,"theta",x)); + auto k=flat(mode=="gaussian"&&!a.value("sub_sample",false)?x:layer(m,n,"phi",x)); + if(type=="DisentangledNonLocal2d") {q=ggml_sub(m,q,ggml_mean(m,q));k=ggml_sub(m,k,ggml_mean(m,k));} + auto keys=ggml_cont(m,ggml_transpose(m,k)),queries=ggml_cont(m,ggml_transpose(m,q)); + auto score=ggml_mul_mat(m,keys,queries); + if(mode=="dot_product")score=ggml_scale(m,score,1.f/float(k->ne[0])); + else { + float scale=mode=="embedded_gaussian"&&a.value("use_scale",true)?1.f/std::sqrt(float(k->ne[1])):1.f; + if(type=="DisentangledNonLocal2d")scale/=a.at("temperature").get(); + score=ggml_soft_max(m,ggml_scale(m,score,scale)); + } + auto values=flat(layer(m,n,"g",x));auto y=ggml_mul_mat(m,values,score); + y=ggml_reshape_4d(m,ggml_cont(m,ggml_transpose(m,y)),x->ne[0],x->ne[1],values->ne[1],x->ne[3]); + if(type=="DisentangledNonLocal2d") { + auto mask=ggml_soft_max(m,ggml_reshape_3d(m,layer(m,n,"conv_mask",x),spatial,1,x->ne[3])); + auto unary=ggml_reshape_4d(m,ggml_mul_mat(m,mask,values),1,1,values->ne[1],x->ne[3]); + y=ggml_add(m,y,unary); + } + return {ggml_add(m,x,layer(m,n,"conv_out",y))}; + } + if(type=="NLHead" || type=="DNLHead" || type=="GCHead") { + auto x=transform(m,a,inputs).at(0);auto const& convs=child(n,"convs"); + auto y=layer(m,convs,"0",x); + y=layer(m,n,type=="NLHead"?"nl_block":type=="DNLHead"?"dnl_block":"gc_block",y); + y=layer(m,convs,"1",y); + if(a.value("concat_input",false))y=layer(m,n,"conv_cat",cat(m,{x,y})); + return {layer(m,n,"conv_seg",y)}; + } + if(type=="DetectionBlock") { + auto x=inputs.at(0);for(int i=1;i<=5;++i)x=layer(m,n,"conv"+std::to_string(i),x); + return {x}; + } + if(type=="YOLOV3Neck") { + auto x=layer(m,n,"detect1",inputs.back());Tensors out{x}; + for(size_t i=1;ine[0]*2,tmp->ne[1]*2,false); + x=layer(m,n,"detect"+std::to_string(i+1),cat(m,{tmp,inputs[inputs.size()-1-i]}));out.push_back(x); + } + return out; + } + if(type=="DilatedEncoder") { + auto x=layer(m,n,"lateral_norm",layer(m,n,"lateral_conv",inputs.back())); + x=layer(m,n,"fpn_norm",layer(m,n,"fpn_conv",x));return {layer(m,n,"dilated_encoder_blocks",x)}; + } + if(type=="HRFPN") { + Tensors xs{inputs.at(0)}; + for(size_t i=1;ine[0]*(1LL<ne[1]*(1LL<();++i) { + auto y=i?ggml_pool_2d(m,x,op,1<ne[2]/2;auto key=std::to_string(i); + auto channel=[&](int64_t start){return ggml_cont(m,ggml_view_4d(m,x,x->ne[0],x->ne[1],half,x->ne[3],x->nb[1],x->nb[2],x->nb[3],size_t(start)*x->nb[2]));}; + auto cf=layer(m,child(n,"conv_cls"),key,channel(0)); + auto pf=layer(m,child(n,"conv_pose"),key,channel(half)); + cls.push_back(layer(m,child(n,"out_cls"),key,cf)); + boxes.push_back(layer(m,child(n,"out_bbox"),key,pf)); + vis.push_back(layer(m,child(n,"out_kpt_vis"),key,pf)); + poses.push_back(a.value("pose_vec_channels",0)>0?layer(m,child(n,"out_pose"),key,pf):pf); + } + cls.insert(cls.end(),boxes.begin(),boxes.end());cls.insert(cls.end(),vis.begin(),vis.end());cls.insert(cls.end(),poses.begin(),poses.end()); + return cls; + } + if(type=="CSPRepLayer") { + auto x=inputs.at(0);auto y=layer(m,n,"bottlenecks",layer(m,n,"conv1",x)); + return {layer(m,n,"conv3",ggml_add(m,y,layer(m,n,"conv2",x)))}; + } + if(type=="RepVGGBlock") { + auto x=inputs.at(0);auto y=ggml_add(m,layer(m,n,"branch_3x3",x),layer(m,n,"branch_1x1",x)); + if(c.contains("branch_norm")&&child(n,"branch_norm").at("type")!="Identity")y=ggml_add(m,y,layer(m,n,"branch_norm",x)); + return {layer(m,n,"act",y)}; + } + if(type=="FFN") { + auto x=inputs.at(0);auto y=layer(m,n,"layers",x); + return {a.value("add_identity",true)?ggml_add(m,inputs.size()>1?inputs[1]:x,y):y}; + } + if(type=="MultiheadAttention") { + auto x=inputs.at(0); + if(c.contains("attn")) { + auto q=inputs.size()>1&&inputs[1]?ggml_add(m,x,inputs[1]):x; + auto key=inputs.size()>3&&inputs[3]?inputs[3]:x,value=inputs.size()>4&&inputs[4]?inputs[4]:key; + auto position=inputs.size()>5?inputs[5]:inputs.size()>1?inputs[1]:nullptr; + if(position)key=ggml_add(m,key,position); + auto y=run(m,child(n,"attn"),{q,key,value,inputs.size()>6?inputs[6]:nullptr}).at(0); + return {ggml_add(m,inputs.size()>2?inputs[2]:x,y)}; + } + auto w=weights(m,n);auto kernel=w.weights("in_proj_weight");auto bias=w.find("in_proj_bias"); + int64_t channels=x->ne[0],heads=a.at("num_heads").get(),dim=channels/heads; + auto project=[&](tensor t,int index) { + auto k=ggml_cont(m,ggml_view_2d(m,kernel,channels,channels,kernel->nb[1],size_t(index*channels)*kernel->nb[1])); + auto y=ggml_mul_mat(m,k,t); + if(bias)y=ggml_add(m,y,ggml_view_1d(m,bias,channels,size_t(index*channels)*bias->nb[0])); + y=ggml_reshape_4d(m,y,dim,heads,t->ne[1],t->ne[2]); + return ggml_cont(m,ggml_permute(m,y,0,2,1,3)); + }; + auto q=project(inputs.at(0),0),k=project(inputs.at(1),1),v=project(inputs.at(2),2); + auto score=ggml_scale(m,ggml_mul_mat(m,k,q),1.f/std::sqrt(float(dim))); + if(inputs.size()>3&&inputs[3])score=ggml_add(m,score,inputs[3]);score=ggml_soft_max(m,score); + auto y=ggml_mul_mat(m,ggml_cont(m,ggml_transpose(m,v)),score); + y=ggml_cont(m,ggml_permute(m,y,0,2,1,3)); + y=ggml_reshape_3d(m,y,channels,x->ne[1],x->ne[2]); + return {layer(m,n,"out_proj",y)}; + } + if(type=="DetrTransformerEncoder") { + auto x=inputs.at(0);auto const& layers=child(n,"layers"); + for(auto const& key:layers.at("order"))x=run(m,child(layers,key.get()),{x,inputs.at(1)}).at(0); + return {x}; + } + if(type=="DetrTransformerEncoderLayer") { + auto x=run(m,child(n,"self_attn"),inputs).at(0);auto const& norms=child(n,"norms"); + x=layer(m,norms,"0",x);x=layer(m,n,"ffn",x);return {layer(m,norms,"1",x)}; + } + if(type=="SinePositionalEncoding") { + if(!a.contains("spatial_dim")) { + auto x=inputs.at(0);int features=a.at("num_feats").get(); + if(features%2)throw std::runtime_error("sine position encoding requires even features"); + auto freq=ggml_exp(m,ggml_scale(m,ggml_arange(m,0,float(features/2),1),-2.f*std::log(a.value("temperature",10000.f))/features)); + auto encode=[&](int64_t extent,bool horizontal) { + auto coord=ggml_arange(m,1,float(extent+1),1); + if(a.value("normalize",false))coord=ggml_scale_bias(m,coord,a.value("scale",2.f*float(M_PI))/float(extent+a.value("eps",1e-6f)),a.value("offset",0.f)*a.value("scale",2.f*float(M_PI))/float(extent+a.value("eps",1e-6f))); + coord=ggml_reshape_3d(m,coord,1,horizontal?extent:1,horizontal?1:extent); + auto shape=ggml_new_tensor_3d(m,GGML_TYPE_F32,1,x->ne[0],x->ne[1]);coord=ggml_reshape_2d(m,ggml_repeat(m,coord,shape),1,x->ne[0]*x->ne[1]); + auto full=ggml_new_tensor_2d(m,GGML_TYPE_F32,features/2,x->ne[0]*x->ne[1]);auto angle=ggml_mul(m,ggml_repeat(m,coord,full),freq); + auto sin=ggml_reshape_3d(m,ggml_sin(m,angle),1,features/2,angle->ne[1]),cos=ggml_reshape_3d(m,ggml_cos(m,angle),1,features/2,angle->ne[1]); + return ggml_reshape_2d(m,ggml_concat(m,sin,cos,0),features,angle->ne[1]); + };return {ggml_concat(m,encode(x->ne[1],false),encode(x->ne[0],true),0)}; + } + if(a.at("spatial_dim")!=2)throw std::runtime_error("positional encoding protocol unsupported"); + auto x=inputs.at(0);auto dim=weights(m,n).weights("dim_t"); + auto gx=ggml_reshape_2d(m,ggml_arange(m,0,float(x->ne[0]),1),x->ne[0],1); + auto gy=ggml_reshape_2d(m,ggml_arange(m,0,float(x->ne[1]),1),1,x->ne[1]); + auto shape=ggml_new_tensor_2d(m,GGML_TYPE_F32,x->ne[0],x->ne[1]); + auto encode=[&](tensor grid) { + auto p=ggml_reshape_2d(m,ggml_repeat(m,grid,shape),1,x->ne[0]*x->ne[1]); + auto full=ggml_new_tensor_2d(m,GGML_TYPE_F32,dim->ne[0],x->ne[0]*x->ne[1]); + auto frequency=ggml_div(m,ggml_repeat(m,p,full),dim); + return ggml_concat(m,ggml_cos(m,frequency),ggml_sin(m,frequency),0); + }; + // HybridEncoder transposes [token, feature, h/w] before flattening: + // each token contains all h features followed by all w features. + return {ggml_concat(m,encode(gy),encode(gx),0)}; + } + if(type=="HybridEncoder") { + Tensors features;auto const& proj=child(n,"input_proj"); + for(size_t i=0;i0) { + auto const& encoder=child(n,"encoder");int i=0; + for(auto const& index:a.at("use_encoder_idx")) { + auto x=features.at(index.get());auto position=layer(m,n,"sincos_pos_enc",x); + auto tokens=ggml_cont(m,ggml_transpose(m,ggml_reshape_3d(m,x,x->ne[0]*x->ne[1],x->ne[2],x->ne[3]))); + tokens=run(m,child(encoder,std::to_string(i++)),{tokens,position}).at(0); + features[index.get()]=ggml_reshape_4d(m,ggml_cont(m,ggml_transpose(m,tokens)),x->ne[0],x->ne[1],x->ne[2],x->ne[3]); + } + } + Tensors inner{features.back()}; + for(size_t i=features.size()-1;i>0;--i) { + auto key=std::to_string(features.size()-1-i);auto high=layer(m,child(n,"lateral_convs"),key,inner.front());inner.front()=high; + auto up=resize(m,high,high->ne[0]*2,high->ne[1]*2,false); + inner.insert(inner.begin(),layer(m,child(n,"fpn_blocks"),key,cat(m,{up,features[i-1]}))); + } + Tensors out{inner.front()}; + for(size_t i=0;i+1()));out=selected;} + if(c.contains("projector"))out=run(m,child(n,"projector"),out); + return out; + } + if(type=="FPNHead") { + auto features=transform(m,a,inputs);auto const& scales=child(n,"scale_heads"); + auto y=layer(m,scales,"0",features[0]); + for(size_t i=1;ine[0],y->ne[1],true,a.value("align_corners",false))); + } + return {layer(m,n,"conv_seg",y)}; + } + if(type=="EMAModule") { + auto x=inputs.at(0);auto spatial=x->ne[0]*x->ne[1]; + auto features=ggml_reshape_3d(m,ggml_cont(m,x),spatial,x->ne[2],x->ne[3]); + auto vectors=ggml_cont(m,ggml_transpose(m,features)); + auto bases=weights(m,n).weights("bases");tensor attention=nullptr; + for(int i=0;i();++i) { + attention=ggml_soft_max(m,ggml_mul_mat(m,ggml_cont(m,ggml_transpose(m,bases)),vectors)); + auto normalized=ggml_cont(m,ggml_transpose(m,attention)); + auto divisor=ggml_clamp(m,ggml_sum_rows(m,ggml_abs(m,normalized)),1e-12f,INFINITY); + normalized=ggml_div(m,normalized,divisor); + auto next=ggml_mul_mat(m,features,normalized); + auto length=ggml_sqrt(m,ggml_sum_rows(m,ggml_sqr(m,next))); + next=ggml_div(m,next,ggml_clamp(m,length,1e-12f,INFINITY)); + bases=ggml_cont(m,ggml_transpose(m,next)); + } + auto y=ggml_mul_mat(m,bases,attention); + y=ggml_cont(m,ggml_transpose(m,y)); + return {ggml_reshape_4d(m,y,x->ne[0],x->ne[1],x->ne[2],x->ne[3])}; + } + if(type=="EMAHead") { + auto x=transform(m,a,inputs).at(0);auto identity=layer(m,n,"ema_in_conv",x); + auto y=layer(m,n,"ema_module",layer(m,n,"ema_mid_conv",identity)); + y=layer(m,n,"ema_out_conv",ggml_relu(m,y)); + y=layer(m,n,"bottleneck",ggml_relu(m,ggml_add(m,identity,y))); + if(a.value("concat_input",false))y=layer(m,n,"conv_cat",cat(m,{x,y})); + return {layer(m,n,"conv_seg",y)}; + } + if(type=="SpatialGatherModule") { + auto x=inputs.at(0),probs=inputs.at(1);auto spatial=x->ne[0]*x->ne[1]; + probs=ggml_soft_max(m,ggml_scale(m,ggml_reshape_3d(m,ggml_cont(m,probs),spatial,probs->ne[2],probs->ne[3]),a.value("scale",1.f))); + auto features=ggml_reshape_3d(m,ggml_cont(m,x),spatial,x->ne[2],x->ne[3]); + auto y=ggml_mul_mat(m,probs,features); + return {ggml_reshape_4d(m,y,y->ne[0],1,y->ne[1],y->ne[2])}; + } + if(type=="SelfAttentionBlock" || type=="ObjectAttentionBlock") { + auto original=inputs.at(0), key_input=inputs.size()>1?inputs.at(1):original; + auto q=layer(m,n,"query_project",original), k=layer(m,n,"key_project",key_input), v=layer(m,n,"value_project",key_input); + if(c.contains("query_downsample"))q=layer(m,n,"query_downsample",q); + if(c.contains("key_downsample")) {k=layer(m,n,"key_downsample",k);v=layer(m,n,"key_downsample",v);} + auto qw=q->ne[0],qh=q->ne[1]; + auto tokens=[&](tensor t){return ggml_cont(m,ggml_transpose(m,ggml_reshape_3d(m,ggml_cont(m,t),t->ne[0]*t->ne[1],t->ne[2],t->ne[3])));}; + auto scores=ggml_mul_mat(m,tokens(k),tokens(q)); + if(a.value("matmul_norm",false))scores=ggml_scale(m,scores,1.f/std::sqrt(float(q->ne[2]))); + auto values=ggml_reshape_3d(m,ggml_cont(m,v),v->ne[0]*v->ne[1],v->ne[2],v->ne[3]); + auto y=ggml_cont(m,ggml_transpose(m,ggml_mul_mat(m,values,ggml_soft_max(m,scores)))); + y=ggml_reshape_4d(m,y,qw,qh,v->ne[2],v->ne[3]); + if(c.contains("out_project"))y=layer(m,n,"out_project",y); + if(c.contains("output_project"))y=layer(m,n,"output_project",y); + if(type=="ObjectAttentionBlock")y=layer(m,n,"bottleneck",cat(m,{y,original})); + return {y}; + } + if(type=="OCRHead") { + auto features=inputs;auto previous=features.back();features.pop_back(); + auto x=layer(m,n,"bottleneck",transform(m,a,features).at(0)); + auto context=run(m,child(n,"spatial_gather_module"),{x,previous}).at(0); + auto y=run(m,child(n,"object_context_block"),{x,context}).at(0); + return {layer(m,n,"conv_seg",y)}; + } + if(type=="LRASPPHead") { + auto features=transform(m,a,inputs);auto x=features.back();bool align=a.value("align_corners",false); + auto pooled=layer(m,n,"image_pool",x); + x=ggml_mul(m,layer(m,n,"aspp_conv",x),resize(m,pooled,x->ne[0],x->ne[1],true,align)); + x=layer(m,n,"conv_up_input",x); + for(int i=int(features.size())-2;i>=0;--i) { + x=resize(m,x,features[i]->ne[0],features[i]->ne[1],true,align); + x=layer(m,child(n,"conv_ups"),"conv_up"+std::to_string(i),cat(m,{x,layer(m,child(n,"convs"),"conv"+std::to_string(i),features[i])})); + } + return {layer(m,n,"conv_seg",x)}; + } + if(type=="ChannelMapper") { + Tensors out;auto const& convs=child(n,"convs"); + for(size_t i=0;ine[0]*x->ne[1];int groups=a.at("groups").get(); + affine=ggml_cont(m,ggml_permute(m,affine,1,2,0,3)); + affine=ggml_reshape_4d(m,affine,3,2,spatial*groups,x->ne[3]); + auto regular=ggml_cont(m,ggml_transpose(m,weights(m,n).weights("regular_matrix"))); + auto offset=ggml_mul_mat(m,regular,affine); + offset=ggml_cont(m,ggml_permute(m,offset,1,0,2,3)); + offset=ggml_reshape_4d(m,offset,18*groups,x->ne[0],x->ne[1],x->ne[3]); + offset=ggml_cont(m,ggml_permute(m,offset,2,0,1,3)); + auto y=run(m,child(n,"adapt_conv"),{x,offset}).at(0); + return {layer(m,n,"act",ggml_add(m,layer(m,n,"norm",y),x))}; + } + if(type=="FeatureMapProcessor") { + Tensors out; + auto idx=a.value("select_index",Json{}); + if(idx.is_null())out=inputs; + else if(idx.is_array())for(auto const& i:idx)out.push_back(select(inputs,i)); + else out.push_back(select(inputs,idx)); + bool align=a.value("align_corners",false); + if(a.value("concat",false)) { + for(size_t i=1;ine[0],out[0]->ne[1],true,align); + out={cat(m,out)}; + } + float scale=a.value("scale_factor",1.f); + for(auto& x:out) { + if(a.value("apply_relu",false))x=ggml_relu(m,x); + if(scale!=1.f)x=resize(m,x,int64_t(x->ne[0]*scale),int64_t(x->ne[1]*scale),true,align); + } + return out; + } + if(type=="GlobalAveragePooling") { + Tensors out; + for(auto x:inputs) { + auto y=layer(m,n,"gap",x); + // MMDet ReID may use fixed-kernel pooling with spatial output + // larger than 1x1. Its forward flattens all non-batch dimensions. + out.push_back(ggml_reshape_2d(m,ggml_cont(m,y),y->ne[0]*y->ne[1]*y->ne[2],y->ne[3])); + } + return out; + } + if(type=="Heatmap1DHead") { + auto x=ggml_soft_max(m,layer(m,n,"fc",inputs[0])); + auto coords=ggml_arange(m,0,float(a.at("heatmap_size").get()),1); + return {ggml_sum_rows(m,ggml_mul(m,x,coords))}; + } + if(type=="MultilabelClassificationHead")return {layer(m,n,"fc",inputs[0])}; + if(type=="InternetHead") { + auto x=inputs.back(); + auto heat=cat(m,{layer(m,n,"right_hand_head",x),layer(m,n,"left_hand_head",x)},3); + x=layer(m,n,"neck",x); + return {heat,layer(m,n,"root_head",x),layer(m,n,"hand_type_head",x)}; + } + if(type=="CPMHead") { + Tensors out; + auto const& dec=child(n,"multi_deconv_layers"), final=child(n,"multi_final_layers"); + for(size_t i=0;i0;--i) { + auto key=std::to_string(inputs.size()-1-i); + auto high=layer(m,reduce,key,inner.front());inner.front()=high; + auto up=layer(m,n,"upsample",high); + inner.insert(inner.begin(),layer(m,top,key,cat(m,{up,inputs[i-1]}))); + } + Tensors out{inner.front()}; + for(size_t i=0;i+1(); + result.push_back(c.contains("out_convs")?layer(m,child(n,"out_convs"),std::to_string(i),out.at(i)):out.at(i)); + } + return result; + } + if(type=="Scale") return {ggml_mul(m,inputs[0],weights(m,n).weights("scale"))}; + if(type=="ScaleNorm") { + auto x=inputs[0]; + auto norm=ggml_sqrt(m,ggml_sum_rows(m,ggml_sqr(m,x))); + norm=ggml_clamp(m,ggml_scale(m,norm,a.at("scale").get()),a.at("eps").get(),INFINITY); + return {ggml_mul(m,ggml_div(m,x,norm),weights(m,n).weights("g"))}; + } + if(type=="RTMCCBlock") { + if(a.value("attn_type",std::string{"self-attn"})!="self-attn" || a.value("pos_enc",false) || a.value("use_rel_bias",false)) + throw std::runtime_error("RTMCC positional/cross attention not implemented"); + auto x=inputs[0];auto w=weights(m,n); + auto uv=layer(m,n,"act_fn",layer(m,n,"uv",layer(m,n,"ln",x))); + int e=a.at("e").get(), s=a.at("s").get(); + auto part=[&](int start,int size) {return ggml_cont(m,ggml_view_2d(m,uv,size,uv->ne[1],uv->nb[1],size_t(start)*uv->nb[0]));}; + auto u=part(0,e), v=part(e,e), base=part(2*e,s); + auto gamma=w.weights("gamma"), beta=w.weights("beta"); + auto affine=[&](int i) { + auto g=ggml_view_1d(m,gamma,s,size_t(i)*gamma->nb[1]); + auto b=ggml_view_1d(m,beta,s,size_t(i)*beta->nb[1]); + return ggml_add(m,ggml_mul(m,base,g),b); + }; + auto q=affine(0), k=affine(1); + auto scores=ggml_sqr(m,ggml_relu(m,ggml_scale(m,ggml_mul_mat(m,k,q),1.f/a.at("sqrt_s").get()))); + auto values=ggml_mul_mat(m,ggml_cont(m,ggml_transpose(m,v)),scores); + auto y=layer(m,n,"o",ggml_mul(m,u,values)); + if(a.value("shortcut",false)) y=ggml_add(m,layer(m,n,"res_scale",x),y); + return {y}; + } + if(type=="RTMCCHead") { + auto x=layer(m,n,"final_layer",inputs.back()); + x=ggml_reshape_3d(m,ggml_cont(m,x),x->ne[0]*x->ne[1],x->ne[2],x->ne[3]); + x=layer(m,n,"gau",layer(m,n,"mlp",x)); + return {layer(m,n,"cls_x",x),layer(m,n,"cls_y",x)}; + } + if(type=="RTMWHead") { + auto flatten=[](model_ref mm,tensor x){return ggml_reshape_3d(mm,ggml_cont(mm,x),x->ne[0]*x->ne[1],x->ne[2],x->ne[3]);}; + auto top=layer(m,n,"mlp",flatten(m,layer(m,n,"final_layer",inputs.at(1)))); + auto bottom=cat(m,{layer(m,n,"conv_dec",layer(m,n,"ps",inputs.at(1))),inputs.at(0)}); + bottom=layer(m,n,"mlp2",flatten(m,layer(m,n,"final_layer2",bottom))); + auto x=layer(m,n,"gau",cat(m,{top,bottom},0)); + return {layer(m,n,"cls_x",x),layer(m,n,"cls_y",x)}; + } + if(type=="TemporalRegressionHead") { + auto x=layer(m,n,"conv",inputs.back()); + int joints=a.at("num_joints").get(); + return {ggml_reshape_3d(m,ggml_cont(m,x),3,joints,ggml_nelements(x)/(3*joints))}; + } + if(type=="FPN" || type=="PAFPN") { + int start=a.value("start_level",0); + auto const& lateral=child(n,"lateral_convs"); + auto const& fpn=child(n,"fpn_convs"); + Tensors xs; + for(size_t i=0;i(),inputs.at(start+i))); + auto up=a.value("upsample_cfg",Json::object()); + bool bilinear=up.value("mode",std::string{"nearest"})=="bilinear"; + for(size_t i=xs.size()-1;i>0;--i) { + xs[i-1]=ggml_add(m,xs[i-1],resize(m,xs[i],xs[i-1]->ne[0],xs[i-1]->ne[1],bilinear,up.value("align_corners",false))); + } + Tensors out; + for(size_t i=0;i(),xs[i])); + if(type=="PAFPN") { + for(size_t i=0;i+1(); + if(extra==false || extra.is_null()) { + while(int(out.size())()-1); + else if(extra=="on_lateral") x=xs.back(); + else if(extra=="on_output") x=out.back(); + else throw std::runtime_error("unknown extra conv source"); + for(size_t i=out.size();int(i)xs.size() && a.value("relu_before_extra_convs",false)) x=ggml_relu(m,x); + x=layer(m,fpn,fpn.at("order").at(i).get(),x);out.push_back(x); + } + } + return out; + } + if(type=="PPM" || type=="ASPPModule" || type=="DepthwiseSeparableASPPModule") { + Tensors out; + for(auto const& key:n.at("order")) { + tensor y=layer(m,n,key.get(),inputs.at(0)); + out.push_back(type=="PPM"?resize(m,y,inputs[0]->ne[0],inputs[0]->ne[1],true,a.value("align_corners",false)):y); + } + return out; + } + if(type=="FCNHead" || type=="DepthwiseSeparableFCNHead") { + auto x=transform(m,a,inputs).at(0); + auto y=layer(m,n,"convs",x); + if(a.value("concat_input",false)) y=layer(m,n,"conv_cat",cat(m,{x,y})); + return {layer(m,n,"conv_seg",y)}; + } + if(type=="PSPHead") { + auto x=transform(m,a,inputs).at(0); + auto ys=run(m,child(n,"psp_modules"),{x}); + ys.insert(ys.begin(),x); + x=layer(m,n,"bottleneck",cat(m,ys)); + return {layer(m,n,"conv_seg",x)}; + } + if(type=="ASPPHead" || type=="DepthwiseSeparableASPPHead") { + auto x=transform(m,a,inputs).at(0); + auto ys=run(m,child(n,"aspp_modules"),{x}); + auto pooled=layer(m,n,"image_pool",x); + ys.insert(ys.begin(),resize(m,pooled,x->ne[0],x->ne[1],true,a.value("align_corners",false))); + x=layer(m,n,"bottleneck",cat(m,ys)); + if(type=="DepthwiseSeparableASPPHead") { + if(c.contains("c1_bottleneck")) { + auto c1=layer(m,n,"c1_bottleneck",inputs[0]); + x=cat(m,{resize(m,x,c1->ne[0],c1->ne[1],true,a.value("align_corners",false)),c1}); + } + x=layer(m,n,"sep_bottleneck",x); + } + return {layer(m,n,"conv_seg",x)}; + } + if(type=="UPerHead") { + auto xs=transform(m,a,inputs); + auto ys=run(m,child(n,"psp_modules"),{xs.back()}); + ys.insert(ys.begin(),xs.back()); + auto psp=layer(m,n,"bottleneck",cat(m,ys)); + Tensors lateral; + auto const& lc=child(n,"lateral_convs"), fc=child(n,"fpn_convs"); + for(size_t i=0;i(),xs.at(i))); + lateral.push_back(psp); + bool align=a.value("align_corners",false); + for(size_t i=lateral.size()-1;i>0;--i) + lateral[i-1]=ggml_add(m,lateral[i-1],resize(m,lateral[i],lateral[i-1]->ne[0],lateral[i-1]->ne[1],true,align)); + Tensors out; + for(size_t i=0;i(),lateral.at(i))); + out.push_back(lateral.back()); + for(size_t i=1;ine[0],out[0]->ne[1],true,align); + auto x=layer(m,n,"fpn_bottleneck",cat(m,out)); + return {layer(m,n,"conv_seg",x)}; + } + if(type=="HeatmapHead" || type=="Heatmap3DHead" || type=="AssociativeEmbeddingHead") { + tensor x=inputs.back(); + for(auto const& key:{"deconv_layers","conv_layers","final_layer"}) + if(c.contains(key)) x=layer(m,n,key,x); + if(type=="Heatmap3DHead") { + int depth=a.at("depth_size").get(); + return {ggml_reshape_4d(m,x,x->ne[0],x->ne[1],depth,x->ne[2]/depth*x->ne[3])}; + } + if(type=="AssociativeEmbeddingHead") { + int channels=a.at("num_keypoints").get(); + auto part=[&](int start,int count){return ggml_cont(m,ggml_view_4d(m,x,x->ne[0],x->ne[1],count,x->ne[3],x->nb[1],x->nb[2],x->nb[3],size_t(start)*x->nb[2]));}; + return {part(0,channels),part(channels,int(x->ne[2])-channels)}; + } + return {x}; + } + throw std::runtime_error("missing neck/head forward"); +} +} // namespace visp::mmlab diff --git a/src/visp/mmlab/model.cpp b/src/visp/mmlab/model.cpp new file mode 100644 index 0000000..2504588 --- /dev/null +++ b/src/visp/mmlab/model.cpp @@ -0,0 +1,94 @@ +#include "config.h" +#include +#include +#include + +namespace visp::mmlab { +Tensors build_architecture(model_ref m, std::string const& name, Json const& spec, Tensors const& inputs) { + // Architecture factories are compiled into the runner. GGUF selects one; + // it cannot supply code or change the architecture's forward connections. + static std::unordered_map const architectures = { +#define MMLAB_ARCH(scope, name) {scope "." name, scope}, +#include "architectures.inc" +#undef MMLAB_ARCH + }; + auto it=architectures.find(name); + if(it==architectures.end())throw std::runtime_error("unregistered MMLab architecture: "+name); + auto expected=name=="mmdet.GLIP"?"MMDetTextCond":it->second=="mmdet"?"MMDetBackbone":it->second=="mmseg"?"MMSegWrap":"MMPoseWrap"; + if(spec.at("type")!=expected)throw std::runtime_error("architecture/module protocol mismatch"); + return models(m,spec,inputs); +} +Tensors models(model_ref m, Json const& n, Tensors const& inputs) { + auto type=n.at("type").get(); + auto const& c=n.at("children"); + if(type=="MMSegWrap"&&n.at("attrs").value("multimodal",false)) { + auto image=inputs.at(0),encoder_input=image; + if(n.at("attrs").value("asymetric_input",false)) { + double scale=n.at("attrs").at("encoder_resolution").get(); + encoder_input=resize(m,image,int64_t(image->ne[0]*scale),int64_t(image->ne[1]*scale),true); + } + auto features=run(m,child(n,"image_encoder"),{encoder_input});features.insert(features.begin(),image); + features.push_back(run(m,child(n,"text_encoder"),inputs).at(0)); + return run(m,child(n,"decode_head"),features); + } + auto out=run(m,child(n,"backbone"),inputs); + if(c.contains("neck")) out=run(m,child(n,"neck"),out); + if(type=="MMDetBackbone") return out; + if(type=="MMDetTextCond") { + out.push_back(parameter(m,n,"text_embedded")); + return run(m,child(n,"bbox_head"),out); + } + if(type=="MMSegWrap") { + if(n.at("attrs").value("multimodal",false)) + throw std::runtime_error("special segmentation predict protocol not implemented"); + if(n.at("attrs").value("cascade",false)) { + auto const& heads=child(n,"decode_head");Tensors result; + for(auto const& key:heads.at("order")) { + auto features=out; + if(!result.empty())features.insert(features.end(),result.begin(),result.end()); + auto head=child(heads,key.get()); + if(head.at("type")=="PointHead") { + head["attrs"]["native_test_cfg"]=n.at("attrs").at("test_cfg"); + head["attrs"]["native_input_w"]=inputs.at(0)->ne[0];head["attrs"]["native_input_h"]=inputs.at(0)->ne[1]; + } + result=run(m,head,features); + } + return result; + } + auto head=child(n,"decode_head");head["attrs"]["native_input_w"]=inputs.at(0)->ne[0];head["attrs"]["native_input_h"]=inputs.at(0)->ne[1]; + if(auto probe=std::getenv("VISP_SEG_PROBE")) { + std::string name=probe; + if(name=="pix"||name=="pix_ms") { + auto pixels=run(m,child(head,"pixel_decoder"),out); + if(name=="pix")return {pixels.front()};pixels.erase(pixels.begin());return pixels; + } + } + return run(m,head,out); + } + if(type=="MMPoseWrap") return run(m,child(n,"head"),out); + throw std::runtime_error("missing model architecture"); +} +tensor config_forward(model_ref m, tensor x, model_file const& f) { + auto j=read_config(f); + bool image=j.at("input_shape").size()==4; + if(image) x=cwhn_to_contiguous_2d(m,x); + auto out=build_architecture(m,j.at("architecture").get(),j.at("module"),{x}); + if(out.empty()) throw std::runtime_error("mmlab-config: empty output"); + tensor result=nullptr; + for(size_t i=0;i +#include + +namespace visp::mmlab { +namespace { +int pair(Json const& j, int i) { return j.is_array()?j.at(i).get():j.get(); } +tensor slice(model_ref m, tensor x, int axis, int64_t start, int64_t count) { + auto shape=i64x4{x->ne[0],x->ne[1],x->ne[2],x->ne[3]};shape[axis]=count; + return ggml_cont(m,ggml_view_4d(m,x,shape[0],shape[1],shape[2],shape[3], + x->nb[1],x->nb[2],x->nb[3],size_t(start)*x->nb[axis])); +} +tensor deform(model_ref m, Json const& n, Tensors const& inputs) { + auto const& a=n.at("attrs");auto type=n.at("type").get(); + auto w=weights(m,n);auto kernel=w.weights("weight"), x=inputs.at(0); + auto s=a.at("stride"),p=a.at("padding"),d=a.at("dilation"); + if(pair(s,0)!=pair(s,1)||pair(p,0)!=pair(p,1)||pair(d,0)!=1||pair(d,1)!=1) + throw std::runtime_error("asymmetric/dilated deform convolution unsupported"); + int stride=pair(s,0),pad=pair(p,0),groups=a.value("groups",1),dg=a.value("deform_groups",1); + int64_t area=kernel->ne[0]*kernel->ne[1]; + bool modulated=type.find("Modulated")!=std::string::npos; + tensor offset, mask=nullptr; + if(type.find("Pack")!=std::string::npos) { + auto packed=layer(m,n,"conv_offset",x); + offset=slice(m,packed,2,0,2*dg*area); + if(modulated)mask=ggml_sigmoid(m,slice(m,packed,2,2*dg*area,dg*area)); + } else { + offset=inputs.at(1); + if(modulated)mask=inputs.at(2); + } + auto ow=(x->ne[0]+2*pad-kernel->ne[0])/stride+1; + auto oh=(x->ne[1]+2*pad-kernel->ne[1])/stride+1; + auto fit=[&](tensor t) { + if(t->ne[0]ne[1]ne[2],1,ow*t->nb[0],ow*oh*t->nb[0],ow*oh*t->ne[2]*t->nb[0],0); + }; + offset=fit(offset);if(mask)mask=fit(mask); + if(dg<=0||x->ne[2]%dg || (groups!=1&&groups!=dg)) + throw std::runtime_error("unsupported deform/convolution group combination"); + Tensors parts;auto channels=x->ne[2]/dg; + for(int g=0;gne[3]/groups),kernel->ne[3]/groups); + parts.push_back(conv_2d_deform(m,slice(m,x,2,g*channels,channels),kg, + slice(m,offset,2,g*2*area,2*area),mask?slice(m,mask,2,g*area,area):nullptr,stride,pad)); + } + auto y=parts[0]; + for(size_t i=1;ine[0],1)); + return y; +} +tensor average_pool(model_ref m, Json const& a, tensor x) { + auto k=a.at("kernel_size"),s=a.at("stride"),p=a.at("padding");if(s.is_null())s=k; + int kw=pair(k,1),kh=pair(k,0),sw=pair(s,1),sh=pair(s,0),pw=pair(p,1),ph=pair(p,0); + auto width=x->ne[0],height=x->ne[1];bool ceil=a.value("ceil_mode",false); + auto count=[&](int64_t n,int kernel,int stride,int padding) { + auto y=int64_t(std::floor(double(n+2*padding-kernel+(ceil?stride-1:0))/stride))+1; + if(ceil&&(y-1)*stride>=n+padding)--y; + if(y<=0)throw std::runtime_error("invalid average pool output");return y; + }; + auto ow=count(width,kw,sw,pw),oh=count(height,kh,sh,ph); + // Extra ceil windows must contain zeros, but those zeros do not enlarge + // PyTorch's divisor. Account for the divisor separately below. + auto ew=std::max(0,(ow-1)*sw+kw-2*pw-width),eh=std::max(0,(oh-1)*sh+kh-2*ph-height); + if(ew||eh)x=ggml_pad_ext(m,x,0,int(ew),0,int(eh),0,0,0,0); + auto y=ggml_pool_2d(m,x,GGML_OP_POOL_AVG,kw,kh,sw,sh,pw,ph); + auto override=a.value("divisor_override",Json{}); + if(!override.is_null()) { + float divisor=override.get();if(divisor<=0)throw std::runtime_error("invalid pool divisor"); + return ggml_scale(m,y,float(kw*kh)/divisor); + } + if(!ceil&&a.value("count_include_pad",true))return y; + bool include=a.value("count_include_pad",true); + auto extent=[&](int64_t input,int64_t output,int kernel,int stride,int padding) { + auto start=ggml_scale_bias(m,ggml_arange(m,0,float(output),1),float(stride),float(-padding)); + auto end=ggml_scale_bias(m,start,1,float(kernel)); + end=ggml_clamp(m,end,-INFINITY,float(input+(include?padding:0))); + start=ggml_clamp(m,start,float(include?-padding:0),INFINITY); + return ggml_sub(m,end,start); + }; + auto cw=ggml_reshape_2d(m,extent(width,ow,kw,sw,pw),ow,1),ch=ggml_reshape_2d(m,extent(height,oh,kh,sh,ph),1,oh); + auto shape=ggml_new_tensor_2d(m,GGML_TYPE_F32,ow,oh); + auto divisor=ggml_mul(m,ggml_repeat(m,cw,shape),ch); + return ggml_div(m,ggml_scale(m,y,float(kw*kh)),divisor); +} +float pool_padding(int64_t size, int kernel, int stride, int padding, bool ceil) { + if(!ceil) return float(padding); + int64_t count=int64_t(std::ceil(double(size+2*padding-kernel)/stride))+1; + if((count-1)*stride>=size+padding) --count; + float adjusted=float((count-1)*stride+kernel-size)/2; + if(int(adjusted)!=padding) throw std::runtime_error("ceil pool cannot preserve kernel padding for this shape"); + return adjusted; +} +} +tensor adaptive_pool(model_ref m, tensor x, int w, int h, bool maximum) { + if(w<=0 || h<=0) throw std::runtime_error("invalid adaptive pool output"); + Tensors rows; + for(int iy=0;iyne[1]/h, y1=((iy+1)*x->ne[1]+h-1)/h; + for(int ix=0;ixne[0]/w, x1=((ix+1)*x->ne[0]+w-1)/w; + tensor crop=ggml_cont(m,ggml_view_4d(m,x,x1-x0,y1-y0,x->ne[2],x->ne[3], + x->nb[1],x->nb[2],x->nb[3],size_t(y0)*x->nb[1]+size_t(x0)*x->nb[0])); + cols.push_back(ggml_pool_2d(m,crop,maximum?GGML_OP_POOL_MAX:GGML_OP_POOL_AVG,int(x1-x0),int(y1-y0), + int(x1-x0),int(y1-y0),0,0)); + } + rows.push_back(cat(m,cols,0)); + } + return cat(m,rows,1); +} +Tensors modules(model_ref m, Json const& n, Tensors const& inputs) { + auto type=n.at("type").get(); + auto const& a=n.at("attrs"); + tensor x=inputs.at(0); + model_ref w=weights(m,n); + if(type=="Sequential" || type=="ResLayer" || type=="Res2Layer" || type=="ModuleList") { + Tensors out=inputs; + for(auto const& key:n.at("order")) out=run(m,child(n,key.get()),out); + return out; + } else if(type=="Identity" || type=="Dropout" || type=="Dropout2d" || type=="DropPath") { + // All exported models are eval-only. + } else if(type=="DeformConv2d" || type=="ModulatedDeformConv2d" || type=="DeformConv2dPack" || type=="ModulatedDeformConv2dPack") { + x=deform(m,n,inputs); + } else if(type=="SAConv2d") { + if(a.value("groups",1)!=1)throw std::runtime_error("SAC grouped convolution unsupported"); + auto stride=a.at("stride"),padding=a.at("padding"),dilation=a.at("dilation"); + if(pair(stride,0)!=pair(stride,1)||pair(padding,0)!=pair(padding,1)||pair(dilation,0)!=pair(dilation,1))throw std::runtime_error("SAC asymmetric parameters unsupported"); + x=ggml_add(m,x,layer(m,n,"pre_context",adaptive_pool(m,x,1,1))); + auto average=ggml_pool_2d(m,pad_reflect_ext(m,x,2,2,2,2),GGML_OP_POOL_AVG,5,5,1,1,0,0); + auto gate=layer(m,n,"switch",average),kernel=parameter(m,n,"weight"); + auto convolution=[&](tensor weight,char const* offset_name,int factor) { + int pad=pair(padding,0)*factor,dil=pair(dilation,0)*factor,step=pair(stride,0); + if(!a.value("use_deform",false))return ggml_conv_2d(m,ggml_cast(m,weight,GGML_TYPE_F16),x,step,step,pad,pad,dil,dil); + auto offset=layer(m,n,offset_name,average);int kw=int(weight->ne[0]),kh=int(weight->ne[1]); + int unit_pad=pad-(kw-1)*(dil-1)/2; + if(kw!=kh||kw%2==0||unit_pad<0)throw std::runtime_error("SAC dilated sampling requires odd square kernels"); + if(dil!=1) { + Tensors channels; + for(int h=0;htype!=GGML_TYPE_F16) kernel=ggml_cast(m,kernel,GGML_TYPE_F16); + x=ggml_conv_1d(m,kernel,x,pair(a.at("stride"),0),pair(a.at("padding"),0),pair(a.at("dilation"),0)); + if(auto bias=w.find("bias")) x=ggml_add(m,x,ggml_reshape_3d(m,bias,1,bias->ne[0],1)); + } else if(type=="Conv2d" || type=="Conv2dAdaptivePadding" || type=="ConvWS2d" || type=="ConvAWS2d") { + if(a.value("padding_mode",std::string{"zeros"})!="zeros") throw std::runtime_error("nonzero convolution padding mode unsupported"); + auto stride=a.at("stride"), pad=a.at("padding"), dilation=a.at("dilation"); + if(type=="Conv2dAdaptivePadding") { + auto kernel=w.weights("weight"); + auto pw=std::max(0,((x->ne[0]+pair(stride,1)-1)/pair(stride,1)-1)*pair(stride,1)+(kernel->ne[0]-1)*pair(dilation,1)+1-x->ne[0]); + auto ph=std::max(0,((x->ne[1]+pair(stride,0)-1)/pair(stride,0)-1)*pair(stride,0)+(kernel->ne[1]-1)*pair(dilation,0)+1-x->ne[1]); + if(pw||ph)x=ggml_pad_ext(m,x,int(pw/2),int(pw-pw/2),int(ph/2),int(ph-ph/2),0,0,0,0); + } + int groups=a.value("groups",1); + if(groups==1) x=conv_2d_ex(w,x,pair(stride,1),pair(stride,0),pair(pad,1),pair(pad,0),pair(dilation,1),pair(dilation,0)); + else if(groups==a.at("in_channels").get() && groups==a.at("out_channels").get()) + x=conv_2d_depthwise_ex(w,x,pair(stride,1),pair(stride,0),pair(pad,1),pair(pad,0),pair(dilation,1),pair(dilation,0)); + else { + if(pair(stride,0)!=pair(stride,1)||pair(pad,0)!=pair(pad,1)||pair(dilation,0)!=pair(dilation,1)) + throw std::runtime_error("asymmetric grouped convolution unsupported"); + x=conv_2d_grouped(w,x,pair(stride,0),pair(pad,0),pair(dilation,0),groups); + } + } else if(type=="ConvTranspose2d") { + if(pair(a.at("dilation"),0)!=1 || pair(a.at("dilation"),1)!=1) throw std::runtime_error("dilated transposed convolution unsupported"); + if(pair(a.at("stride"),0)!=pair(a.at("stride"),1) || pair(a.at("padding"),0)!=pair(a.at("padding"),1) + || pair(a.at("output_padding"),0)!=pair(a.at("output_padding"),1)) + throw std::runtime_error("asymmetric transposed convolution unsupported"); + x=conv_transpose_2d(w,x,pair(a.at("stride"),0),pair(a.at("padding"),0),a.value("groups",1),pair(a.at("output_padding"),0)); + } else if(type=="BatchNorm1d") { + auto scale=w.weights("weight"), bias=w.weights("bias"); + x=ggml_add(m,ggml_mul(m,x,ggml_reshape_3d(m,scale,1,scale->ne[0],1)),ggml_reshape_3d(m,bias,1,bias->ne[0],1)); + } else if(type=="BatchNorm2d" || type=="SyncBatchNorm" || type=="_BatchNormXd") x=batch_norm_2d(w,x); + else if(type=="GroupNorm") x=group_norm(w,x,a.at("num_groups").get(),a.value("eps",1e-5f)); + else if(type=="LayerNorm") { + auto shape=a.at("normalized_shape"); + if(shape.size()==3) { + if(x->ne[0]!=1||x->ne[1]!=1)throw std::runtime_error("channel LayerNorm requires spatial 1x1"); + x=ggml_reshape_4d(m,ggml_norm(m,ggml_reshape_2d(m,ggml_cont(m,x),x->ne[2],x->ne[3]),a.value("eps",1e-5f)),1,1,x->ne[2],x->ne[3]); + } else x=ggml_norm(m,x,a.value("eps",1e-5f)); + if(auto weight=w.find("weight")) x=ggml_mul(m,x,weight); + if(auto bias=w.find("bias")) x=ggml_add(m,x,bias); + } + else if(type=="Linear" || type=="NonDynamicallyQuantizableLinear") x=linear(w,x); + else if(type=="ReLU") x=ggml_relu(m,x); + else if(type=="ReLU6") x=ggml_clamp(m,x,0,6); + else if(type=="PReLU") x=prelu(w,x); + else if(type=="Sigmoid") x=ggml_sigmoid(m,x); + else if(type=="Softmax") { + if(a.at("dim")!=2 && a.at("dim")!=-1)throw std::runtime_error("softmax dimension protocol unsupported"); + x=ggml_soft_max(m,x); + } + else if(type=="SiLU" || type=="Swish") x=ggml_silu(m,x); + else if(type=="Hardsigmoid") x=ggml_hardsigmoid(m,x); + else if(type=="HSigmoid") { + float divisor=a.value("divisor",6.f); + if(divisor==0)throw std::runtime_error("hard sigmoid divisor is zero"); + x=ggml_clamp(m,ggml_scale_bias(m,x,1.f/divisor,a.value("bias",3.f)/divisor),a.value("min_value",0.f),a.value("max_value",1.f)); + } + else if(type=="Hardswish" || type=="HSwish") x=ggml_hardswish(m,x); + else if(type=="GELU") x=a.value("approximate",std::string{"none"})=="tanh" ? ggml_gelu(m,x):ggml_gelu_erf(m,x); + else if(type=="LeakyReLU") x=ggml_leaky_relu(m,x,a.value("negative_slope",0.01f),false); + else if(type=="MaxPool2d" || type=="AvgPool2d") { + if(type=="AvgPool2d")return {average_pool(m,a,x)}; + auto k=a.at("kernel_size"), s=a.at("stride"), p=a.at("padding"); + if(s.is_null()) s=k; + if(type=="MaxPool2d") { + auto d=a.value("dilation",Json(1)); + if(pair(d,0)!=1 || pair(d,1)!=1) throw std::runtime_error("dilated max pooling unsupported"); + } + bool ceil=a.value("ceil_mode",false); + x=ggml_pool_2d(m,x,type=="MaxPool2d"?GGML_OP_POOL_MAX:GGML_OP_POOL_AVG, + pair(k,1),pair(k,0),pair(s,1),pair(s,0), + pool_padding(x->ne[0],pair(k,1),pair(s,1),pair(p,1),ceil), + pool_padding(x->ne[1],pair(k,0),pair(s,0),pair(p,0),ceil)); + } else if(type=="PixelShuffle") { + int r=a.at("upscale_factor").get(); + if(x->ne[2]%(r*r))throw std::runtime_error("pixel shuffle channel mismatch"); + auto w0=x->ne[0], h0=x->ne[1], c0=x->ne[2]/(r*r), batch=x->ne[3]; + x=ggml_reshape_4d(m,ggml_cont(m,x),w0,h0,r,r*c0*batch); + x=ggml_cont(m,ggml_permute(m,x,1,2,0,3)); + x=ggml_reshape_4d(m,x,w0*r,h0,r,c0*batch); + x=ggml_cont(m,ggml_permute(m,x,0,2,1,3)); + x=ggml_reshape_4d(m,x,w0*r,h0*r,c0,batch); + } else if(type=="Upsample") { + auto size=a.value("size",Json{}), scale=a.value("scale_factor",Json{}); + int64_t w,h; + if(!size.is_null()) {w=pair(size,1);h=pair(size,0);} + else { + double sh=scale.is_array()?scale.at(0).get():scale.get(); + double sw=scale.is_array()?scale.at(1).get():scale.get(); + w=int64_t(x->ne[0]*sw);h=int64_t(x->ne[1]*sh); + } + auto mode=a.value("mode",std::string{"nearest"}); + if(mode!="nearest" && mode!="bilinear") throw std::runtime_error("unsupported resize mode"); + auto align=a.value("align_corners",Json{}); + x=resize(m,x,w,h,mode=="bilinear",!align.is_null()&&align.get()); + } else if(type=="AdaptiveAvgPool2d" || type=="AdaptiveMaxPool2d") { + auto size=a.at("output_size"); + x=adaptive_pool(m,x,pair(size,1),pair(size,0),type=="AdaptiveMaxPool2d"); + } else if(type=="TridentConv") { + int branch=a.at("test_branch_idx").get(); + if(branch<0)throw std::runtime_error("Trident all-branch inference requires a multi-batch protocol"); + int dilation=a.at("dilations").at(branch).get(),padding=a.at("paddings").at(branch).get(); + auto stride=a.at("stride");auto wm=weights(m,n); + x=ggml_conv_2d(m,ggml_cast(m,wm.weights("weight"),GGML_TYPE_F16),x,pair(stride,1),pair(stride,0),padding,padding,dilation,dilation); + if(auto bias=wm.find("bias"))x=ggml_add(m,x,ggml_reshape_4d(m,bias,1,1,bias->ne[0],1)); + } else if(type=="ConvModule") { + for(auto const& item:a.value("order",Json::array({"conv","norm","act"}))) { + auto op=item.get(); + if(op=="conv") x=layer(m,n,"conv",x); + else if(op=="norm" && a.value("with_norm",false)) x=layer(m,n,a.at("norm_name").get(),x); + else if(op=="act" && a.value("with_activation",false)) x=layer(m,n,"activate",x); + } + } else if(type=="DepthwiseSeparableConvModule") { + x=layer(m,n,"depthwise_conv",x); x=layer(m,n,"pointwise_conv",x); + } else throw std::runtime_error("missing primitive forward"); + return {x}; +} +} // namespace visp::mmlab diff --git a/src/visp/mmlab/text_fusion.cpp b/src/visp/mmlab/text_fusion.cpp new file mode 100644 index 0000000..a20a76d --- /dev/null +++ b/src/visp/mmlab/text_fusion.cpp @@ -0,0 +1,88 @@ +#include "config.h" + +#include + +namespace visp::mmlab { +namespace { +tensor channels(model_ref m, tensor x, int64_t start, int64_t count) { + return ggml_cont(m,ggml_view_4d(m,x,x->ne[0],x->ne[1],count,x->ne[3], + x->nb[1],x->nb[2],x->nb[3],size_t(start)*x->nb[2])); +} +} + +Tensors text_fusion(model_ref m, Json const& n, Tensors const& inputs) { + auto type=n.at("type").get(); + auto const& a=n.at("attrs"); + if(type=="Conv3x3Norm") { + auto x=run(m,child(n,"conv"),inputs).at(0); + return {layer(m,n,"bn",x)}; + } + if(type=="GLIPDyReLU") { + auto x=inputs.at(0),pooled=layer(m,n,"avg_pool",x); + auto coeff=layer(m,n,"fc",ggml_reshape_2d(m,pooled,x->ne[2],x->ne[3])); + coeff=ggml_reshape_4d(m,coeff,1,1,coeff->ne[0],x->ne[3]); + auto first=ggml_add(m,ggml_mul(m,x,ggml_scale(m,channels(m,coeff,0,x->ne[2]),2)), + ggml_scale_bias(m,channels(m,coeff,x->ne[2],x->ne[2]),1,-.5f)); + auto second=ggml_add(m,ggml_mul(m,x,ggml_scale_bias(m,channels(m,coeff,2*x->ne[2],x->ne[2]),2,-1)), + ggml_scale_bias(m,channels(m,coeff,3*x->ne[2],x->ne[2]),1,-.5f)); + return {ggml_add(m,second,ggml_relu(m,ggml_sub(m,first,second)))}; + } + if(type=="DyConv") { + Tensors out; + auto const& c=n.at("children"); + bool deform=c.at("offset").at("type")!="Identity"; + bool attention=c.at("attnconv").at("type")!="Identity"; + for(size_t i=0;ine[0],x->ne[1],true,true); + if(attention)y=ggml_mul(m,y,layer(m,n,"h_sigmoid",layer(m,n,"attnconv",y))); + return y; + }; + auto sum=branch("1",x,false);int count=1; + if(i>0){sum=ggml_add(m,sum,branch("2",inputs[i-1],false));++count;} + if(i+1(); + if(priors!=1)throw std::runtime_error("text fusion requires one prior per pixel"); + if(inputs.back()->ne[0]!=a.at("lang_dim").get() || inputs.back()->ne[2]!=1 || inputs.back()->ne[3]!=1) + throw std::runtime_error("text fusion requires batch-one token embeddings"); + Tensors features(inputs.begin(),inputs.end()-1); + auto embedding=ggml_l2_norm(m,inputs.back(),1e-12f); + auto tokens=layer(m,n,"dot_product_projection_text",ggml_scale(m,embedding,.5f)); + auto bias=ggml_mul_mat(m,ggml_reshape_2d(m,parameter(m,n,"bias_lang"),embedding->ne[0],1),embedding); + bias=ggml_cont(m,ggml_transpose(m,ggml_add(m,bias,parameter(m,n,"bias0")))); + auto visual=run(m,child(n,"dyhead_tower"),features); + Tensors scores,boxes,centers; + auto const& scales=child(n,"scales"); + if(visual.size()!=scales.at("order").size())throw std::runtime_error("text fusion level/scale mismatch"); + for(size_t i=0;ine[0]*x->ne[1],x->ne[2]))); + auto score=ggml_add(m,ggml_div(m,ggml_mul_mat(m,tokens,query),ggml_exp(m,parameter(m,n,"log_scale"))),bias); + scores.push_back(ggml_clamp(m,score,-50000,50000)); + boxes.push_back(one(m,child(scales,scales.at("order").at(i).get()),layer(m,n,"bbox_pred",x))); + centers.push_back(layer(m,n,"centerness",x)); + } + scores.insert(scores.end(),boxes.begin(),boxes.end()); + scores.insert(scores.end(),centers.begin(),centers.end()); + return scores; + } + throw std::runtime_error("unimplemented text fusion module: "+type); +} +} // namespace visp::mmlab diff --git a/src/visp/mmlab/transformers.cpp b/src/visp/mmlab/transformers.cpp new file mode 100644 index 0000000..2143950 --- /dev/null +++ b/src/visp/mmlab/transformers.cpp @@ -0,0 +1,233 @@ +#include "config.h" +#include +#include + +namespace visp::mmlab { +namespace { +tensor tokens(model_ref m,tensor x) { + return ggml_reshape_3d(m,ggml_cont(m,ggml_permute(m,x,1,2,0,3)),x->ne[2],x->ne[0]*x->ne[1],x->ne[3]); +} +tensor spatial(model_ref m,tensor x,int64_t w,int64_t h) { + return ggml_cont(m,ggml_permute(m,ggml_reshape_4d(m,x,x->ne[0],w,h,x->ne[2]),2,0,1,3)); +} +tensor norm_spatial(model_ref m,Json const& n,std::string const& name,tensor x) { + return spatial(m,layer(m,n,name,tokens(m,x)),x->ne[0],x->ne[1]); +} +int pair(Json const& j,int index){return j.is_array()?j.at(index).get():j.get();} +tensor window_partition(model_ref m,tensor x,int window) { + auto channels=x->ne[0],width=x->ne[1],height=x->ne[2],batch=x->ne[3]; + x=ggml_reshape_4d(m,x,channels*window,width/window,window,height/window*batch); + x=ggml_cont(m,ggml_permute(m,x,0,2,1,3)); + return ggml_reshape_3d(m,x,channels,window*window,width/window*height/window*batch); +} +tensor window_reverse(model_ref m,tensor x,int64_t width,int64_t height,int window) { + auto channels=x->ne[0],batch=x->ne[2]/(width/window)/(height/window); + x=ggml_reshape_4d(m,x,channels*window,window,width/window,height/window*batch); + x=ggml_cont(m,ggml_permute(m,x,0,2,1,3));return ggml_reshape_4d(m,x,channels,width,height,batch); +} +} +Tensors transformers(model_ref m,Json const& n,Tensors const& inputs) { + auto type=n.at("type").get();auto const& a=n.at("attrs");auto const& c=n.at("children");auto x=inputs.at(0); + if(type=="GeneralizedAttention") { + if(a.at("spatial_range").get()>=0||x->ne[3]!=1)throw std::runtime_error("GeneralizedAttention requires global batch-one attention"); + auto flags=a.at("attention_type").get>();int heads=a.at("num_heads").get(),dim=a.at("qk_embed_dim").get(); + auto query_input=c.contains("q_downsample")?layer(m,n,"q_downsample",x):x,key_input=c.contains("kv_downsample")?layer(m,n,"kv_downsample",x):x; + auto qw=query_input->ne[0],qh=query_input->ne[1],kw=key_input->ne[0],kh=key_input->ne[1]; + auto split=[&](tensor t){return ggml_cont(m,ggml_permute(m,ggml_reshape_3d(m,t,dim,heads,t->ne[1]),0,2,1,3));}; + tensor query=nullptr,key=nullptr,energy=nullptr; + if(flags[0]||flags[1])query=split(tokens(m,layer(m,n,"query_conv",query_input))); + if(flags[0]||flags[2])key=split(tokens(m,layer(m,n,"key_conv",key_input))); + auto bias=[&](char const* name){return ggml_reshape_3d(m,parameter(m,n,name),dim,1,heads);}; + bool saliency=flags[2]&&!flags[0]&&!flags[1]&&!flags[3]; + if(flags[0]||flags[2]) { + auto q=flags[0]?query:bias("appr_bias");if(flags[0]&&flags[2])q=ggml_add(m,q,bias("appr_bias")); + energy=ggml_mul_mat(m,key,q); + } + if(flags[1]||flags[3]) { + int dimensions=a.at("position_embedding_dim").get(); + auto position=[&](int64_t queries,int64_t keys,char const* projection) { + auto q=ggml_reshape_3d(m,ggml_scale(m,ggml_arange(m,0,float(queries),1),a.at("q_stride").get()),1,1,queries); + auto k=ggml_reshape_3d(m,ggml_scale(m,ggml_arange(m,0,float(keys),1),a.at("kv_stride").get()),1,keys,1); + auto shape=ggml_new_tensor_3d(m,GGML_TYPE_F32,1,keys,queries); + auto difference=ggml_scale(m,ggml_sub(m,ggml_repeat(m,q,shape),k),a.at("position_magnitude").get()); + auto frequency=ggml_exp(m,ggml_scale(m,ggml_arange(m,0,float(dimensions/4),1),-4.f*std::log(1000.f)/dimensions)); + auto full=ggml_new_tensor_3d(m,GGML_TYPE_F32,dimensions/4,keys,queries); + auto angle=ggml_mul(m,ggml_repeat(m,difference,full),frequency); + auto embedding=ggml_concat(m,ggml_sin(m,angle),ggml_cos(m,angle),0); + auto p=ggml_scale(m,layer(m,n,projection,embedding),1.f/std::sqrt(2.f)); + return ggml_cont(m,ggml_permute(m,ggml_reshape_4d(m,p,dim,heads,keys,queries),0,2,1,3)); + }; + auto px=position(qw,kw,"appr_geom_fc_x"),py=position(qh,kh,"appr_geom_fc_y"); + auto geometry_query=flags[1]?query:bias("geom_bias"); + if(flags[1]&&flags[3])geometry_query=ggml_add(m,geometry_query,bias("geom_bias")); + Tensors columns; + for(int64_t h=0;hnb[1],geometry_query->nb[2],size_t(h*qw+w)*geometry_query->nb[1]):geometry_query; + auto xposition=ggml_view_3d(m,px,dim,kw,heads,px->nb[1],px->nb[2],size_t(w)*px->nb[3]); + auto yposition=ggml_view_3d(m,py,dim,kh,heads,py->nb[1],py->nb[2],size_t(h)*py->nb[3]); + auto ex=ggml_reshape_3d(m,ggml_mul_mat(m,xposition,q),kw,1,heads),ey=ggml_reshape_3d(m,ggml_mul_mat(m,yposition,q),1,kh,heads); + auto square=ggml_new_tensor_3d(m,GGML_TYPE_F32,kw,kh,heads); + columns.push_back(ggml_reshape_3d(m,ggml_add(m,ggml_repeat(m,ex,square),ey),kw*kh,1,heads)); + } + auto geometry=cat(m,columns,1);energy=energy?ggml_add(m,geometry,energy):geometry; + } + if(!energy)throw std::runtime_error("GeneralizedAttention requires an attention term"); + auto value=split(tokens(m,layer(m,n,"value_conv",key_input))); + auto output=ggml_mul_mat(m,ggml_cont(m,ggml_transpose(m,value)),ggml_soft_max(m,energy)); + output=ggml_reshape_3d(m,ggml_cont(m,ggml_permute(m,output,0,2,1,3)),dim*heads,saliency?1:qw*qh,1); + output=layer(m,n,"proj_conv",spatial(m,output,saliency?1:qw,saliency?1:qh)); + if(a.at("q_stride").get()>1)output=resize(m,output,x->ne[0],x->ne[1],true); + return {ggml_add(m,x,ggml_mul(m,output,parameter(m,n,"gamma")))}; + } + if(type=="MixFFN")return {ggml_add(m,inputs.size()>1?inputs[1]:x,layer(m,n,"layers",x))}; + if(type=="EfficientMultiheadAttention"||type=="SpatialReductionAttention"||type=="GlobalSubsampledAttention") { + auto query=tokens(m,x),kv=query; + if(a.value("sr_ratio",1)>1)kv=layer(m,n,"norm",tokens(m,layer(m,n,"sr",x))); + auto y=run(m,child(n,"attn"),{query,kv,kv}).at(0); + return {ggml_add(m,inputs.size()>1?inputs[1]:x,spatial(m,y,x->ne[0],x->ne[1]))}; + } + if(type=="ConditionalPositionEncoding") { + auto y=layer(m,n,"proj",x); + return {a.value("stride",1)==1?ggml_add(m,x,y):y}; + } + if(type=="GSAEncoderLayer") { + auto normalized=norm_spatial(m,n,"norm1",x); + auto zero=ggml_scale(m,x,0); + x=ggml_add(m,x,run(m,child(n,"attn"),{normalized,zero}).at(0)); + auto y=layer(m,n,"ffn",layer(m,n,"norm2",tokens(m,x))); + return {ggml_add(m,x,spatial(m,y,x->ne[0],x->ne[1]))}; + } + if(type=="PCPVT") { + Tensors out;auto indices=a.at("out_indices").get>(); + auto const& stages=child(n,"layers");int i=0; + for(auto const& key:stages.at("order")) { + auto index=key.get();x=layer(m,child(n,"patch_embeds"),index,x); + auto const& blocks=child(stages,index);int j=0; + for(auto const& block:blocks.at("order")) { + x=layer(m,blocks,block.get(),x); + if(j++==0)x=layer(m,child(n,"position_encodings"),index,x); + } + if(a.value("norm_after_stage",false))x=norm_spatial(m,child(n,"norm_list"),index,x); + if(std::find(indices.begin(),indices.end(),i)!=indices.end())out.push_back(x);++i; + }return out; + } + if(type=="AbsolutePositionEmbedding") { + auto position=parameter(m,n,"pos_embed");auto h=a.at("pos_shape").at(0).get(),w=a.at("pos_shape").at(1).get(); + position=ggml_cont(m,ggml_view_3d(m,position,position->ne[0],h*w,1,position->nb[1],position->nb[2],size_t(position->ne[1]-h*w)*position->nb[1])); + position=resize(m,spatial(m,position,w,h),x->ne[0],x->ne[1],true);return {ggml_add(m,x,position)}; + } + if(type=="TransformerEncoderLayer"||type=="PVTEncoderLayer") { + auto y=run(m,child(n,"attn"),{norm_spatial(m,n,"norm1",x),x}).at(0); + return run(m,child(n,"ffn"),{norm_spatial(m,n,"norm2",y),y}); + } + if(type=="MixVisionTransformer"||type=="PyramidVisionTransformer"||type=="PyramidVisionTransformerV2") { + auto const& layers=child(n,"layers");auto indices=a.at("out_indices").get>();Tensors out;int i=0; + for(auto const& key:layers.at("order")) { + auto const& stage=child(layers,key.get());x=layer(m,stage,"0",x);x=layer(m,stage,"1",x);x=norm_spatial(m,stage,"2",x); + if(std::find(indices.begin(),indices.end(),i)!=indices.end())out.push_back(x);++i; + }return out; + } + if(type=="BEiTTransformerEncoderLayer"||type=="MAETransformerEncoderLayer") { + auto wm=weights(m,n);auto y=layer(m,n,"attn",layer(m,n,a.at("norm1_name").get(),x)); + x=ggml_add(m,x,ggml_mul(m,y,wm.weights("gamma_1"))); + y=layer(m,n,"ffn",layer(m,n,a.at("norm2_name").get(),x)); + return {ggml_add(m,x,ggml_mul(m,y,wm.weights("gamma_2")))}; + } + if(type=="BEiT"||type=="MAE") { + auto feature=layer(m,n,"patch_embed",x);auto width=feature->ne[0],height=feature->ne[1];auto wm=weights(m,n); + x=ggml_concat(m,wm.weights("cls_token"),tokens(m,feature),1); + if(type=="MAE")x=ggml_add(m,x,wm.weights("pos_embed")); + Tensors out;auto indices=a.at("out_indices").get>();auto const& layers=child(n,"layers");int i=0; + for(auto const& key:layers.at("order")) { + x=layer(m,layers,key.get(),x); + if(i+1==int(layers.at("order").size())&&a.value("final_norm",false))x=layer(m,n,a.at("norm1_name").get(),x); + if(std::find(indices.begin(),indices.end(),i)!=indices.end()) { + auto t=ggml_cont(m,ggml_view_3d(m,x,x->ne[0],width*height,x->ne[2],x->nb[1],x->nb[2],x->nb[1]));out.push_back(spatial(m,t,width,height)); + }++i; + }return out; + } + if(type=="WindowMSA"||type=="BEiTAttention"||type=="MAEAttention") { + auto wm=weights(m,n);int heads=a.at("num_heads").get();auto channels=x->ne[0],count=x->ne[1],batch=x->ne[2],dim=channels/heads; + auto qkv=layer(m,n,"qkv",x); + bool qv=a.value("bias",Json{})=="qv_bias"; + auto part=[&](int index) { + auto t=ggml_cont(m,ggml_view_3d(m,qkv,channels,count,batch,qkv->nb[1],qkv->nb[2],size_t(index*channels)*qkv->nb[0])); + if(qv&&index!=1)t=ggml_add(m,t,wm.weights(index==0?"q_bias":"v_bias")); + return ggml_cont(m,ggml_permute(m,ggml_reshape_4d(m,t,dim,heads,count,batch),0,2,1,3)); + }; + auto q=part(0),k=part(1),v=part(2); + auto scores=ggml_scale(m,ggml_mul_mat(m,k,q),a.value("scale",1.f/std::sqrt(float(dim)))); + if(n.at("tensors").contains("relative_position_bias_table")) { + auto indices=parameter(m,n,"relative_position_index");indices=ggml_reshape_1d(m,indices,ggml_nelements(indices)); + if(ggml_nelements(indices)!=count*count)throw std::runtime_error("relative position window must match token count"); + auto bias=ggml_get_rows(m,parameter(m,n,"relative_position_bias_table"),indices); + bias=ggml_cont(m,ggml_permute(m,ggml_reshape_4d(m,bias,heads,count,count,1),2,0,1,3));scores=ggml_add(m,scores,bias); + } + if(inputs.size()>1&&inputs[1])scores=ggml_add(m,scores,inputs[1]); + auto y=ggml_mul_mat(m,ggml_cont(m,ggml_transpose(m,v)),ggml_soft_max(m,scores)); + y=ggml_reshape_3d(m,ggml_cont(m,ggml_permute(m,y,0,2,1,3)),channels,count,batch); + return {layer(m,n,"proj",y)}; + } + if(type=="ShiftWindowMSA") { + int window=a.at("window_size").get(),shift=a.at("shift_size").get();auto width=x->ne[0],height=x->ne[1]; + int pw=(window-width%window)%window,ph=(window-height%window)%window; + auto y=ggml_cont(m,ggml_permute(m,x,1,2,0,3));if(pw||ph)y=ggml_pad(m,y,0,pw,ph,0); + auto padded_w=width+pw,padded_h=height+ph;tensor mask=nullptr; + if(shift) { + y=ggml_roll(m,y,0,-shift,-shift,0); + auto labels=[&](int64_t extent) { + auto index=ggml_arange(m,0,float(extent),1); + return ggml_add(m,ggml_step(m,ggml_scale_bias(m,index,1,float(-extent+window)+.5f)), + ggml_step(m,ggml_scale_bias(m,index,1,float(-extent+shift)+.5f))); + }; + auto row=ggml_reshape_4d(m,ggml_scale(m,labels(padded_h),3),1,1,padded_h,1); + auto col=ggml_reshape_4d(m,labels(padded_w),1,padded_w,1,1); + auto target=ggml_new_tensor_4d(m,GGML_TYPE_F32,1,padded_w,padded_h,1); + auto grid=ggml_add(m,ggml_repeat(m,col,target),row); + grid=window_partition(m,grid,window);int64_t count=window*window,windows=grid->ne[2]; + auto lhs=ggml_reshape_4d(m,grid,count,1,1,windows),rhs=ggml_reshape_4d(m,grid,1,count,1,windows); + auto square=ggml_new_tensor_4d(m,GGML_TYPE_F32,count,count,1,windows); + mask=ggml_scale(m,ggml_clamp(m,ggml_abs(m,ggml_sub(m,ggml_repeat(m,lhs,square),rhs)),0,1),-100); + } + y=window_partition(m,y,window);y=run(m,child(n,"w_msa"),{y,mask}).at(0); + y=window_reverse(m,y,padded_w,padded_h,window);if(shift)y=ggml_roll(m,y,0,shift,shift,0); + if(pw||ph)y=ggml_cont(m,ggml_view_4d(m,y,y->ne[0],width,height,y->ne[3],y->nb[1],y->nb[2],y->nb[3],0)); + return {ggml_cont(m,ggml_permute(m,y,2,0,1,3))}; + } + if(type=="SwinBlock") { + auto y=layer(m,n,"attn",norm_spatial(m,n,"norm1",x));x=ggml_add(m,x,y); + auto t=tokens(m,x);auto normalized=layer(m,n,"norm2",t); + return {spatial(m,run(m,child(n,"ffn"),{normalized,t}).at(0),x->ne[0],x->ne[1])}; + } + if(type=="PatchMerging") { + if(c.contains("adap_padding"))x=layer(m,n,"adap_padding",x); + auto const& sample=child(n,"sampler").at("attrs"); + auto kernel=sample.at("kernel_size"),stride=sample.at("stride"),padding=sample.at("padding"),dilation=sample.at("dilation"); + if(pair(kernel,0)!=2||pair(kernel,1)!=2||pair(stride,0)!=2||pair(stride,1)!=2||pair(padding,0)||pair(padding,1)||pair(dilation,0)!=1||pair(dilation,1)!=1) + throw std::runtime_error("PatchMerging requires 2x2 stride-two sampling"); + Tensors patches;auto width=x->ne[0]/2,height=x->ne[1]/2,channels=x->ne[2]; + auto channel_first=ggml_cont(m,ggml_permute(m,x,1,2,0,3)); + for(int h=0;h<2;++h)for(int w=0;w<2;++w) { + auto y=ggml_cont(m,slice(m,channel_first,{}, {w,x->ne[0],2},{h,x->ne[1],2},{})); + y=ggml_reshape_3d(m,y,channels,width*height,1); + patches.push_back(ggml_cont(m,ggml_transpose(m,y))); + } + auto y=ggml_cont(m,ggml_permute(m,cat(m,patches,2),2,1,0,3)); + y=ggml_reshape_3d(m,y,4*channels,width*height,1); + if(c.contains("norm"))y=layer(m,n,"norm",y);y=layer(m,n,"reduction",y); + return {spatial(m,y,width,height)}; + } + if(type=="SwinBlockSequence") { + x=layer(m,n,"blocks",x);auto y=c.contains("downsample")?layer(m,n,"downsample",x):x;return {y,x}; + } + if(type=="SwinTransformer") { + x=layer(m,n,"patch_embed",x); + if(a.value("use_abs_pos_embed",false))x=spatial(m,ggml_add(m,tokens(m,x),weights(m,n).weights("absolute_pos_embed")),x->ne[0],x->ne[1]); + auto const& stages=child(n,"stages");auto indices=a.at("out_indices").get>();Tensors out;int i=0; + for(auto const& key:stages.at("order")) {auto ys=run(m,child(stages,key.get()),{x});x=ys.at(0); + if(std::find(indices.begin(),indices.end(),i)!=indices.end())out.push_back(norm_spatial(m,n,"norm"+std::to_string(i),ys.at(1)));++i;} + return out; + } + throw std::runtime_error("native Transformer protocol unsupported: "+type); +} +} diff --git a/src/visp/mmlab/types.inc b/src/visp/mmlab/types.inc new file mode 100644 index 0000000..e03168a --- /dev/null +++ b/src/visp/mmlab/types.inc @@ -0,0 +1,317 @@ +// Registered native forward implementations. Export and runtime share this list. +MMLAB_TYPE("Sequential", modules) +MMLAB_TYPE("ModuleList", modules) +MMLAB_TYPE("Identity", modules) +MMLAB_TYPE("Dropout", modules) +MMLAB_TYPE("Dropout2d", modules) +MMLAB_TYPE("DropPath", modules) +MMLAB_TYPE("AdaptivePadding", backbones) +MMLAB_TYPE("PatchEmbed", backbones) +MMLAB_TYPE("TransformerEncoderLayer", backbones) +MMLAB_TYPE("VisionTransformer", backbones) +MMLAB_TYPE("SwinTransformer", transformers) +MMLAB_TYPE("PyramidVisionTransformer", transformers) +MMLAB_TYPE("PyramidVisionTransformerV2", transformers) +MMLAB_TYPE("SpatialReductionAttention", transformers) +MMLAB_TYPE("PVTEncoderLayer", transformers) +MMLAB_TYPE("AbsolutePositionEmbedding", transformers) +MMLAB_TYPE("MixVisionTransformer", transformers) +MMLAB_TYPE("EfficientMultiheadAttention", transformers) +MMLAB_TYPE("MixFFN", transformers) +MMLAB_TYPE("SegformerHead", heads) +MMLAB_TYPE("BEiT", transformers) +MMLAB_TYPE("BEiTAttention", transformers) +MMLAB_TYPE("BEiTTransformerEncoderLayer", transformers) +MMLAB_TYPE("MAE", transformers) +MMLAB_TYPE("MAEAttention", transformers) +MMLAB_TYPE("MAETransformerEncoderLayer", transformers) +MMLAB_TYPE("SwinBlock", transformers) +MMLAB_TYPE("SwinBlockSequence", transformers) +MMLAB_TYPE("ShiftWindowMSA", transformers) +MMLAB_TYPE("WindowMSA", transformers) +MMLAB_TYPE("PatchMerging", transformers) +MMLAB_TYPE("Unfold", transformers) +MMLAB_TYPE("ConvNeXt", backbones) +MMLAB_TYPE("ConvNeXtBlock", backbones) +MMLAB_TYPE("LayerNorm2d", backbones) +MMLAB_TYPE("PoolFormer", backbones) +MMLAB_TYPE("PoolFormerBlock", backbones) +MMLAB_TYPE("Pooling", backbones) +MMLAB_TYPE("Mlp", backbones) +MMLAB_TYPE("Conv2d", modules) +MMLAB_TYPE("ConvWS2d", modules) +MMLAB_TYPE("ConvAWS2d", modules) +MMLAB_TYPE("Conv2dAdaptivePadding", modules) +MMLAB_TYPE("DeformConv2d", modules) +MMLAB_TYPE("ModulatedDeformConv2d", modules) +MMLAB_TYPE("DeformConv2dPack", modules) +MMLAB_TYPE("ModulatedDeformConv2dPack", modules) +MMLAB_TYPE("Conv1d", modules) +MMLAB_TYPE("ConvTranspose2d", modules) +MMLAB_TYPE("BatchNorm2d", modules) +MMLAB_TYPE("BatchNorm1d", modules) +MMLAB_TYPE("SyncBatchNorm", modules) +MMLAB_TYPE("_BatchNormXd", modules) +MMLAB_TYPE("GroupNorm", modules) +MMLAB_TYPE("LayerNorm", modules) +MMLAB_TYPE("Linear", modules) +MMLAB_TYPE("ReLU", modules) +MMLAB_TYPE("ReLU6", modules) +MMLAB_TYPE("PReLU", modules) +MMLAB_TYPE("Sigmoid", modules) +MMLAB_TYPE("SiLU", modules) +MMLAB_TYPE("Swish", modules) +MMLAB_TYPE("Hardsigmoid", modules) +MMLAB_TYPE("HSigmoid", modules) +MMLAB_TYPE("Hardswish", modules) +MMLAB_TYPE("HSwish", modules) +MMLAB_TYPE("GELU", modules) +MMLAB_TYPE("LeakyReLU", modules) +MMLAB_TYPE("MaxPool2d", modules) +MMLAB_TYPE("AvgPool2d", modules) +MMLAB_TYPE("Upsample", modules) +MMLAB_TYPE("AdaptiveAvgPool2d", modules) +MMLAB_TYPE("AdaptiveMaxPool2d", modules) +MMLAB_TYPE("Softmax", modules) +MMLAB_TYPE("ConvModule", modules) +MMLAB_TYPE("DepthwiseSeparableConvModule", modules) +MMLAB_TYPE("BasicBlock", backbones) +MMLAB_TYPE("Bottleneck", backbones) +MMLAB_TYPE("ResLayer", modules) +MMLAB_TYPE("ResNet", backbones) +MMLAB_TYPE("ResNetV1c", backbones) +MMLAB_TYPE("ResNetV1d", backbones) +MMLAB_TYPE("RegNet", backbones) +MMLAB_TYPE("Res2Net", backbones) +MMLAB_TYPE("Bottle2neck", backbones) +MMLAB_TYPE("Res2Layer", modules) +MMLAB_TYPE("ResNeSt", backbones) +MMLAB_TYPE("RSoftmax", backbones) +MMLAB_TYPE("SplitAttentionConv2d", backbones) +MMLAB_TYPE("MobileNetV2", backbones) +MMLAB_TYPE("EfficientNet", backbones) +MMLAB_TYPE("EdgeResidual", backbones) +MMLAB_TYPE("MobileNetV3", backbones) +MMLAB_TYPE("InvertedResidual", backbones) +MMLAB_TYPE("InvertedResidualV3", backbones) +MMLAB_TYPE("SELayer", backbones) +MMLAB_TYPE("LRASPPHead", heads) +MMLAB_TYPE("ChannelMapper", heads) +MMLAB_TYPE("CTResNetNeck", heads) +MMLAB_TYPE("HybridEncoder", heads) +MMLAB_TYPE("CSPRepLayer", heads) +MMLAB_TYPE("RepVGGBlock", heads) +MMLAB_TYPE("RTMOHead", heads) +MMLAB_TYPE("RTMOHeadModule", heads) +MMLAB_TYPE("DetrTransformerEncoder", heads) +MMLAB_TYPE("DetrTransformerEncoderLayer", heads) +MMLAB_TYPE("MultiheadAttention", heads) +MMLAB_TYPE("FFN", heads) +MMLAB_TYPE("SinePositionalEncoding", heads) +MMLAB_TYPE("NonDynamicallyQuantizableLinear", modules) +MMLAB_TYPE("CGNet", backbones) +MMLAB_TYPE("BiSeNetV1", backbones) +MMLAB_TYPE("SpatialPath", backbones) +MMLAB_TYPE("AttentionRefinementModule", backbones) +MMLAB_TYPE("ContextPath", backbones) +MMLAB_TYPE("SSDVGG", backbones) +MMLAB_TYPE("SSDNeck", heads) +MMLAB_TYPE("L2Norm", heads) +MMLAB_TYPE("ERFNet", backbones) +MMLAB_TYPE("NonBottleneck1d", backbones) +MMLAB_TYPE("DownsamplerBlock", backbones) +MMLAB_TYPE("UpsamplerBlock", backbones) +MMLAB_TYPE("UNet", backbones) +MMLAB_TYPE("BasicConvBlock", backbones) +MMLAB_TYPE("InterpConv", backbones) +MMLAB_TYPE("DeconvModule", backbones) +MMLAB_TYPE("UpConvBlock", backbones) +MMLAB_TYPE("FastSCNN", backbones) +MMLAB_TYPE("LearningToDownsample", backbones) +MMLAB_TYPE("GlobalFeatureExtractor", backbones) +MMLAB_TYPE("FeatureFusionModule", backbones) +MMLAB_TYPE("CSPNeXt", backbones) +MMLAB_TYPE("CSPDarknet", backbones) +MMLAB_TYPE("Focus", backbones) +MMLAB_TYPE("CSPNeXtBlock", backbones) +MMLAB_TYPE("DarknetBottleneck", backbones) +MMLAB_TYPE("CSPLayer", backbones) +MMLAB_TYPE("SPPBottleneck", backbones) +MMLAB_TYPE("ChannelAttention", backbones) +MMLAB_TYPE("TCN", backbones) +MMLAB_TYPE("HRNet", backbones) +MMLAB_TYPE("HourglassNet", backbones) +MMLAB_TYPE("HourglassModule", backbones) +MMLAB_TYPE("HRModule", backbones) +MMLAB_TYPE("BasicTemporalBlock", backbones) +MMLAB_TYPE("ContextGuidedBlock", backbones) +MMLAB_TYPE("GlobalContextExtractor", backbones) +MMLAB_TYPE("InputInjection", backbones) +MMLAB_TYPE("FPN", heads) +MMLAB_TYPE("DAHead", heads) +MMLAB_TYPE("APCHead", heads) +MMLAB_TYPE("ISAHead", heads) +MMLAB_TYPE("CCHead", heads) +MMLAB_TYPE("CrissCrossAttention", heads) +MMLAB_TYPE("DDRNet", backbones) +MMLAB_TYPE("DDRHead", heads) +MMLAB_TYPE("DAPPM", heads) +MMLAB_TYPE("PAPPM", heads) +MMLAB_TYPE("PIDNet", backbones) +MMLAB_TYPE("PIDHead", heads) +MMLAB_TYPE("BasePIDHead", heads) +MMLAB_TYPE("PagFM", backbones) +MMLAB_TYPE("Bag", backbones) +MMLAB_TYPE("LightBag", backbones) +MMLAB_TYPE("DPTHead", heads) +MMLAB_TYPE("ReassembleBlocks", heads) +MMLAB_TYPE("PreActResidualConvUnit", heads) +MMLAB_TYPE("FeatureFusionBlock", heads) +MMLAB_TYPE("ICNet", backbones) +MMLAB_TYPE("ICNeck", heads) +MMLAB_TYPE("CascadeFeatureFusion", heads) +MMLAB_TYPE("SETRUPHead", heads) +MMLAB_TYPE("SegmenterMaskTransformerHead", heads) +MMLAB_TYPE("EncHead", heads) +MMLAB_TYPE("EncModule", heads) +MMLAB_TYPE("Encoding", heads) +MMLAB_TYPE("BiSeNetV2", backbones) +MMLAB_TYPE("DetailBranch", backbones) +MMLAB_TYPE("SemanticBranch", backbones) +MMLAB_TYPE("StemBlock", backbones) +MMLAB_TYPE("GELayer", backbones) +MMLAB_TYPE("CEBlock", backbones) +MMLAB_TYPE("BGALayer", backbones) +MMLAB_TYPE("ANNHead", heads) +MMLAB_TYPE("AFNB", heads) +MMLAB_TYPE("APNB", heads) +MMLAB_TYPE("PPMConcat", heads) +MMLAB_TYPE("STDCNet", backbones) +MMLAB_TYPE("STDCModule", backbones) +MMLAB_TYPE("STDCContextPathNet", backbones) +MMLAB_TYPE("ACM", heads) +MMLAB_TYPE("DMHead", heads) +MMLAB_TYPE("DCM", heads) +MMLAB_TYPE("PAM", heads) +MMLAB_TYPE("CAM", heads) +MMLAB_TYPE("JPU", heads) +MMLAB_TYPE("Feature2Pyramid", heads) +MMLAB_TYPE("MultiLevelNeck", heads) +MMLAB_TYPE("BFP", heads) +MMLAB_TYPE("ContextBlock", heads) +MMLAB_TYPE("NonLocal2d", heads) +MMLAB_TYPE("DisentangledNonLocal2d", heads) +MMLAB_TYPE("NLHead", heads) +MMLAB_TYPE("DNLHead", heads) +MMLAB_TYPE("GCHead", heads) +MMLAB_TYPE("PAFPN", heads) +MMLAB_TYPE("HRFPN", heads) +MMLAB_TYPE("DilatedEncoder", heads) +MMLAB_TYPE("YOLOV3Neck", heads) +MMLAB_TYPE("DetectionBlock", heads) +MMLAB_TYPE("Darknet", backbones) +MMLAB_TYPE("ResBlock", backbones) +MMLAB_TYPE("FPNHead", heads) +MMLAB_TYPE("EMAHead", heads) +MMLAB_TYPE("EMAModule", heads) +MMLAB_TYPE("OCRHead", heads) +MMLAB_TYPE("SpatialGatherModule", heads) +MMLAB_TYPE("ObjectAttentionBlock", heads) +MMLAB_TYPE("SelfAttentionBlock", heads) +MMLAB_TYPE("FCNHead", heads) +MMLAB_TYPE("DepthwiseSeparableFCNHead", heads) +MMLAB_TYPE("PSPHead", heads) +MMLAB_TYPE("ASPPHead", heads) +MMLAB_TYPE("ASPPModule", heads) +MMLAB_TYPE("DepthwiseSeparableASPPHead", heads) +MMLAB_TYPE("DepthwiseSeparableASPPModule", heads) +MMLAB_TYPE("UPerHead", heads) +MMLAB_TYPE("PPM", heads) +MMLAB_TYPE("HeatmapHead", heads) +MMLAB_TYPE("DEKRHead", heads) +MMLAB_TYPE("AdaptiveActivationBlock", heads) +MMLAB_TYPE("FeatureMapProcessor", heads) +MMLAB_TYPE("GlobalAveragePooling", heads) +MMLAB_TYPE("Heatmap3DHead", heads) +MMLAB_TYPE("Heatmap1DHead", heads) +MMLAB_TYPE("InternetHead", heads) +MMLAB_TYPE("MultilabelClassificationHead", heads) +MMLAB_TYPE("AssociativeEmbeddingHead", heads) +MMLAB_TYPE("CPMHead", heads) +MMLAB_TYPE("CPM", backbones) +MMLAB_TYPE("CpmBlock", backbones) +MMLAB_TYPE("CSPNeXtPAFPN", heads) +MMLAB_TYPE("YOLOXPAFPN", heads) +MMLAB_TYPE("RTMWHead", heads) +MMLAB_TYPE("PixelShuffle", modules) +MMLAB_TYPE("TemporalRegressionHead", heads) +MMLAB_TYPE("RTMCCHead", heads) +MMLAB_TYPE("RTMCCBlock", heads) +MMLAB_TYPE("ScaleNorm", heads) +MMLAB_TYPE("Scale", heads) +MMLAB_TYPE("MMSegWrap", models) +MMLAB_TYPE("_ScalarAffine", modules) +MMLAB_TYPE("CLIPTextEncoder", heads) +MMLAB_TYPE("MLP", heads) +MMLAB_TYPE("BaseTransformerLayer", heads) +MMLAB_TYPE("MLPMaskDecoder", heads) +MMLAB_TYPE("SideAdapterNetwork", heads) +MMLAB_TYPE("RecWithAttnbias", heads) +MMLAB_TYPE("SideAdapterCLIPHead", heads) +MMLAB_TYPE("QuickGELU", modules) +MMLAB_TYPE("MultiScaleDeformableAttention", heads) +MMLAB_TYPE("Mask2FormerTransformerEncoder", heads) +MMLAB_TYPE("DeformableDetrTransformerEncoderLayer", heads) +MMLAB_TYPE("MSDeformAttnPixelDecoder", heads) +MMLAB_TYPE("Mask2FormerHead", heads) +MMLAB_TYPE("Mask2FormerTransformerDecoder", heads) +MMLAB_TYPE("PointHead", heads) +MMLAB_TYPE("Embedding", modules) +MMLAB_TYPE("PixelDecoder", heads) +MMLAB_TYPE("TransformerEncoderPixelDecoder", heads) +MMLAB_TYPE("MaskFormerHead", heads) +MMLAB_TYPE("DetrTransformerDecoder", heads) +MMLAB_TYPE("DetrTransformerDecoderLayer", heads) +MMLAB_TYPE("Mask2FormerTransformerDecoderLayer", heads) +MMLAB_TYPE("GeneralizedAttention", transformers) +MMLAB_TYPE("KernelUpdator", heads) +MMLAB_TYPE("KernelUpdateHead", heads) +MMLAB_TYPE("IterativeDecodeHead", heads) +MMLAB_TYPE("SAConv2d", modules) +MMLAB_TYPE("DyHead", heads) +MMLAB_TYPE("DyHeadBlock", heads) +MMLAB_TYPE("DyDCNv2", heads) +MMLAB_TYPE("DyReLU", heads) +MMLAB_TYPE("GLIPDyReLU", text_fusion) +MMLAB_TYPE("Conv3x3Norm", text_fusion) +MMLAB_TYPE("DyConv", text_fusion) +MMLAB_TYPE("VLFusionModule", text_fusion) +MMLAB_TYPE("ATSSVLFusionHead", text_fusion) +MMLAB_TYPE("MMDetTextCond", models) +MMLAB_TYPE("TridentResNet", backbones) +MMLAB_TYPE("TridentBottleneck", backbones) +MMLAB_TYPE("TridentConv", modules) +MMLAB_TYPE("DetectoRS_ResNet", backbones) +MMLAB_TYPE("FPN_CARAFE", heads) +MMLAB_TYPE("CARAFEPack", heads) +MMLAB_TYPE("FPG", heads) +MMLAB_TYPE("UpInterpolationConv", heads) +MMLAB_TYPE("LastConv", heads) +MMLAB_TYPE("StemConv", backbones) +MMLAB_TYPE("OverlapPatchEmbed", backbones) +MMLAB_TYPE("MSCAAttention", backbones) +MMLAB_TYPE("MSCASpatialAttention", backbones) +MMLAB_TYPE("MSCABlock", backbones) +MMLAB_TYPE("MSCAN", backbones) +MMLAB_TYPE("LightHamHead", heads) +MMLAB_TYPE("Hamburger", heads) +MMLAB_TYPE("NMF2D", heads) +MMLAB_TYPE("ModuleDict", modules) +MMLAB_TYPE("ConcatCell", heads) +MMLAB_TYPE("NASFCOS_FPN", heads) +MMLAB_TYPE("PCPVT", transformers) +MMLAB_TYPE("ConditionalPositionEncoding", transformers) +MMLAB_TYPE("GSAEncoderLayer", transformers) +MMLAB_TYPE("GlobalSubsampledAttention", transformers) +MMLAB_TYPE("MMPoseWrap", models) +MMLAB_TYPE("MMDetBackbone", models) diff --git a/src/visp/nn.cpp b/src/visp/nn.cpp index 2cf16b2..f8d364b 100644 --- a/src/visp/nn.cpp +++ b/src/visp/nn.cpp @@ -1,3 +1,4 @@ +#include #include #include "nn.h" #include "util/string.h" @@ -85,12 +86,33 @@ tensor add_bias_2d(model_ref m, tensor x) { return x; } +// PReLU — **채널마다 다른 기울기**를 갖는다(`nn.PReLU(num_parameters=C)`). +// y = max(0,x) + w * min(0,x) = relu(x) + w * (x - relu(x)) +// ⚠️ **`leaky_relu(0.25)` 로 근사하지 마라.** 0.25 는 torch 의 **기본 초기값**이라 +// 랜덤 초기화 모델에서는 근사가 정확히 맞아떨어져 **PASS 가 나온다** — 학습된 +// 가중치에서만 틀린다(cgnet 실측: 랜덤 1.91e-04 PASS → 학습 5.09e+00 FAIL). +// `num_parameters=1` 이면 w 가 스칼라라 같은 식이 그대로 성립한다. +tensor prelu(model_ref m, tensor x) { + tensor w = m.weights("weight"); + if (!(m.flags & model_build_flag::cwhn)) { + // WHCN 은 채널이 ne[2] 다 — [1,1,C,1] 로 펴야 broadcast 된다(add_bias_2d 와 같은 규칙). + w = ggml_reshape_4d(m, w, 1, 1, w->ne[0], 1); + } + tensor pos = ggml_relu(m, x); + tensor neg = ggml_sub(m, x, pos); // min(x, 0) + return ggml_add(m, pos, ggml_mul(m, neg, w)); +} + // conv_2d 의 본체 — weight 를 인자로 받고 bias 는 붙이지 않는다. // conv_2d / conv_2d_wt 가 공유한다. dilation 기본값 1 은 기존 동작과 동일하다. -static tensor conv_2d_impl(model_ref m, tensor x, tensor weight, int stride, int pad, - int dilation) { +static tensor conv_2d_impl(model_ref m, tensor x, tensor weight, int sw, int sh, int pw, + int ph, int dw, int dh) { + // ⚠️ 축별 인자다. ggml 은 원래 축별로 받는데(`s0`=W, `s1`=H) 여기서 하나로 묶어 + // 넘기고 있었다 — 비대칭 conv(ERFNet 의 3x1/1x3)이 **조용히 대칭으로** 돌았다. if (m.flags & model_build_flag::cwhn) { - if (weight->ne[1] == 1 && weight->ne[2] == 1 && stride == 1 && dilation == 1) { + // 1x1 fast path 는 padding 을 아예 안 태운다 → pad 가 0 일 때만 쓸 수 있다. + if (weight->ne[1] == 1 && weight->ne[2] == 1 && sw == 1 && sh == 1 && dw == 1 && + dh == 1 && pw == 0 && ph == 0) { auto [c, w, h, b] = nelements(x); weight = ggml_reshape_2d(m, weight, weight->ne[0], weight->ne[3]); x = ggml_reshape_2d(m, x, x->ne[0], w * h * b); @@ -100,29 +122,45 @@ static tensor conv_2d_impl(model_ref m, tensor x, tensor weight, int stride, int } else if (m.flags & model_build_flag::conv_2d_direct_cwhn) { weight = permute_cwhn_to_whcn(m, weight); x = permute_cwhn_to_whcn(m, x); - x = ggml_conv_2d_direct(m, weight, x, stride, stride, pad, pad, dilation, dilation); + x = ggml_conv_2d_direct(m, weight, x, sw, sh, pw, ph, dw, dh); x = permute_whcn_to_cwhn(m, x); } else { weight = ggml_cont(m, permute_cwhn_to_whcn(m, weight)); x = ggml_cont(m, permute_cwhn_to_whcn(m, x)); - x = ggml_conv_2d(m, weight, x, stride, stride, pad, pad, dilation, dilation); + x = ggml_conv_2d(m, weight, x, sw, sh, pw, ph, dw, dh); x = ggml_cont(m, permute_whcn_to_cwhn(m, x)); } } else { // WHCN layout - x = ggml_conv_2d_direct(m, weight, x, stride, stride, pad, pad, dilation, dilation); + // conv 를 하나의 CONV_2D 노드로 낼지, im2col + mul_mat 두 노드로 쪼갤지 고른다. + // 둘 다 ggml 의 정식 표현이라 결과는 같아야 한다. 장치의 CONV_2D 커널이 틀린 값을 + // 낼 때 im2col 조합으로 우회할 수 있고, 반대로 어느 쪽이 계약을 안 지키는지도 가른다. + if (std::getenv("VISP_CONV_IM2COL")) { + x = ggml_conv_2d(m, weight, x, sw, sh, pw, ph, dw, dh); + } else { + x = ggml_conv_2d_direct(m, weight, x, sw, sh, pw, ph, dw, dh); + } } return x; } tensor conv_2d(model_ref m, tensor x, int stride, int pad, int dilation) { - x = conv_2d_impl(m, x, m.weights("weight"), stride, pad, dilation); + x = conv_2d_impl(m, x, m.weights("weight"), stride, stride, pad, pad, dilation, dilation); + return add_bias_2d(m, x); +} + +// 축별 conv — 커널·padding·dilation 이 H 와 W 에서 다른 경우(ERFNet 의 3x1/1x3 분리 conv). +// 인자 순서는 ggml 과 같은 **W 먼저**다(`ggml_conv_2d(s0=W, s1=H, …)`). +tensor conv_2d_ex(model_ref m, tensor x, int stride_w, int stride_h, int pad_w, int pad_h, + int dilation_w, int dilation_h) { + x = conv_2d_impl(m, x, m.weights("weight"), stride_w, stride_h, pad_w, pad_h, dilation_w, + dilation_h); return add_bias_2d(m, x); } tensor conv_2d_wt(model_ref m, tensor x, tensor weight, tensor bias, int stride, int pad, int dilation) { - x = conv_2d_impl(m, x, weight, stride, pad, dilation); + x = conv_2d_impl(m, x, weight, stride, stride, pad, pad, dilation, dilation); if (bias) { // WHCN 은 채널이 ne[2] 라 broadcast 를 위해 [1,1,C,1] 로 편다 (add_bias_2d 와 같은 규칙). if (!(m.flags & model_build_flag::cwhn)) { @@ -133,6 +171,29 @@ tensor conv_2d_wt(model_ref m, tensor x, tensor weight, tensor bias, int stride, return x; } +tensor conv_2d_wt_dw(model_ref m, tensor x, tensor weight, tensor bias, int stride, int pad, + int dilation) { + // ⚠️ **입력을 whcn 연속으로 만들어 넘긴다.** `ggml_compute_forward_conv_2d_dw` 는 + // src 가 contiguous 면 whcn 커널을, contiguous_channels 면 cwhn 커널을 타는데 + // cwhn 쪽은 **kernel 에도** `nb[0] >= nb[2]` 를 요구한다. 동적 커널은 그래프에서 + // torch 순서(`[kw,kh,1,C]` 연속)로 오므로 그 단언에 걸린다 — cwhn 모델이어도 + // 입력을 whcn 으로 돌려 whcn 커널을 타게 하는 편이 레이아웃 가정을 안 만든다. + bool cwhn = bool(m.flags & model_build_flag::cwhn); + x = cwhn ? ggml_cont(m, permute_cwhn_to_whcn(m, x)) : ggml_cont(m, x); + weight = ggml_cont(m, weight); + x = ggml_conv_2d_dw_direct(m, weight, x, stride, stride, pad, pad, dilation, dilation); + if (cwhn) { + x = ggml_cont(m, permute_whcn_to_cwhn(m, x)); + } + if (bias) { + if (!cwhn) { + bias = ggml_reshape_4d(m, bias, 1, 1, bias->ne[0], 1); + } + x = ggml_add_inplace(m, x, bias); + } + return x; +} + tensor conv_2d_grouped(model_ref m, tensor x, int stride, int pad, int dilation, int groups) { if (groups <= 1) { return conv_2d(m, x, stride, pad, dilation); @@ -169,22 +230,32 @@ tensor conv_2d_grouped(model_ref m, tensor x, int stride, int pad, int dilation, return add_bias_2d(m, y); } -tensor conv_2d_depthwise(model_ref m, tensor x, int stride, int pad) { +// `dilation` 은 torch 와 같은 뜻이다. ⚠️ **안 받으면 조용히 커진다** — 호출자가 padding 만 +// 넘기고 dilation 을 못 넘기면 출력이 `in + 2p - (k-1)` 이 되어 입력보다 크게 나오고, +// 그 크기는 **다음 op(concat)에서** 어긋나 죽는다(mmseg DeepLabV3+ 의 depthwise ASPP: +// dilation 12/24/36 인데 64x64 가 86/110/134 로 커졌다). +tensor conv_2d_depthwise_ex(model_ref m, tensor x, int sw, int sh, int pw, int ph, int dw, + int dh) { tensor weight = m.weights("weight"); if (m.flags & model_build_flag::cwhn) { weight = ggml_permute(m, weight, 3, 2, 0, 1); x = permute_cwhn_to_whcn(m, x); - x = ggml_conv_2d_dw_direct(m, weight, x, stride, stride, pad, pad, 1, 1); + x = ggml_conv_2d_dw_direct(m, weight, x, sw, sh, pw, ph, dw, dh); x = permute_whcn_to_cwhn(m, x); } else { - x = ggml_conv_2d_dw_direct(m, weight, x, stride, stride, pad, pad, 1, 1); + x = ggml_conv_2d_dw_direct(m, weight, x, sw, sh, pw, ph, dw, dh); } x = add_bias_2d(m, x); return x; } -tensor conv_transpose_2d(model_ref m, tensor x, int stride, int pad, int groups) { +tensor conv_2d_depthwise(model_ref m, tensor x, int stride, int pad, int dilation) { + return conv_2d_depthwise_ex(m, x, stride, stride, pad, pad, dilation, dilation); +} + +tensor conv_transpose_2d(model_ref m, tensor x, int stride, int pad, int groups, + int output_padding) { tensor weight = m.weights("weight"); // ⚠️ **커널은 F16 이어야 한다.** `ggml_compute_forward_conv_transpose_2d` 는 // `GGML_ASSERT(src0->type == GGML_TYPE_F16)` 로 시작한다(ggml-cpu/ops.cpp). @@ -227,12 +298,28 @@ tensor conv_transpose_2d(model_ref m, tensor x, int stride, int pad, int groups) // **다음 op 에서** 어긋나 죽는다 — 크래시 지점이 원인 지점이 아니다 // (centernet 실측: deconv 가 34x34 를 내고 다음 DCN 의 offset 32 와 안 맞았다). // mmdet 의 deconv 는 전부 대칭 padding 이라 양쪽을 같은 값으로 자르면 된다. - if (pad > 0) { - const int64_t w = x->ne[0] - 2 * pad, h = x->ne[1] - 2 * pad; - GGML_ASSERT(w > 0 && h > 0); - x = ggml_cont(m, ggml_view_4d(m, x, w, h, x->ne[2], x->ne[3], - x->nb[1], x->nb[2], x->nb[3], - pad * x->nb[0] + pad * x->nb[1])); + // ⚠️ **`output_padding` 은 「덜 자른다」로 처리한다.** torch 는 출력 오른쪽·아래를 + // 그만큼 더 갖는데, p0 버퍼는 자르기 전이라 그 자리를 대개 이미 갖고 있다. + // torch 출력 인덱스 j 는 버퍼 인덱스 j+pad 다. 버퍼가 모자라는 만큼 + // (output_padding > pad) 은 실제로 기여가 0 인 자리라 0 으로 채운다. + // 안 처리하면 출력이 output_padding 만큼 작고, 뒤의 residual add 가 + // `ggml_can_repeat` 로 죽는다 — 크래시 지점이 원인 지점이 아니다 + // (erfnet UpsamplerBlock: stride 2 · pad 1 · output_padding 1 → 2n-1 vs 2n). + if (pad > 0 || output_padding > 0) { + const int64_t bw = x->ne[0], bh = x->ne[1]; + const int64_t ow = bw - 2 * pad + output_padding; + const int64_t oh = bh - 2 * pad + output_padding; + GGML_ASSERT(ow > 0 && oh > 0); + const int64_t vw = ow < bw - pad ? ow : bw - pad; // 버퍼에서 꺼낼 수 있는 만큼 + const int64_t vh = oh < bh - pad ? oh : bh - pad; + if (pad > 0 || vw != bw || vh != bh) { + x = ggml_cont(m, ggml_view_4d(m, x, vw, vh, x->ne[2], x->ne[3], + x->nb[1], x->nb[2], x->nb[3], + pad * x->nb[0] + pad * x->nb[1])); + } + if (vw < ow || vh < oh) { + x = ggml_pad(m, x, (int) (ow - vw), (int) (oh - vh), 0, 0); + } } if (m.flags & model_build_flag::cwhn) { x = ggml_cont(m, permute_whcn_to_cwhn(m, x)); @@ -289,7 +376,13 @@ tensor pad_reflect_axis(model_ref m, tensor x, int dim, int lo, int hi) { for (int k = 1; k <= hi; ++k) { parts.push_back(pad_reflect_slice(m, x, dim, n - 1 - k)); } - return ggml_concat_n(m, parts.data(), (int)parts.size(), dim); + // 두 입력 CONCAT 을 차례로 잇는다. GTXPY 의 CONCAT 은 src[0]·src[1] 만 읽어서 + // `ggml_concat_n`(입력 3개 이상)은 ISS·FPGA 에서 reshape 오류로 실패했다. + tensor y = parts[0]; + for (size_t i = 1; i < parts.size(); ++i) { + y = ggml_concat(m, y, parts[i], dim); + } + return y; } } // namespace diff --git a/src/visp/nn.h b/src/visp/nn.h index 27ab87f..953e89a 100644 --- a/src/visp/nn.h +++ b/src/visp/nn.h @@ -30,6 +30,23 @@ tensor space_to_depth_quad(model_ref m, tensor x, int sw, int sh); tensor permute_cwhn_to_whcn(model_ref m, tensor x); tensor permute_whcn_to_cwhn(model_ref m, tensor x); +// 정수 텐서를 f32 로 올린다. +// +// ⚠️ **ggml 의 이항 연산(add/sub/mul/div)은 정수 타입을 모른다.** +// `ggml-cpu/binary-ops.cpp` 가 `binary_op: unsupported types: dst: i32, src0: i32, src1: i32` +// 로 **abort** 한다. 인덱스 계산이 그 자리다 — `ggml_get_rows` 가 I32 를 강제하므로 +// 한 번 i32 가 된 텐서가 그대로 산술까지 흘러온다(mmseg point_rend 의 점 좌표 차). +// +// ⚠️ 그 abort 는 **크래시로 안 보인다.** OpenMP 병렬 구간 안에서 나면 나머지 스레드가 +// 교착에 빠져 프로세스가 CPU 0% 로 매달린다 — 「느린 모델」로 오인된다(12분 관측). +// 행의 지문은 「경과 시간은 느는데 CPU 시간이 0」이다. +// +// 좌표·인덱스는 정수라 f32 로 계산해도 값이 안 깨진다(f32 는 2^24 까지 정확). +// **이미 f32 면 그대로 돌려주므로 비용이 없다** — 그래서 무조건 감싸도 된다. +inline tensor as_f32(model_ref m, tensor x) { + return x->type == GGML_TYPE_F32 ? x : ggml_cast(m, ggml_cont(m, x), GGML_TYPE_F32); +} + // "Contiguous 2D" refers to the layout configured in `m` model flags, ie. the preferred // memory layout for 2D operations like convolution. inline bool is_whcn(model_ref m) { return !(m.flags & model_build_flag::cwhn); } @@ -49,6 +66,11 @@ std::array nelements_whcn(model_ref const&, tensor t); // Input and weight are expected to be in "contiguous 2D" layout as configured in `m`. tensor conv_2d(model_ref m, tensor x, int stride = 1, int pad = 0, int dilation = 1); +// 축별 conv — H 와 W 의 padding/stride/dilation 이 다른 경우(ERFNet 의 3x1 · 1x3 분리 conv). +// ⚠️ 인자는 ggml 과 같은 **W 먼저**다. torch 의 (H, W) 순서와 반대이므로 렌더러에서 뒤집는다. +tensor conv_2d_ex(model_ref m, tensor x, int stride_w, int stride_h, int pad_w, int pad_h, + int dilation_w = 1, int dilation_h = 1); + // grouped conv (nn.Conv2d(groups=g)). groups<=1 이면 conv_2d 와 동일. // ggml 에 grouped conv 커널이 없어 그룹별 view → conv → 채널축 concat 으로 분해한다. tensor conv_2d_grouped(model_ref m, tensor x, int stride = 1, int pad = 0, @@ -59,16 +81,37 @@ tensor conv_2d_grouped(model_ref m, tensor x, int stride = 1, int pad = 0, tensor conv_2d_wt(model_ref m, tensor x, tensor weight, tensor bias, int stride = 1, int pad = 0, int dilation = 1); -tensor conv_2d_depthwise(model_ref m, tensor x, int stride = 1, int pad = 0); +// 동적 weight 의 **depthwise** conv (groups == 채널수). DMNet 의 DCM 이 이 자리다 — +// 필터를 입력에서 만들어 채널마다 다른 커널로 돈다. +// weight 는 torch `[C,1,kh,kw]` → ggml `[kw,kh,1,C]` 로 그대로 온다(GGUF 경유가 아니라 +// 그래프 텐서라 `conv_2d_depthwise` 의 레이아웃 되돌림이 필요 없다). +tensor conv_2d_wt_dw(model_ref m, tensor x, tensor weight, tensor bias, + int stride = 1, int pad = 0, int dilation = 1); + +tensor conv_2d_depthwise(model_ref m, tensor x, int stride = 1, int pad = 0, int dilation = 1); + +// 축별 depthwise conv — 커널·padding 이 H 와 W 에서 다른 strip conv 용 +// (SegNeXt MSCAN 의 1x7/7x1 · 1x11/11x1 · 1x21/21x1). 인자 순서는 ggml 과 같은 **W 먼저**다. +// ⚠️ 단일 int 헬퍼로 내면 한쪽 padding 이 통째로 버려지거나 없던 쪽에 붙는다. +tensor conv_2d_depthwise_ex(model_ref m, tensor x, int stride_w, int stride_h, int pad_w, + int pad_h, int dilation_w, int dilation_h); tensor conv_2d_deform( model_ref m, tensor x, tensor weight, tensor offset, tensor mask, int stride, int pad); // `pad` 는 torch 의 ConvTranspose2d padding 과 같은 뜻이다(출력 가장자리를 그만큼 버린다). // ggml 에는 padding 을 받는 conv_transpose 가 없어 여기서 잘라낸다. // `groups` 는 torch 와 같은 뜻이다. ggml 에 grouped conv_transpose 가 없어 그룹마다 // 커널·입력을 잘라 돌리고 채널로 이어붙인다. -tensor conv_transpose_2d(model_ref m, tensor x, int stride, int pad = 0, int groups = 1); +// `output_padding` 도 torch 와 같은 뜻이다(출력 오른쪽·아래에 그만큼 더 붙인다). +// ⚠️ 렌더러와 **같이** 고쳐야 한다 — 한쪽만 고치면 크기가 1 어긋난 채 조용히 돈다. +tensor conv_transpose_2d( + model_ref m, tensor x, int stride, int pad = 0, int groups = 1, int output_padding = 0); tensor batch_norm_2d(model_ref, tensor x); +// PReLU. 기울기 `weight` 는 채널당 하나(또는 전체 하나)다. +// ⚠️ `ggml_leaky_relu(0.25)` 로 근사하지 마라 — 0.25 는 torch 기본 초기값이라 +// **랜덤 가중치에서만 맞는다.** 학습된 모델에서 조용히 틀린다. +tensor prelu(model_ref m, tensor x); + // 2D image to patch embedding using convolution and optional norm. CWHN input and output. tensor patch_embed(model_ref, tensor x, int patch_size); diff --git a/src/visp/postproc.cpp b/src/visp/postproc.cpp index fe02fd2..783c554 100755 --- a/src/visp/postproc.cpp +++ b/src/visp/postproc.cpp @@ -700,18 +700,18 @@ std::vector roi_align( // ── 전처리: 이미지(HWC u8) → 모델 입력(CWHN f32) ───────────────────────────── std::vector preprocess(uint8_t const* img, int img_h, int img_w, int img_c, - int out_size, float const mean[3], float const std[3], + int dst_w, int dst_h, float const mean[3], float const std[3], bool to_rgb, int* out_w, int* out_h) { - if (out_w) *out_w = out_size; - if (out_h) *out_h = out_size; - // bilinear resize → out_size×out_size, normalize, CWHN(index=(h*W+w)*C+c) - std::vector out((size_t)out_size * out_size * 3); - float sh = (float)img_h / out_size, sw = (float)img_w / out_size; - for (int oh = 0; oh < out_size; ++oh) { + if (out_w) *out_w = dst_w; + if (out_h) *out_h = dst_h; + // bilinear resize → dst_w×dst_h, normalize, CWHN(index=(h*W+w)*C+c) + std::vector out((size_t)dst_w * dst_h * 3); + float sh = (float)img_h / dst_h, sw = (float)img_w / dst_w; + for (int oh = 0; oh < dst_h; ++oh) { float fy = (oh + 0.5f) * sh - 0.5f; int y0 = (int)std::floor(fy); float wy = fy - y0; int y0c = std::min(std::max(y0, 0), img_h - 1), y1c = std::min(y0 + 1, img_h - 1); - for (int ow = 0; ow < out_size; ++ow) { + for (int ow = 0; ow < dst_w; ++ow) { float fx = (ow + 0.5f) * sw - 0.5f; int x0 = (int)std::floor(fx); float wx = fx - x0; int x0c = std::min(std::max(x0, 0), img_w - 1), x1c = std::min(x0 + 1, img_w - 1); @@ -722,89 +722,31 @@ std::vector preprocess(uint8_t const* img, int img_h, int img_w, int img_ }; float v = at(y0c, x0c) * (1 - wy) * (1 - wx) + at(y0c, x1c) * (1 - wy) * wx + at(y1c, x0c) * wy * (1 - wx) + at(y1c, x1c) * wy * wx; - out[((size_t)oh * out_size + ow) * 3 + c] = (v - mean[c]) / std[c]; + out[((size_t)oh * dst_w + ow) * 3 + c] = (v - mean[c]) / std[c]; } } } return out; } - -// ── YOLO dense (v8/v10/v26) ───────────────────────────────────────────────── -std::vector detect_yolo_dense( - float const* box, float const* score, - std::vector> const& feat_hw, yolo_dense_params const& p) { - - // 레벨별 격자점을 이어 붙인다. 순서는 그래프가 concat 한 순서와 **같아야** 한다 - // (레벨 0 = 가장 큰 feature map). 어긋나면 박스가 통째로 엉뚱한 데 찍힌다. - std::vector points; - std::vector pt_stride; - size_t n_total = 0; - for (size_t l = 0; l < feat_hw.size(); ++l) { - const int H = feat_hw[l].first, W = feat_hw[l].second; - const float s = l < p.strides.size() ? p.strides[l] : p.strides.back(); - std::vector pts = gen_points(H, W, s, p.point_offset); - points.insert(points.end(), pts.begin(), pts.end()); - pt_stride.insert(pt_stride.end(), (size_t)H * W, s); - n_total += (size_t)H * W; - } - const int N = (int)n_total; - if (N == 0) { - return {}; - } - - // ltrb 는 **격자 단위**다 — stride 를 곱해야 픽셀이 된다. - // 그래프 덤프는 채널 우선(flat[c*N+a])이고 `distance2bbox` 는 앵커 우선([i*4+k])이라 - // 여기서 전치도 같이 한다. - std::vector dist((size_t)N * 4); - for (int i = 0; i < N; ++i) { - const float s = pt_stride[i]; - for (int k = 0; k < 4; ++k) { - dist[(size_t)i * 4 + k] = box[(size_t)k * N + i] * s; - } - } - std::vector xyxy((size_t)N * 4); - distance2bbox(points.data(), dist.data(), N, xyxy.data(), p.input_w, p.input_h); - - // 클래스 로짓 → 시그모이드. 앵커마다 최고 클래스만 남긴다(YOLO 규약). - std::vector dets; - dets.reserve(256); - for (int i = 0; i < N; ++i) { - int best = 0; - float best_logit = score[(size_t)0 * N + i]; - for (int c = 1; c < p.num_classes; ++c) { - const float v = score[(size_t)c * N + i]; - if (v > best_logit) { - best_logit = v; - best = c; - } - } - // 시그모이드는 **최댓값 하나만** 계산한다 — 단조라 argmax 가 안 바뀐다. - const float sc = 1.0f / (1.0f + std::exp(-best_logit)); - if (sc < p.score_thr) { - continue; - } - dets.push_back(detection{xyxy[(size_t)i * 4 + 0], xyxy[(size_t)i * 4 + 1], - xyxy[(size_t)i * 4 + 2], xyxy[(size_t)i * 4 + 3], sc, best}); - } - - std::sort(dets.begin(), dets.end(), - [](detection const& a, detection const& b) { return a.score > b.score; }); - - // one2one(YOLOv10/26) 은 중복을 모델이 이미 없앴다 — NMS 를 또 걸면 겹친 물체를 지운다. - if (!p.nms_free) { - std::vector keep = nms(dets, p.nms_thr); - std::vector out; - out.reserve(keep.size()); - for (int idx : keep) { - out.push_back(dets[idx]); - } - dets.swap(out); - } - if ((int)dets.size() > p.max_det) { - dets.resize(p.max_det); - } - return dets; +std::vector preprocess_keep_ratio(uint8_t const* img, int img_h, int img_w, int img_c, + int dst_w, int dst_h, float const mean[3], + float const std[3], bool to_rgb, float const pad_value[3], + float* scale_x, float* scale_y) { + // mmcv `rescale_size`: 새 크기는 int(w*f + 0.5). + const double f = std::min(double(dst_w) / img_w, double(dst_h) / img_h); + const int rw = std::clamp(int(img_w * f + 0.5), 1, dst_w); + const int rh = std::clamp(int(img_h * f + 0.5), 1, dst_h); + if (scale_x) *scale_x = float(rw) / float(img_w); + if (scale_y) *scale_y = float(rh) / float(img_h); + std::vector resized = preprocess(img, img_h, img_w, img_c, rw, rh, mean, std, to_rgb); + std::vector out((size_t)dst_w * dst_h * 3); + for (size_t i = 0; i < (size_t)dst_w * dst_h; ++i) + for (int c = 0; c < 3; ++c) out[i * 3 + c] = (pad_value[c] - mean[c]) / std[c]; + for (int y = 0; y < rh; ++y) + std::copy_n(resized.begin() + (size_t)y * rw * 3, (size_t)rw * 3, + out.begin() + (size_t)y * dst_w * 3); + return out; } // ── YOLOv3 ─────────────────────────────────────────────────────────────────── diff --git a/src/visp/postproc.h b/src/visp/postproc.h index 12598eb..1a9f5c1 100755 --- a/src/visp/postproc.h +++ b/src/visp/postproc.h @@ -198,25 +198,6 @@ std::vector detect_yolox( std::vector> const& obj, std::vector> const& feat_hw, yolox_params const& p); -// ── YOLO dense (v8/v10/v26 계열, 격자 ltrb) ───────────────────────────────── -// g2c 생성 그래프는 **디코드 앞에서 끊는다** — 격자 단위 ltrb 와 클래스 **로짓**을 낸다. -// 여기서 stride 곱 · 격자점 더하기 · 시그모이드 · top-k 를 한다(전부 호스트). -struct yolo_dense_params { - std::vector strides{8, 16, 32}; - float point_offset = 0.5f; // ultralytics make_anchors 규약 - int num_classes = 80; - float score_thr = 0.25f; - float nms_thr = 0.7f; - bool nms_free = true; // one2one(YOLOv10/26) 은 NMS 없이 top-k 만 - int max_det = 300; - int input_w = 0, input_h = 0; -}; -// box[4*N] · score[nc*N] 은 **채널 우선**(flat[c*N + a]) — 그래프 덤프 레이아웃 그대로다. -// N = Σ(W_l·H_l). feat_hw 는 레벨별 (H, W). -std::vector detect_yolo_dense( - float const* box, float const* score, - std::vector> const& feat_hw, yolo_dense_params const& p); - // ── SABL (Side-Aware Boundary Localization) ───────────────────────────────── // 박스를 델타로 한 번에 내지 않는다. **변(l,r,t,d)마다** 앵커를 `num_buckets` 칸으로 나눠 // ① 어느 칸인지 **분류**하고 ② 그 칸 안의 **오프셋**을 회귀한다. 그래서 box 갈래가 둘이다. @@ -392,9 +373,33 @@ std::vector roi_align( float const* level_rois = nullptr); // ── 전처리: 이미지(HWC u8) → 모델 입력 텐서(CWHN f32) ─────────────────────── -// resize(size×size) + normalize((v-mean)/std). BGR/RGB·mean/std 는 인자. +// resize(dst_w×dst_h) + normalize((v-mean)/std). BGR/RGB·mean/std 는 인자. +// +// ⚠️ **가로·세로를 따로 받는다.** 예전엔 `out_size` 하나로 정사각 리사이즈만 했는데, +// 그러면 비정사각 모델이 **조용히 틀린다** — mmseg cityscapes(512×1024)를 넣으면 +// 512×512 로 줄여 출력이 절반만 나오고 에러는 안 난다(2026-09-01 실측). +// mmpose 는 전부 256×192 라 하나도 못 돌았다. +// 정사각 모델은 두 인자에 같은 값을 주면 예전과 동일하다. std::vector preprocess(uint8_t const* img, int img_h, int img_w, int img_c, - int out_size, float const mean[3], float const std[3], + int dst_w, int dst_h, float const mean[3], float const std[3], bool to_rgb, int* out_w = nullptr, int* out_h = nullptr); +// 비율 유지 전처리 — MMDetection `Resize(keep_ratio=True)` + `Pad`. +// 배율 f = min(dst_w/img_w, dst_h/img_h) 로 줄인 이미지를 **왼쪽 위**에 두고, 나머지를 +// `pad_value`(정규화 전 픽셀 값)로 채운다. mmdet 은 오른쪽·아래만 채우므로 가운데 정렬 +// letterbox 와 다르다. `scale_x`/`scale_y` 는 (리사이즈 크기 / 원본 크기)이며, 박스를 +// 원본 좌표로 되돌릴 때 이 값으로 나눈다. +std::vector preprocess_keep_ratio(uint8_t const* img, int img_h, int img_w, int img_c, + int dst_w, int dst_h, float const mean[3], + float const std[3], bool to_rgb, float const pad_value[3], + float* scale_x, float* scale_y); + +// 정사각 편의 오버로드 — 기존 호출부 호환. +inline std::vector preprocess(uint8_t const* img, int img_h, int img_w, int img_c, + int out_size, float const mean[3], float const std[3], + bool to_rgb, int* out_w = nullptr, int* out_h = nullptr) { + return preprocess(img, img_h, img_w, img_c, out_size, out_size, mean, std, + to_rgb, out_w, out_h); +} + } // namespace visp diff --git a/src/visp/runtime_constants.inc b/src/visp/runtime_constants.inc new file mode 100644 index 0000000..a021233 --- /dev/null +++ b/src/visp/runtime_constants.inc @@ -0,0 +1,229 @@ +// GGML enum constants in the recipe expression ABI. + constants.emplace("GGML_STATUS_ALLOC_FAILED", Value(int64_t(GGML_STATUS_ALLOC_FAILED))); + constants.emplace("GGML_STATUS_FAILED", Value(int64_t(GGML_STATUS_FAILED))); + constants.emplace("GGML_STATUS_SUCCESS", Value(int64_t(GGML_STATUS_SUCCESS))); + constants.emplace("GGML_STATUS_ABORTED", Value(int64_t(GGML_STATUS_ABORTED))); + constants.emplace("GGML_TYPE_F32", Value(int64_t(GGML_TYPE_F32))); + constants.emplace("GGML_TYPE_F16", Value(int64_t(GGML_TYPE_F16))); + constants.emplace("GGML_TYPE_Q4_0", Value(int64_t(GGML_TYPE_Q4_0))); + constants.emplace("GGML_TYPE_Q4_1", Value(int64_t(GGML_TYPE_Q4_1))); + constants.emplace("GGML_TYPE_Q5_0", Value(int64_t(GGML_TYPE_Q5_0))); + constants.emplace("GGML_TYPE_Q5_1", Value(int64_t(GGML_TYPE_Q5_1))); + constants.emplace("GGML_TYPE_Q8_0", Value(int64_t(GGML_TYPE_Q8_0))); + constants.emplace("GGML_TYPE_Q8_1", Value(int64_t(GGML_TYPE_Q8_1))); + constants.emplace("GGML_TYPE_Q2_K", Value(int64_t(GGML_TYPE_Q2_K))); + constants.emplace("GGML_TYPE_Q3_K", Value(int64_t(GGML_TYPE_Q3_K))); + constants.emplace("GGML_TYPE_Q4_K", Value(int64_t(GGML_TYPE_Q4_K))); + constants.emplace("GGML_TYPE_Q5_K", Value(int64_t(GGML_TYPE_Q5_K))); + constants.emplace("GGML_TYPE_Q6_K", Value(int64_t(GGML_TYPE_Q6_K))); + constants.emplace("GGML_TYPE_Q8_K", Value(int64_t(GGML_TYPE_Q8_K))); + constants.emplace("GGML_TYPE_IQ2_XXS", Value(int64_t(GGML_TYPE_IQ2_XXS))); + constants.emplace("GGML_TYPE_IQ2_XS", Value(int64_t(GGML_TYPE_IQ2_XS))); + constants.emplace("GGML_TYPE_IQ3_XXS", Value(int64_t(GGML_TYPE_IQ3_XXS))); + constants.emplace("GGML_TYPE_IQ1_S", Value(int64_t(GGML_TYPE_IQ1_S))); + constants.emplace("GGML_TYPE_IQ4_NL", Value(int64_t(GGML_TYPE_IQ4_NL))); + constants.emplace("GGML_TYPE_IQ3_S", Value(int64_t(GGML_TYPE_IQ3_S))); + constants.emplace("GGML_TYPE_IQ2_S", Value(int64_t(GGML_TYPE_IQ2_S))); + constants.emplace("GGML_TYPE_IQ4_XS", Value(int64_t(GGML_TYPE_IQ4_XS))); + constants.emplace("GGML_TYPE_I8", Value(int64_t(GGML_TYPE_I8))); + constants.emplace("GGML_TYPE_I16", Value(int64_t(GGML_TYPE_I16))); + constants.emplace("GGML_TYPE_I32", Value(int64_t(GGML_TYPE_I32))); + constants.emplace("GGML_TYPE_I64", Value(int64_t(GGML_TYPE_I64))); + constants.emplace("GGML_TYPE_F64", Value(int64_t(GGML_TYPE_F64))); + constants.emplace("GGML_TYPE_IQ1_M", Value(int64_t(GGML_TYPE_IQ1_M))); + constants.emplace("GGML_TYPE_BF16", Value(int64_t(GGML_TYPE_BF16))); + constants.emplace("GGML_TYPE_TQ1_0", Value(int64_t(GGML_TYPE_TQ1_0))); + constants.emplace("GGML_TYPE_TQ2_0", Value(int64_t(GGML_TYPE_TQ2_0))); + constants.emplace("GGML_TYPE_MXFP4", Value(int64_t(GGML_TYPE_MXFP4))); + constants.emplace("GGML_TYPE_NVFP4", Value(int64_t(GGML_TYPE_NVFP4))); + constants.emplace("GGML_TYPE_COUNT", Value(int64_t(GGML_TYPE_COUNT))); + constants.emplace("GGML_PREC_DEFAULT", Value(int64_t(GGML_PREC_DEFAULT))); + constants.emplace("GGML_PREC_F32", Value(int64_t(GGML_PREC_F32))); + constants.emplace("GGML_FTYPE_UNKNOWN", Value(int64_t(GGML_FTYPE_UNKNOWN))); + constants.emplace("GGML_FTYPE_ALL_F32", Value(int64_t(GGML_FTYPE_ALL_F32))); + constants.emplace("GGML_FTYPE_MOSTLY_F16", Value(int64_t(GGML_FTYPE_MOSTLY_F16))); + constants.emplace("GGML_FTYPE_MOSTLY_Q4_0", Value(int64_t(GGML_FTYPE_MOSTLY_Q4_0))); + constants.emplace("GGML_FTYPE_MOSTLY_Q4_1", Value(int64_t(GGML_FTYPE_MOSTLY_Q4_1))); + constants.emplace("GGML_FTYPE_MOSTLY_Q4_1_SOME_F16", Value(int64_t(GGML_FTYPE_MOSTLY_Q4_1_SOME_F16))); + constants.emplace("GGML_FTYPE_MOSTLY_Q8_0", Value(int64_t(GGML_FTYPE_MOSTLY_Q8_0))); + constants.emplace("GGML_FTYPE_MOSTLY_Q5_0", Value(int64_t(GGML_FTYPE_MOSTLY_Q5_0))); + constants.emplace("GGML_FTYPE_MOSTLY_Q5_1", Value(int64_t(GGML_FTYPE_MOSTLY_Q5_1))); + constants.emplace("GGML_FTYPE_MOSTLY_Q2_K", Value(int64_t(GGML_FTYPE_MOSTLY_Q2_K))); + constants.emplace("GGML_FTYPE_MOSTLY_Q3_K", Value(int64_t(GGML_FTYPE_MOSTLY_Q3_K))); + constants.emplace("GGML_FTYPE_MOSTLY_Q4_K", Value(int64_t(GGML_FTYPE_MOSTLY_Q4_K))); + constants.emplace("GGML_FTYPE_MOSTLY_Q5_K", Value(int64_t(GGML_FTYPE_MOSTLY_Q5_K))); + constants.emplace("GGML_FTYPE_MOSTLY_Q6_K", Value(int64_t(GGML_FTYPE_MOSTLY_Q6_K))); + constants.emplace("GGML_FTYPE_MOSTLY_IQ2_XXS", Value(int64_t(GGML_FTYPE_MOSTLY_IQ2_XXS))); + constants.emplace("GGML_FTYPE_MOSTLY_IQ2_XS", Value(int64_t(GGML_FTYPE_MOSTLY_IQ2_XS))); + constants.emplace("GGML_FTYPE_MOSTLY_IQ3_XXS", Value(int64_t(GGML_FTYPE_MOSTLY_IQ3_XXS))); + constants.emplace("GGML_FTYPE_MOSTLY_IQ1_S", Value(int64_t(GGML_FTYPE_MOSTLY_IQ1_S))); + constants.emplace("GGML_FTYPE_MOSTLY_IQ4_NL", Value(int64_t(GGML_FTYPE_MOSTLY_IQ4_NL))); + constants.emplace("GGML_FTYPE_MOSTLY_IQ3_S", Value(int64_t(GGML_FTYPE_MOSTLY_IQ3_S))); + constants.emplace("GGML_FTYPE_MOSTLY_IQ2_S", Value(int64_t(GGML_FTYPE_MOSTLY_IQ2_S))); + constants.emplace("GGML_FTYPE_MOSTLY_IQ4_XS", Value(int64_t(GGML_FTYPE_MOSTLY_IQ4_XS))); + constants.emplace("GGML_FTYPE_MOSTLY_IQ1_M", Value(int64_t(GGML_FTYPE_MOSTLY_IQ1_M))); + constants.emplace("GGML_FTYPE_MOSTLY_BF16", Value(int64_t(GGML_FTYPE_MOSTLY_BF16))); + constants.emplace("GGML_FTYPE_MOSTLY_MXFP4", Value(int64_t(GGML_FTYPE_MOSTLY_MXFP4))); + constants.emplace("GGML_FTYPE_MOSTLY_NVFP4", Value(int64_t(GGML_FTYPE_MOSTLY_NVFP4))); + constants.emplace("GGML_OP_NONE", Value(int64_t(GGML_OP_NONE))); + constants.emplace("GGML_OP_DUP", Value(int64_t(GGML_OP_DUP))); + constants.emplace("GGML_OP_ADD", Value(int64_t(GGML_OP_ADD))); + constants.emplace("GGML_OP_ADD_ID", Value(int64_t(GGML_OP_ADD_ID))); + constants.emplace("GGML_OP_ADD1", Value(int64_t(GGML_OP_ADD1))); + constants.emplace("GGML_OP_ACC", Value(int64_t(GGML_OP_ACC))); + constants.emplace("GGML_OP_SUB", Value(int64_t(GGML_OP_SUB))); + constants.emplace("GGML_OP_MUL", Value(int64_t(GGML_OP_MUL))); + constants.emplace("GGML_OP_DIV", Value(int64_t(GGML_OP_DIV))); + constants.emplace("GGML_OP_SQR", Value(int64_t(GGML_OP_SQR))); + constants.emplace("GGML_OP_SQRT", Value(int64_t(GGML_OP_SQRT))); + constants.emplace("GGML_OP_LOG", Value(int64_t(GGML_OP_LOG))); + constants.emplace("GGML_OP_SIN", Value(int64_t(GGML_OP_SIN))); + constants.emplace("GGML_OP_COS", Value(int64_t(GGML_OP_COS))); + constants.emplace("GGML_OP_SUM", Value(int64_t(GGML_OP_SUM))); + constants.emplace("GGML_OP_SUM_ROWS", Value(int64_t(GGML_OP_SUM_ROWS))); + constants.emplace("GGML_OP_CUMSUM", Value(int64_t(GGML_OP_CUMSUM))); + constants.emplace("GGML_OP_MEAN", Value(int64_t(GGML_OP_MEAN))); + constants.emplace("GGML_OP_ARGMAX", Value(int64_t(GGML_OP_ARGMAX))); + constants.emplace("GGML_OP_COUNT_EQUAL", Value(int64_t(GGML_OP_COUNT_EQUAL))); + constants.emplace("GGML_OP_REPEAT", Value(int64_t(GGML_OP_REPEAT))); + constants.emplace("GGML_OP_REPEAT_BACK", Value(int64_t(GGML_OP_REPEAT_BACK))); + constants.emplace("GGML_OP_CONCAT", Value(int64_t(GGML_OP_CONCAT))); + constants.emplace("GGML_OP_SILU_BACK", Value(int64_t(GGML_OP_SILU_BACK))); + constants.emplace("GGML_OP_NORM", Value(int64_t(GGML_OP_NORM))); + constants.emplace("GGML_OP_RMS_NORM", Value(int64_t(GGML_OP_RMS_NORM))); + constants.emplace("GGML_OP_RMS_NORM_BACK", Value(int64_t(GGML_OP_RMS_NORM_BACK))); + constants.emplace("GGML_OP_GROUP_NORM", Value(int64_t(GGML_OP_GROUP_NORM))); + constants.emplace("GGML_OP_L2_NORM", Value(int64_t(GGML_OP_L2_NORM))); + constants.emplace("GGML_OP_MUL_MAT", Value(int64_t(GGML_OP_MUL_MAT))); + constants.emplace("GGML_OP_MUL_MAT_ID", Value(int64_t(GGML_OP_MUL_MAT_ID))); + constants.emplace("GGML_OP_OUT_PROD", Value(int64_t(GGML_OP_OUT_PROD))); + constants.emplace("GGML_OP_SCALE", Value(int64_t(GGML_OP_SCALE))); + constants.emplace("GGML_OP_SET", Value(int64_t(GGML_OP_SET))); + constants.emplace("GGML_OP_CPY", Value(int64_t(GGML_OP_CPY))); + constants.emplace("GGML_OP_CONT", Value(int64_t(GGML_OP_CONT))); + constants.emplace("GGML_OP_RESHAPE", Value(int64_t(GGML_OP_RESHAPE))); + constants.emplace("GGML_OP_VIEW", Value(int64_t(GGML_OP_VIEW))); + constants.emplace("GGML_OP_PERMUTE", Value(int64_t(GGML_OP_PERMUTE))); + constants.emplace("GGML_OP_TRANSPOSE", Value(int64_t(GGML_OP_TRANSPOSE))); + constants.emplace("GGML_OP_GET_ROWS", Value(int64_t(GGML_OP_GET_ROWS))); + constants.emplace("GGML_OP_GET_ROWS_BACK", Value(int64_t(GGML_OP_GET_ROWS_BACK))); + constants.emplace("GGML_OP_SET_ROWS", Value(int64_t(GGML_OP_SET_ROWS))); + constants.emplace("GGML_OP_DIAG", Value(int64_t(GGML_OP_DIAG))); + constants.emplace("GGML_OP_DIAG_MASK_INF", Value(int64_t(GGML_OP_DIAG_MASK_INF))); + constants.emplace("GGML_OP_DIAG_MASK_ZERO", Value(int64_t(GGML_OP_DIAG_MASK_ZERO))); + constants.emplace("GGML_OP_SOFT_MAX", Value(int64_t(GGML_OP_SOFT_MAX))); + constants.emplace("GGML_OP_SOFT_MAX_BACK", Value(int64_t(GGML_OP_SOFT_MAX_BACK))); + constants.emplace("GGML_OP_ROPE", Value(int64_t(GGML_OP_ROPE))); + constants.emplace("GGML_OP_ROPE_BACK", Value(int64_t(GGML_OP_ROPE_BACK))); + constants.emplace("GGML_OP_CLAMP", Value(int64_t(GGML_OP_CLAMP))); + constants.emplace("GGML_OP_CONV_TRANSPOSE_1D", Value(int64_t(GGML_OP_CONV_TRANSPOSE_1D))); + constants.emplace("GGML_OP_IM2COL", Value(int64_t(GGML_OP_IM2COL))); + constants.emplace("GGML_OP_IM2COL_BACK", Value(int64_t(GGML_OP_IM2COL_BACK))); + constants.emplace("GGML_OP_IM2COL_3D", Value(int64_t(GGML_OP_IM2COL_3D))); + constants.emplace("GGML_OP_CONV_2D", Value(int64_t(GGML_OP_CONV_2D))); + constants.emplace("GGML_OP_CONV_3D", Value(int64_t(GGML_OP_CONV_3D))); + constants.emplace("GGML_OP_CONV_2D_DW", Value(int64_t(GGML_OP_CONV_2D_DW))); + constants.emplace("GGML_OP_CONV_TRANSPOSE_2D", Value(int64_t(GGML_OP_CONV_TRANSPOSE_2D))); + constants.emplace("GGML_OP_POOL_1D", Value(int64_t(GGML_OP_POOL_1D))); + constants.emplace("GGML_OP_POOL_2D", Value(int64_t(GGML_OP_POOL_2D))); + constants.emplace("GGML_OP_POOL_2D_BACK", Value(int64_t(GGML_OP_POOL_2D_BACK))); + constants.emplace("GGML_OP_UPSCALE", Value(int64_t(GGML_OP_UPSCALE))); + constants.emplace("GGML_OP_PAD", Value(int64_t(GGML_OP_PAD))); + constants.emplace("GGML_OP_PAD_REFLECT_1D", Value(int64_t(GGML_OP_PAD_REFLECT_1D))); + constants.emplace("GGML_OP_ROLL", Value(int64_t(GGML_OP_ROLL))); + constants.emplace("GGML_OP_ARANGE", Value(int64_t(GGML_OP_ARANGE))); + constants.emplace("GGML_OP_TIMESTEP_EMBEDDING", Value(int64_t(GGML_OP_TIMESTEP_EMBEDDING))); + constants.emplace("GGML_OP_ARGSORT", Value(int64_t(GGML_OP_ARGSORT))); + constants.emplace("GGML_OP_TOP_K", Value(int64_t(GGML_OP_TOP_K))); + constants.emplace("GGML_OP_LEAKY_RELU", Value(int64_t(GGML_OP_LEAKY_RELU))); + constants.emplace("GGML_OP_TRI", Value(int64_t(GGML_OP_TRI))); + constants.emplace("GGML_OP_FILL", Value(int64_t(GGML_OP_FILL))); + constants.emplace("GGML_OP_FLASH_ATTN_EXT", Value(int64_t(GGML_OP_FLASH_ATTN_EXT))); + constants.emplace("GGML_OP_FLASH_ATTN_BACK", Value(int64_t(GGML_OP_FLASH_ATTN_BACK))); + constants.emplace("GGML_OP_SSM_CONV", Value(int64_t(GGML_OP_SSM_CONV))); + constants.emplace("GGML_OP_SSM_SCAN", Value(int64_t(GGML_OP_SSM_SCAN))); + constants.emplace("GGML_OP_WIN_PART", Value(int64_t(GGML_OP_WIN_PART))); + constants.emplace("GGML_OP_WIN_UNPART", Value(int64_t(GGML_OP_WIN_UNPART))); + constants.emplace("GGML_OP_GET_REL_POS", Value(int64_t(GGML_OP_GET_REL_POS))); + constants.emplace("GGML_OP_ADD_REL_POS", Value(int64_t(GGML_OP_ADD_REL_POS))); + constants.emplace("GGML_OP_RWKV_WKV6", Value(int64_t(GGML_OP_RWKV_WKV6))); + constants.emplace("GGML_OP_GATED_LINEAR_ATTN", Value(int64_t(GGML_OP_GATED_LINEAR_ATTN))); + constants.emplace("GGML_OP_RWKV_WKV7", Value(int64_t(GGML_OP_RWKV_WKV7))); + constants.emplace("GGML_OP_SOLVE_TRI", Value(int64_t(GGML_OP_SOLVE_TRI))); + constants.emplace("GGML_OP_GATED_DELTA_NET", Value(int64_t(GGML_OP_GATED_DELTA_NET))); + constants.emplace("GGML_OP_UNARY", Value(int64_t(GGML_OP_UNARY))); + constants.emplace("GGML_OP_MAP_CUSTOM1", Value(int64_t(GGML_OP_MAP_CUSTOM1))); + constants.emplace("GGML_OP_MAP_CUSTOM2", Value(int64_t(GGML_OP_MAP_CUSTOM2))); + constants.emplace("GGML_OP_MAP_CUSTOM3", Value(int64_t(GGML_OP_MAP_CUSTOM3))); + constants.emplace("GGML_OP_CUSTOM", Value(int64_t(GGML_OP_CUSTOM))); + constants.emplace("GGML_OP_CROSS_ENTROPY_LOSS", Value(int64_t(GGML_OP_CROSS_ENTROPY_LOSS))); + constants.emplace("GGML_OP_CROSS_ENTROPY_LOSS_BACK", Value(int64_t(GGML_OP_CROSS_ENTROPY_LOSS_BACK))); + constants.emplace("GGML_OP_OPT_STEP_ADAMW", Value(int64_t(GGML_OP_OPT_STEP_ADAMW))); + constants.emplace("GGML_OP_OPT_STEP_SGD", Value(int64_t(GGML_OP_OPT_STEP_SGD))); + constants.emplace("GGML_OP_GLU", Value(int64_t(GGML_OP_GLU))); + constants.emplace("GGML_OP_CONV_2D_DEFORM", Value(int64_t(GGML_OP_CONV_2D_DEFORM))); + constants.emplace("GGML_OP_COUNT", Value(int64_t(GGML_OP_COUNT))); + constants.emplace("GGML_UNARY_OP_ABS", Value(int64_t(GGML_UNARY_OP_ABS))); + constants.emplace("GGML_UNARY_OP_SGN", Value(int64_t(GGML_UNARY_OP_SGN))); + constants.emplace("GGML_UNARY_OP_NEG", Value(int64_t(GGML_UNARY_OP_NEG))); + constants.emplace("GGML_UNARY_OP_STEP", Value(int64_t(GGML_UNARY_OP_STEP))); + constants.emplace("GGML_UNARY_OP_TANH", Value(int64_t(GGML_UNARY_OP_TANH))); + constants.emplace("GGML_UNARY_OP_ELU", Value(int64_t(GGML_UNARY_OP_ELU))); + constants.emplace("GGML_UNARY_OP_RELU", Value(int64_t(GGML_UNARY_OP_RELU))); + constants.emplace("GGML_UNARY_OP_SIGMOID", Value(int64_t(GGML_UNARY_OP_SIGMOID))); + constants.emplace("GGML_UNARY_OP_GELU", Value(int64_t(GGML_UNARY_OP_GELU))); + constants.emplace("GGML_UNARY_OP_GELU_QUICK", Value(int64_t(GGML_UNARY_OP_GELU_QUICK))); + constants.emplace("GGML_UNARY_OP_SILU", Value(int64_t(GGML_UNARY_OP_SILU))); + constants.emplace("GGML_UNARY_OP_HARDSWISH", Value(int64_t(GGML_UNARY_OP_HARDSWISH))); + constants.emplace("GGML_UNARY_OP_HARDSIGMOID", Value(int64_t(GGML_UNARY_OP_HARDSIGMOID))); + constants.emplace("GGML_UNARY_OP_EXP", Value(int64_t(GGML_UNARY_OP_EXP))); + constants.emplace("GGML_UNARY_OP_EXPM1", Value(int64_t(GGML_UNARY_OP_EXPM1))); + constants.emplace("GGML_UNARY_OP_SOFTPLUS", Value(int64_t(GGML_UNARY_OP_SOFTPLUS))); + constants.emplace("GGML_UNARY_OP_GELU_ERF", Value(int64_t(GGML_UNARY_OP_GELU_ERF))); + constants.emplace("GGML_UNARY_OP_XIELU", Value(int64_t(GGML_UNARY_OP_XIELU))); + constants.emplace("GGML_UNARY_OP_FLOOR", Value(int64_t(GGML_UNARY_OP_FLOOR))); + constants.emplace("GGML_UNARY_OP_CEIL", Value(int64_t(GGML_UNARY_OP_CEIL))); + constants.emplace("GGML_UNARY_OP_ROUND", Value(int64_t(GGML_UNARY_OP_ROUND))); + constants.emplace("GGML_UNARY_OP_TRUNC", Value(int64_t(GGML_UNARY_OP_TRUNC))); + constants.emplace("GGML_UNARY_OP_COUNT", Value(int64_t(GGML_UNARY_OP_COUNT))); + constants.emplace("GGML_GLU_OP_REGLU", Value(int64_t(GGML_GLU_OP_REGLU))); + constants.emplace("GGML_GLU_OP_GEGLU", Value(int64_t(GGML_GLU_OP_GEGLU))); + constants.emplace("GGML_GLU_OP_SWIGLU", Value(int64_t(GGML_GLU_OP_SWIGLU))); + constants.emplace("GGML_GLU_OP_SWIGLU_OAI", Value(int64_t(GGML_GLU_OP_SWIGLU_OAI))); + constants.emplace("GGML_GLU_OP_GEGLU_ERF", Value(int64_t(GGML_GLU_OP_GEGLU_ERF))); + constants.emplace("GGML_GLU_OP_GEGLU_QUICK", Value(int64_t(GGML_GLU_OP_GEGLU_QUICK))); + constants.emplace("GGML_GLU_OP_COUNT", Value(int64_t(GGML_GLU_OP_COUNT))); + constants.emplace("GGML_OBJECT_TYPE_TENSOR", Value(int64_t(GGML_OBJECT_TYPE_TENSOR))); + constants.emplace("GGML_OBJECT_TYPE_GRAPH", Value(int64_t(GGML_OBJECT_TYPE_GRAPH))); + constants.emplace("GGML_OBJECT_TYPE_WORK_BUFFER", Value(int64_t(GGML_OBJECT_TYPE_WORK_BUFFER))); + constants.emplace("GGML_LOG_LEVEL_NONE", Value(int64_t(GGML_LOG_LEVEL_NONE))); + constants.emplace("GGML_LOG_LEVEL_DEBUG", Value(int64_t(GGML_LOG_LEVEL_DEBUG))); + constants.emplace("GGML_LOG_LEVEL_INFO", Value(int64_t(GGML_LOG_LEVEL_INFO))); + constants.emplace("GGML_LOG_LEVEL_WARN", Value(int64_t(GGML_LOG_LEVEL_WARN))); + constants.emplace("GGML_LOG_LEVEL_ERROR", Value(int64_t(GGML_LOG_LEVEL_ERROR))); + constants.emplace("GGML_LOG_LEVEL_CONT", Value(int64_t(GGML_LOG_LEVEL_CONT))); + constants.emplace("GGML_TENSOR_FLAG_INPUT", Value(int64_t(GGML_TENSOR_FLAG_INPUT))); + constants.emplace("GGML_TENSOR_FLAG_OUTPUT", Value(int64_t(GGML_TENSOR_FLAG_OUTPUT))); + constants.emplace("GGML_TENSOR_FLAG_PARAM", Value(int64_t(GGML_TENSOR_FLAG_PARAM))); + constants.emplace("GGML_TENSOR_FLAG_LOSS", Value(int64_t(GGML_TENSOR_FLAG_LOSS))); + constants.emplace("GGML_TENSOR_FLAG_COMPUTE", Value(int64_t(GGML_TENSOR_FLAG_COMPUTE))); + constants.emplace("GGML_TRI_TYPE_UPPER_DIAG", Value(int64_t(GGML_TRI_TYPE_UPPER_DIAG))); + constants.emplace("GGML_TRI_TYPE_UPPER", Value(int64_t(GGML_TRI_TYPE_UPPER))); + constants.emplace("GGML_TRI_TYPE_LOWER_DIAG", Value(int64_t(GGML_TRI_TYPE_LOWER_DIAG))); + constants.emplace("GGML_TRI_TYPE_LOWER", Value(int64_t(GGML_TRI_TYPE_LOWER))); + constants.emplace("GGML_OP_POOL_MAX", Value(int64_t(GGML_OP_POOL_MAX))); + constants.emplace("GGML_OP_POOL_AVG", Value(int64_t(GGML_OP_POOL_AVG))); + constants.emplace("GGML_OP_POOL_COUNT", Value(int64_t(GGML_OP_POOL_COUNT))); + constants.emplace("GGML_SCALE_MODE_NEAREST", Value(int64_t(GGML_SCALE_MODE_NEAREST))); + constants.emplace("GGML_SCALE_MODE_BILINEAR", Value(int64_t(GGML_SCALE_MODE_BILINEAR))); + constants.emplace("GGML_SCALE_MODE_BICUBIC", Value(int64_t(GGML_SCALE_MODE_BICUBIC))); + constants.emplace("GGML_SCALE_MODE_COUNT", Value(int64_t(GGML_SCALE_MODE_COUNT))); + constants.emplace("GGML_SCALE_FLAG_ALIGN_CORNERS", Value(int64_t(GGML_SCALE_FLAG_ALIGN_CORNERS))); + constants.emplace("GGML_SCALE_FLAG_ANTIALIAS", Value(int64_t(GGML_SCALE_FLAG_ANTIALIAS))); + constants.emplace("GGML_SORT_ORDER_ASC", Value(int64_t(GGML_SORT_ORDER_ASC))); + constants.emplace("GGML_SORT_ORDER_DESC", Value(int64_t(GGML_SORT_ORDER_DESC))); + constants.emplace("GGML_SCHED_PRIO_LOW", Value(int64_t(GGML_SCHED_PRIO_LOW))); + constants.emplace("GGML_SCHED_PRIO_NORMAL", Value(int64_t(GGML_SCHED_PRIO_NORMAL))); + constants.emplace("GGML_SCHED_PRIO_MEDIUM", Value(int64_t(GGML_SCHED_PRIO_MEDIUM))); + constants.emplace("GGML_SCHED_PRIO_HIGH", Value(int64_t(GGML_SCHED_PRIO_HIGH))); + constants.emplace("GGML_SCHED_PRIO_REALTIME", Value(int64_t(GGML_SCHED_PRIO_REALTIME))); diff --git a/src/visp/runtime_graph.cpp b/src/visp/runtime_graph.cpp new file mode 100644 index 0000000..9d1842e --- /dev/null +++ b/src/visp/runtime_graph.cpp @@ -0,0 +1,384 @@ +#include "runtime_graph.h" +#include "mmlab/config.h" +#include "nn.h" +#include "nlohmann/json.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace visp { +namespace { +using Json = nlohmann::json; +// --- 1D conv: ggml_conv_1d + optional bias --- +static tensor conv_1d(model_ref m, tensor w, tensor b, tensor x, + int stride, int pad, int dilation) { + // ⚠️ **g2c 의 conv1d 가중치는 [OC, K, IC] 로 실린다**(2D 의 OIHW 와 축 순서가 다르다). + // GGUF ne 로는 [IC, K, OC]. ggml_conv_1d 는 커널 ne=[K, IC, OC] 를 요구하므로 + // ne0/ne1 을 맞바꾼다. 안 바꾸면 IC 가 1 로 읽혀 im2col 이 "b too small" 로 죽는다. + w = ggml_cont(m, ggml_permute(m, w, 1, 0, 2, 3)); + if (w->type != GGML_TYPE_F16) w = ggml_cast(m, w, GGML_TYPE_F16); + // 그래프 입력은 이미지 관례(whcn)로 들어와 ne=[1,1,C,1] 이 될 수 있다. + // ggml_conv_1d 는 data ne=[L,IC,N] 를 요구하므로 커널의 IC 로 다시 접는다. + // ⚠️ **배치 1 전제다**(하네스가 N=1 로만 잰다). N>1 이면 여기가 틀린다. + int64_t ic = w->ne[1]; + if (x->ne[1] != ic) { + int64_t L = ggml_nelements(x) / ic; + x = ggml_reshape_3d(m, ggml_cont(m, x), L, ic, 1); + } + tensor out = ggml_conv_1d(m, w, x, stride, pad, dilation); + if (b) out = ggml_add(m, out, ggml_reshape_3d(m, b, 1, b->ne[0], 1)); + return out; +} +// --- grid_sample_2d: ggml 에 대응 커널이 없어 custom op 으로 구현 --- +// a: WHCN [W,H,C,N]. g: [2,GW,GH,N] (torch (N,GH,GW,2) 의 ggml 역순), 정규화 좌표 [-1,1]. +// dst: [GW,GH,C,N]. flags: bit0=nearest, bit1=align_corners. 범위 밖은 0 (zeros padding). +#define GTX_GS_AT(t, i0, i1, i2, i3) \ + (*(float const *)((char const *)(t)->data + (i0)*(t)->nb[0] + (i1)*(t)->nb[1] \ + + (i2)*(t)->nb[2] + (i3)*(t)->nb[3])) + +static void grid_sample_2d_op(ggml_tensor * dst, int ith, int nth, void * userdata) { + ggml_tensor const * a = dst->src[0]; + ggml_tensor const * g = dst->src[1]; + int const flags = (int)(intptr_t)userdata; + bool const nearest = (flags & 1) != 0; + bool const align = (flags & 2) != 0; + int64_t const W = a->ne[0], H = a->ne[1]; + int64_t const GW = dst->ne[0], GH = dst->ne[1], C = dst->ne[2], N = dst->ne[3]; + int64_t const rows = GH * C * N; // (gh, c, n) 행 단위로 스레드 분할 + for (int64_t r = ith; r < rows; r += nth) { + int64_t const gh = r % GH, c = (r / GH) % C, n = r / (GH * C); + for (int64_t gw = 0; gw < GW; ++gw) { + float const * gp = (float const *)((char const *)g->data + + gw * g->nb[1] + gh * g->nb[2] + n * g->nb[3]); + float fx = gp[0], fy = gp[1]; // torch grid 마지막 축 = (x, y) + fx = align ? (fx + 1.f) * (W - 1) * 0.5f : ((fx + 1.f) * W - 1.f) * 0.5f; + fy = align ? (fy + 1.f) * (H - 1) * 0.5f : ((fy + 1.f) * H - 1.f) * 0.5f; + float v = 0.f; + if (nearest) { + int64_t const xi = (int64_t)std::nearbyint(fx); + int64_t const yi = (int64_t)std::nearbyint(fy); + if (xi >= 0 && xi < W && yi >= 0 && yi < H) v = GTX_GS_AT(a, xi, yi, c, n); + } else { + int64_t const x0 = (int64_t)std::floor(fx), y0 = (int64_t)std::floor(fy); + float const dx = fx - (float)x0, dy = fy - (float)y0; + for (int64_t j = 0; j < 2; ++j) { + for (int64_t i = 0; i < 2; ++i) { + int64_t const xi = x0 + i, yi = y0 + j; + if (xi < 0 || xi >= W || yi < 0 || yi >= H) continue; + v += GTX_GS_AT(a, xi, yi, c, n) + * (i ? dx : 1.f - dx) * (j ? dy : 1.f - dy); + } + } + } + *(float *)((char *)dst->data + gw * dst->nb[0] + gh * dst->nb[1] + + c * dst->nb[2] + n * dst->nb[3]) = v; + } + } +} + +static tensor grid_sample_2d(model_ref m, tensor a, tensor g, int flags) { + ggml_tensor * args[2] = { a, g }; + return ggml_custom_4d(m, GGML_TYPE_F32, g->ne[1], g->ne[2], a->ne[2], a->ne[3], + args, 2, grid_sample_2d_op, GGML_N_TASKS_MAX, + (void *)(intptr_t)flags); +} +// --- mmcv (Modulated)DeformConv2d 의 bias (라이브러리 conv_2d_deform 은 안 받는다) --- +static tensor deform_bias(model_ref m, tensor x, tensor bias) { + if (!bias) { + return x; + } + if (!(m.flags & model_build_flag::cwhn)) { + bias = ggml_reshape_4d(m, bias, 1, 1, bias->ne[0], 1); // WHCN 은 채널이 ne[2] + } + return ggml_add(m, x, bias); +} +// --- deformable conv: ggml_conv_2d_deform + optional bias --- +// weight(GGUF, OIHW) → ggml ne [KW,KH,IC,OC]. x: WHCN. +// offset: [OW,OH,2*KH*KW,N], mask: [OW,OH,KH*KW,N] (v1 이면 nullptr). +static tensor runtime_deform_conv(model_ref m, tensor x, tensor offset, tensor mask, + int stride, int pad) { + tensor out = ggml_conv_2d_deform(m, m.weights("weight"), x, offset, mask, + stride, stride, pad, pad); + if (tensor bias = m.find("bias")) { + bias = ggml_reshape_4d(m, bias, 1, 1, bias->ne[0], 1); + out = ggml_add_inplace(m, out, bias); + } + return out; +} + +struct Value { + using Array = std::vector; + std::variant data; + Value() = default; + Value(int x) : data(int64_t(x)) {} + Value(int64_t x) : data(x) {} + Value(double x) : data(x) {} + Value(bool x) : data(x) {} + Value(std::string x) : data(std::move(x)) {} + Value(tensor x) : data(x) {} + Value(model_ref x) : data(x) {} + static Value array(Array x) { Value v; v.data = std::move(x); return v; } + double number() const { + if (auto x = std::get_if(&data)) return double(*x); + if (auto x = std::get_if(&data)) return *x; + if (auto x = std::get_if(&data)) return *x; + throw std::runtime_error("recipe: expected a number"); + } + int64_t integer() const { + if (auto x = std::get_if(&data)) return *x; + double n = number(); + if (!std::isfinite(n) || n < double(INT64_MIN) || n >= double(INT64_MAX)) + throw std::runtime_error("recipe: integer is out of range"); + return int64_t(n); + } + bool truth() const { + if (std::holds_alternative(data)) return false; + if (auto x = std::get_if(&data)) return *x != nullptr; + return number() != 0; + } + Array const& array() const { return std::get(data); } +}; + +template auto convert(Value const& v) { + using U = std::remove_cvref_t; + if constexpr (std::is_same_v) return std::get(v.data); + else if constexpr ((std::is_same_v || std::is_same_v)) return static_cast(convert(v)); + else if constexpr (std::is_same_v || std::is_same_v) { + if (std::holds_alternative(v.data)) return U(nullptr); + return U(std::get(v.data)); + } else if constexpr (std::is_same_v) return std::get(v.data).c_str(); + else if constexpr (std::is_same_v) { + auto const& s = std::get(v.data); + if (s.size() >= GGML_MAX_NAME) throw std::runtime_error("recipe: tensor name too long"); + return tensor_name(s.c_str()); + } else if constexpr (std::is_same_v) return v.truth(); + else if constexpr (std::is_integral_v || std::is_enum_v) return U(v.integer()); + else if constexpr (std::is_floating_point_v) return U(v.number()); + else if constexpr (std::is_same_v || std::is_same_v) { + U result{}; + auto const& a = v.array(); + if (a.size() != (std::is_same_v ? 2u : 4u)) throw std::runtime_error("recipe: wrong dimension count"); + for (size_t i=0; i) { + if (!std::holds_alternative(v.data)) return slice_t(v.integer()); + auto const& a = v.array(); + if (a.empty()) return slice_t{}; + if (a.size() == 1) return slice_t(a[0].integer()); + if (a.size() > 3) throw std::runtime_error("recipe: invalid slice"); + return slice_t(a[0].integer(), a[1].integer(), a.size()==3 ? a[2].integer() : 1); + } else if constexpr (std::is_same_v>) { + U result{}; + auto const& a = v.array(); + if (a.size()>result.size()) throw std::runtime_error("recipe: too many concat inputs"); + for (size_t i=0;i(a[i]); + return result; + } else static_assert(!sizeof(U), "Unregistered recipe argument type"); +} + +struct Operations { + using Function = std::function const&)>; + std::unordered_map calls; + std::unordered_map constants; + template + static Value invoke(R(*fn)(A...), std::vector const& args, std::index_sequence) { + if constexpr (std::is_void_v) { fn(convert(args[I])...); return {}; } + else if constexpr (std::is_integral_v && !std::is_same_v) + return Value(int64_t(fn(convert(args[I])...))); + else return Value(fn(convert(args[I])...)); + } + template + void add(char const* name, R(*fn)(A...), std::vector defaults = {}) { + calls.emplace(name, [fn,defaults,name](std::vector const& supplied) { + constexpr auto count=sizeof...(A); + if (supplied.size()>count || supplied.size()+defaults.size(){}); + }); + } + Operations() { +#include "runtime_ops.inc" +#include "runtime_constants.inc" + add("conv_1d", &conv_1d); + add("grid_sample_2d", &grid_sample_2d); + add("deform_bias", &deform_bias); + auto full_deform=calls.at("conv_2d_deform"); + add("runtime_deform_conv", &runtime_deform_conv); + auto short_deform=calls.at("runtime_deform_conv"); + calls["conv_2d_deform"]=[full_deform,short_deform](auto const& a) { + return a.size()==6?short_deform(a):full_deform(a); + }; + constants.emplace("slice_t::max", Value(INT64_MAX)); + constants.emplace("INFINITY", Value(std::numeric_limits::infinity())); + calls.emplace("model_find", [](auto const& a) { + if(a.size()!=2) throw std::runtime_error("recipe: invalid model_find"); + return Value(convert(a[0]).find(convert(a[1]))); + }); + calls.emplace("model_weights", [](auto const& a) { + if(a.size()!=2) throw std::runtime_error("recipe: invalid model_weights"); + auto t=convert(a[0]).find(convert(a[1])); + if(!t) throw std::runtime_error(std::string("recipe: missing weight ")+convert(a[1])); + return Value(t); + }); + } +}; +Operations const& operations() { static Operations const value; return value; } + +struct Evaluator { + model_ref model; + std::unordered_map variables; + size_t budget=10000000; + Value eval(Json const& j, int depth=0) { + if (!budget-- || depth>256) throw std::runtime_error("recipe: expression budget exceeded"); + auto child=[&](Json const& x) {return eval(x,depth+1);}; + if(j.is_null()) return {}; + if(j.is_boolean()) return Value(j.get()); + if(j.is_number_integer()) return Value(j.get()); + if(j.is_number()) return Value(j.get()); + if(j.is_string()) return Value(j.get()); + if(!j.is_object()) throw std::runtime_error("recipe: invalid expression"); + if(j.contains("ref")) { + auto name=j.at("ref").get(); + if(auto it=variables.find(name);it!=variables.end()) return it->second; + if(auto it=operations().constants.find(name);it!=operations().constants.end()) return it->second; + throw std::runtime_error("recipe: undefined reference "+name); + } + if(j.contains("model")) { + auto prefix=j.at("model").get(); + if(prefix.size()>=GGML_MAX_NAME) throw std::runtime_error("recipe: prefix too long"); + return Value(prefix.empty()?model:model.with_prefix(prefix.c_str())); + } + if(j.contains("array")) { + Value::Array a; for(auto const& x:j.at("array")) a.push_back(child(x)); + return Value::array(std::move(a)); + } + if(j.contains("call")) { + auto name=j.at("call").get(); + auto it=operations().calls.find(name); + if(it==operations().calls.end()) throw std::runtime_error("recipe: unregistered operation "+name); + std::vector args;for(auto const& x:j.at("args"))args.push_back(child(x)); + return it->second(args); + } + if(j.contains("field")) { + tensor t=convert(child(j.at("value"))); + if(!t) throw std::runtime_error("recipe: null tensor field"); + auto field=j.at("field").get(); + if(field=="type")return Value(int(t->type)); + Value::Array a; + for(int i=0;i<4;++i) { + if(field=="ne") a.emplace_back(t->ne[i]); + else if(field=="nb")a.emplace_back(int64_t(t->nb[i])); + else throw std::runtime_error("recipe: invalid tensor field"); + } + return Value::array(std::move(a)); + } + if(j.contains("index")) { + auto a=child(j.at("index"));auto i=child(j.at("at")).integer(); + if(i<0 || size_t(i)>=a.array().size())throw std::runtime_error("recipe: index out of range"); + return a.array()[size_t(i)]; + } + if(j.contains("if"))return child(j.at("if")).truth()?child(j.at("then")):child(j.at("else")); + if(j.contains("cast")) { + auto x=child(j.at("value"));auto kind=j.at("cast").get(); + return kind=="float"||kind=="double" ? Value(x.number()):Value(x.integer()); + } + if(j.contains("unary")) { + auto x=child(j.at("value"));auto op=j.at("unary").get(); + if(op=="!")return Value(!x.truth()); + if(op=="+")return x; + if(op=="-")return std::holds_alternative(x.data)?Value(-x.integer()):Value(-x.number()); + } + if(j.contains("binary")) { + auto op=j.at("binary").get();auto a=child(j.at("left")); + if(op=="&&")return Value(a.truth()&&child(j.at("right")).truth()); + if(op=="||")return Value(a.truth()||child(j.at("right")).truth()); + auto b=child(j.at("right")); + if(op=="=="||op=="!=") { + bool eq; + if(std::holds_alternative(a.data)||std::holds_alternative(a.data)) + eq=convert(a)==convert(b); + else eq=a.number()==b.number(); + return Value(op=="=="?eq:!eq); + } + double x=a.number(),y=b.number(); + if(op=="<")return Value(x")return Value(x>y); + if(op=="<=")return Value(x<=y);if(op==">=")return Value(x>=y); + bool ints=std::holds_alternative(a.data)&&std::holds_alternative(b.data); + if(op=="+")return ints?Value(a.integer()+b.integer()):Value(x+y); + if(op=="-")return ints?Value(a.integer()-b.integer()):Value(x-y); + if(op=="*")return ints?Value(a.integer()*b.integer()):Value(x*y); + if(op=="/"||op=="%") { + if(y==0)throw std::runtime_error("recipe: division by zero"); + if(op=="%")return Value(a.integer()%b.integer()); + return ints?Value(a.integer()/b.integer()):Value(x/y); + } + } + throw std::runtime_error("recipe: unknown expression"); + } +}; +Json read_program(model_file const& file) { + auto text=file.get_string("mmlab.graph"); + if(text.empty() || text.size()>64*1024*1024)throw std::runtime_error("recipe: missing or oversized mmlab.graph"); + auto j=Json::parse(text); + if(j.at("version")!=1 || !j.at("instructions").is_array() || j.at("instructions").size()>262144) + throw std::runtime_error("recipe: unsupported version or instruction count"); + return j; +} +} // namespace + +tensor runtime_graph_forward(model_ref m, tensor x, model_file const& file) { + auto program=read_program(file); + Evaluator evaluator{m,{{"x",Value(x)}}}; + tensor result=nullptr; + size_t index=0; + for(auto const& instruction:program.at("instructions")) { + try { + if(instruction.contains("set")) { + auto v=evaluator.eval(instruction.at("value")); + if(auto t=std::get_if(&v.data))result=*t; + evaluator.variables.insert_or_assign(instruction.at("set").get(),std::move(v)); + } else { + auto v=evaluator.eval(instruction.at("eval")); + if(auto t=std::get_if(&v.data))result=*t; + } + } catch(std::exception const& e) { + throw std::runtime_error("recipe instruction "+std::to_string(index)+": "+e.what()); + } + ++index; + } + if(!result)throw std::runtime_error("recipe: no output tensor"); + return result; +} +i64x4 runtime_graph_input_shape(model_file const& file) { + auto program=read_program(file); + auto shape=program.at("input_shape").get>(); + if(shape.empty()||shape.size()>4)throw std::runtime_error("recipe: input rank must be 1..4"); + i64x4 result{1,1,1,1}; + // ggml reverses PyTorch axes, image runners use CWHN at the public boundary. + if(shape.size()==4)result={shape[1],shape[3],shape[2],shape[0]}; + else if(shape.size()==3 && program.value("input_layout",std::string{"cwhn"})=="cwhn") + result={shape[1],shape[2],1,shape[0]}; + else for(size_t i=0;iINT32_MAX)throw std::runtime_error("recipe: invalid input dimension"); + return result; +} +tensor_data_layout runtime_graph_weight_layout(model_file const& file, tensor_data_layout preferred) { + // Conv1d/BatchNorm1d renderers use [L,C,N], rather than the CPU image layout. + // Preserve the layout used by the existing non-image verification harness. + auto program=read_program(file); + return program.at("input_shape").size()==4 ? preferred : tensor_data_layout::whcn; +} +arch_task runtime_graph_task(model_file const& file) { + auto program=read_program(file); + return mmlab::task_from_metadata(program.at("config"),runtime_graph_input_shape(file)); +} +// Referenced explicitly from arch_registry.cpp, so static and shared builds retain it. +void register_runtime_graph() { static arch_registrar registration("mmlab", runtime_graph_forward); } +} // namespace visp diff --git a/src/visp/runtime_graph.h b/src/visp/runtime_graph.h new file mode 100644 index 0000000..6b794b2 --- /dev/null +++ b/src/visp/runtime_graph.h @@ -0,0 +1,10 @@ +#pragma once +#include "arch_registry.h" + +namespace visp { +// A single architecture for config-derived models. No model-specific translation unit. +tensor runtime_graph_forward(model_ref, tensor, model_file const&); +arch_task runtime_graph_task(model_file const&); +i64x4 runtime_graph_input_shape(model_file const&); +tensor_data_layout runtime_graph_weight_layout(model_file const&, tensor_data_layout preferred); +} // namespace visp diff --git a/src/visp/runtime_mmdet.inc b/src/visp/runtime_mmdet.inc new file mode 100644 index 0000000..e783c39 --- /dev/null +++ b/src/visp/runtime_mmdet.inc @@ -0,0 +1,69 @@ +// Shared head/decoder fields. Values come from the GGUF recipe. + if(j["head"].contains("stacked_convs")) c.head.stacked_convs=j["head"]["stacked_convs"].get(); + if(j["head"].contains("reg_stacked_convs")) c.head.reg_stacked_convs=j["head"]["reg_stacked_convs"].get(); + if(j["head"].contains("feat_channels")) c.head.feat_channels=j["head"]["feat_channels"].get(); + if(j["head"].contains("num_base")) c.head.num_base=j["head"]["num_base"].get(); + if(j["head"].contains("num_classes")) c.head.num_classes=j["head"]["num_classes"].get(); + if(j["head"].contains("cls_convs_prefix")) c.head.cls_convs_prefix=j["head"]["cls_convs_prefix"].get(); + if(j["head"].contains("reg_convs_prefix")) c.head.reg_convs_prefix=j["head"]["reg_convs_prefix"].get(); + if(j["head"].contains("cls_head")) c.head.cls_head=j["head"]["cls_head"].get(); + if(j["head"].contains("reg_head")) c.head.reg_head=j["head"]["reg_head"].get(); + if(j["head"].contains("bbox_cls_head")) c.head.bbox_cls_head=j["head"]["bbox_cls_head"].get(); + if(j["head"].contains("pre_conv")) c.head.pre_conv=j["head"]["pre_conv"].get(); + if(j["head"].contains("head_has_norm")) c.head.head_has_norm=j["head"]["head_has_norm"].get(); + if(j["head"].contains("gn_groups")) c.head.gn_groups=j["head"]["gn_groups"].get(); + if(j["head"].contains("per_level_towers")) c.head.per_level_towers=j["head"]["per_level_towers"].get(); + if(j["head"].contains("per_level_heads")) c.head.per_level_heads=j["head"]["per_level_heads"].get(); + if(j["head"].contains("per_level_head_tail")) c.head.per_level_head_tail=j["head"]["per_level_head_tail"].get(); + if(j["head"].contains("centerness_head")) c.head.centerness_head=j["head"]["centerness_head"].get(); + if(j["head"].contains("centerness_on_reg")) c.head.centerness_on_reg=j["head"]["centerness_on_reg"].get(); + if(j["head"].contains("ctr_tanh")) c.head.ctr_tanh=j["head"]["ctr_tanh"].get(); + if(j["head"].contains("corner_emb")) c.head.corner_emb=j["head"]["corner_emb"].get(); + if(j["head"].contains("corner_centripetal")) c.head.corner_centripetal=j["head"]["corner_centripetal"].get(); + if(j["head"].contains("corner_topk")) c.head.corner_topk=j["head"]["corner_topk"].get(); + if(j["head"].contains("local_max_kernel")) c.head.local_max_kernel=j["head"]["local_max_kernel"].get(); + if(j["head"].contains("corner_distance_thr")) c.head.corner_distance_thr=j["head"]["corner_distance_thr"].get(); + if(j["head"].contains("embed_dims")) c.head.embed_dims=j["head"]["embed_dims"].get(); + if(j["head"].contains("n_heads")) c.head.n_heads=j["head"]["n_heads"].get(); + if(j["head"].contains("enc_layers")) c.head.enc_layers=j["head"]["enc_layers"].get(); + if(j["head"].contains("dec_layers")) c.head.dec_layers=j["head"]["dec_layers"].get(); + if(j["head"].contains("n_points")) c.head.n_points=j["head"]["n_points"].get(); + if(j["head"].contains("num_queries")) c.head.num_queries=j["head"]["num_queries"].get(); + if(j["head"].contains("ddq_iou_thr")) c.head.ddq_iou_thr=j["head"]["ddq_iou_thr"].get(); + if(j["head"].contains("scales_prefix")) c.head.scales_prefix=j["head"]["scales_prefix"].get(); + if(j["head"].contains("bbox_exp")) c.head.bbox_exp=j["head"]["bbox_exp"].get(); + if(j["head"].contains("bbox_clamp_stride")) c.head.bbox_clamp_stride=j["head"]["bbox_clamp_stride"].get(); + if(j["head"].contains("bbox_mul_stride")) c.head.bbox_mul_stride=j["head"]["bbox_mul_stride"].get(); + if(j["head"].contains("head_silu")) c.head.head_silu=j["head"]["head_silu"].get(); + if(j["head"].contains("head_leaky")) c.head.head_leaky=j["head"]["head_leaky"].get(); + if(j["head"].contains("reg_max")) c.head.reg_max=j["head"]["reg_max"].get(); + if(j["head"].contains("strides")) c.head.strides=j["head"]["strides"].get>(); + if(j["head"].contains("reg_denoms")) c.head.reg_denoms=j["head"]["reg_denoms"].get>(); + if(j["head"].contains("center_offset")) c.head.center_offset=j["head"]["center_offset"].get(); + if(j["det"].contains("strides")) c.det.strides=j["det"]["strides"].get>(); + if(j["det"].contains("octave_base_scale")) c.det.octave_base_scale=j["det"]["octave_base_scale"].get(); + if(j["det"].contains("octave_scales")) c.det.octave_scales=j["det"]["octave_scales"].get>(); + if(j["det"].contains("ratios")) c.det.ratios=j["det"]["ratios"].get>(); + if(j["det"].contains("center_offset")) c.det.center_offset=j["det"]["center_offset"].get(); + if(j["det"].contains("means")) { auto v=j["det"]["means"].get>(); std::copy(v.begin(),v.end(),c.det.means); } + if(j["det"].contains("stds")) { auto v=j["det"]["stds"].get>(); std::copy(v.begin(),v.end(),c.det.stds); } + if(j["det"].contains("num_classes")) c.det.num_classes=j["det"]["num_classes"].get(); + if(j["det"].contains("use_sigmoid")) c.det.use_sigmoid=j["det"]["use_sigmoid"].get(); + if(j["det"].contains("score_thr")) c.det.score_thr=j["det"]["score_thr"].get(); + if(j["det"].contains("nms_thr")) c.det.nms_thr=j["det"]["nms_thr"].get(); + if(j["det"].contains("nms_pre")) c.det.nms_pre=j["det"]["nms_pre"].get(); + if(j["det"].contains("max_per_img")) c.det.max_per_img=j["det"]["max_per_img"].get(); + if(j["det"].contains("input_w")) c.det.input_w=j["det"]["input_w"].get(); + if(j["det"].contains("base_edge")) c.det.base_edge=j["det"]["base_edge"].get>(); + if(j["det"].contains("ctr_clamp")) c.det.ctr_clamp=j["det"]["ctr_clamp"].get(); + if(j["det"].contains("base_sizes")) c.det.base_sizes=j["det"]["base_sizes"].get>>(); + if(j["det"].contains("conf_thr")) c.det.conf_thr=j["det"]["conf_thr"].get(); + if(j["det"].contains("num_buckets")) c.det.num_buckets=j["det"]["num_buckets"].get(); + if(j["det"].contains("bucket_scale")) c.det.bucket_scale=j["det"]["bucket_scale"].get(); + if(j["det"].contains("anchor_scale")) c.det.anchor_scale=j["det"]["anchor_scale"].get(); + if(j["det"].contains("tblr_normalizer")) c.det.tblr_normalizer=j["det"]["tblr_normalizer"].get(); + if(j["det"].contains("base_anchor_boxes")) c.det.base_anchor_boxes=j["det"]["base_anchor_boxes"].get>>(); + if(j["det"].contains("min_bbox_size")) c.det.min_bbox_size=j["det"]["min_bbox_size"].get(); + if(j["det"].contains("score_voting")) c.det.score_voting=j["det"]["score_voting"].get(); + if(j["det"].contains("fast_nms")) c.det.fast_nms=j["det"]["fast_nms"].get(); + if(j["det"].contains("nms_top_k")) c.det.nms_top_k=j["det"]["nms_top_k"].get(); diff --git a/src/visp/runtime_ops.inc b/src/visp/runtime_ops.inc new file mode 100644 index 0000000..183df1b --- /dev/null +++ b/src/visp/runtime_ops.inc @@ -0,0 +1,285 @@ +// Shared GGML/vision operation registry. Add adapters here when extending the recipe ABI. + add("ggml_nelements", &ggml_nelements); + add("ggml_nrows", &ggml_nrows); + add("ggml_nbytes", &ggml_nbytes); + add("ggml_nbytes_pad", &ggml_nbytes_pad); + add("ggml_element_size", &ggml_element_size); + add("ggml_is_transposed", &ggml_is_transposed); + add("ggml_is_permuted", &ggml_is_permuted); + add("ggml_is_empty", &ggml_is_empty); + add("ggml_is_view", &ggml_is_view); + add("ggml_is_scalar", &ggml_is_scalar); + add("ggml_is_vector", &ggml_is_vector); + add("ggml_is_matrix", &ggml_is_matrix); + add("ggml_is_3d", &ggml_is_3d); + add("ggml_is_contiguous", &ggml_is_contiguous); + add("ggml_is_contiguous_0", &ggml_is_contiguous_0); + add("ggml_is_contiguous_1", &ggml_is_contiguous_1); + add("ggml_is_contiguous_2", &ggml_is_contiguous_2); + add("ggml_is_contiguously_allocated", &ggml_is_contiguously_allocated); + add("ggml_is_contiguous_channels", &ggml_is_contiguous_channels); + add("ggml_is_contiguous_rows", &ggml_is_contiguous_rows); + add("ggml_are_same_shape", &ggml_are_same_shape); + add("ggml_are_same_stride", &ggml_are_same_stride); + add("ggml_can_repeat", &ggml_can_repeat); + add("ggml_used_mem", &ggml_used_mem); + add("ggml_get_no_alloc", &ggml_get_no_alloc); + add("ggml_get_mem_size", &ggml_get_mem_size); + add("ggml_get_max_tensor_size", &ggml_get_max_tensor_size); + add("ggml_new_tensor_1d", &ggml_new_tensor_1d); + add("ggml_new_tensor_2d", &ggml_new_tensor_2d); + add("ggml_new_tensor_3d", &ggml_new_tensor_3d); + add("ggml_new_tensor_4d", &ggml_new_tensor_4d); + add("ggml_dup_tensor", &ggml_dup_tensor); + add("ggml_view_tensor", &ggml_view_tensor); + add("ggml_get_first_tensor", &ggml_get_first_tensor); + add("ggml_get_next_tensor", &ggml_get_next_tensor); + add("ggml_get_tensor", &ggml_get_tensor); + add("ggml_set_name", &ggml_set_name); + add("ggml_dup", &ggml_dup); + add("ggml_dup_inplace", &ggml_dup_inplace); + add("ggml_add", &ggml_add); + add("ggml_add_inplace", &ggml_add_inplace); + add("ggml_add_cast", &ggml_add_cast); + add("ggml_add_id", &ggml_add_id); + add("ggml_add1", &ggml_add1); + add("ggml_add1_inplace", &ggml_add1_inplace); + add("ggml_acc", &ggml_acc); + add("ggml_acc_inplace", &ggml_acc_inplace); + add("ggml_sub", &ggml_sub); + add("ggml_sub_inplace", &ggml_sub_inplace); + add("ggml_mul", &ggml_mul); + add("ggml_mul_inplace", &ggml_mul_inplace); + add("ggml_div", &ggml_div); + add("ggml_div_inplace", &ggml_div_inplace); + add("ggml_sqr", &ggml_sqr); + add("ggml_sqr_inplace", &ggml_sqr_inplace); + add("ggml_sqrt", &ggml_sqrt); + add("ggml_sqrt_inplace", &ggml_sqrt_inplace); + add("ggml_log", &ggml_log); + add("ggml_log_inplace", &ggml_log_inplace); + add("ggml_expm1", &ggml_expm1); + add("ggml_expm1_inplace", &ggml_expm1_inplace); + add("ggml_softplus", &ggml_softplus); + add("ggml_softplus_inplace", &ggml_softplus_inplace); + add("ggml_sin", &ggml_sin); + add("ggml_sin_inplace", &ggml_sin_inplace); + add("ggml_cos", &ggml_cos); + add("ggml_cos_inplace", &ggml_cos_inplace); + add("ggml_sum", &ggml_sum); + add("ggml_sum_rows", &ggml_sum_rows); + add("ggml_cumsum", &ggml_cumsum); + add("ggml_mean", &ggml_mean); + add("ggml_argmax", &ggml_argmax); + add("ggml_count_equal", &ggml_count_equal); + add("ggml_repeat", &ggml_repeat); + add("ggml_repeat_4d", &ggml_repeat_4d); + add("ggml_repeat_back", &ggml_repeat_back); + add("ggml_concat", &ggml_concat); + add("ggml_abs", &ggml_abs); + add("ggml_abs_inplace", &ggml_abs_inplace); + add("ggml_sgn", &ggml_sgn); + add("ggml_sgn_inplace", &ggml_sgn_inplace); + add("ggml_neg", &ggml_neg); + add("ggml_neg_inplace", &ggml_neg_inplace); + add("ggml_step", &ggml_step); + add("ggml_step_inplace", &ggml_step_inplace); + add("ggml_tanh", &ggml_tanh); + add("ggml_tanh_inplace", &ggml_tanh_inplace); + add("ggml_elu", &ggml_elu); + add("ggml_elu_inplace", &ggml_elu_inplace); + add("ggml_relu", &ggml_relu); + add("ggml_leaky_relu", &ggml_leaky_relu); + add("ggml_relu_inplace", &ggml_relu_inplace); + add("ggml_sigmoid", &ggml_sigmoid); + add("ggml_sigmoid_inplace", &ggml_sigmoid_inplace); + add("ggml_gelu", &ggml_gelu); + add("ggml_gelu_inplace", &ggml_gelu_inplace); + add("ggml_gelu_erf", &ggml_gelu_erf); + add("ggml_gelu_erf_inplace", &ggml_gelu_erf_inplace); + add("ggml_gelu_quick", &ggml_gelu_quick); + add("ggml_gelu_quick_inplace", &ggml_gelu_quick_inplace); + add("ggml_silu", &ggml_silu); + add("ggml_silu_inplace", &ggml_silu_inplace); + add("ggml_silu_back", &ggml_silu_back); + add("ggml_hardswish", &ggml_hardswish); + add("ggml_hardsigmoid", &ggml_hardsigmoid); + add("ggml_exp", &ggml_exp); + add("ggml_exp_inplace", &ggml_exp_inplace); + add("ggml_floor", &ggml_floor); + add("ggml_floor_inplace", &ggml_floor_inplace); + add("ggml_ceil", &ggml_ceil); + add("ggml_ceil_inplace", &ggml_ceil_inplace); + add("ggml_round", &ggml_round); + add("ggml_round_inplace", &ggml_round_inplace); + add("ggml_trunc", &ggml_trunc); + add("ggml_trunc_inplace", &ggml_trunc_inplace); + add("ggml_xielu", &ggml_xielu); + add("ggml_glu", &ggml_glu); + add("ggml_reglu", &ggml_reglu); + add("ggml_reglu_swapped", &ggml_reglu_swapped); + add("ggml_geglu", &ggml_geglu); + add("ggml_geglu_swapped", &ggml_geglu_swapped); + add("ggml_swiglu", &ggml_swiglu); + add("ggml_swiglu_swapped", &ggml_swiglu_swapped); + add("ggml_geglu_erf", &ggml_geglu_erf); + add("ggml_geglu_erf_swapped", &ggml_geglu_erf_swapped); + add("ggml_geglu_quick", &ggml_geglu_quick); + add("ggml_geglu_quick_swapped", &ggml_geglu_quick_swapped); + add("ggml_glu_split", &ggml_glu_split); + add("ggml_reglu_split", &ggml_reglu_split); + add("ggml_geglu_split", &ggml_geglu_split); + add("ggml_swiglu_split", &ggml_swiglu_split); + add("ggml_geglu_erf_split", &ggml_geglu_erf_split); + add("ggml_geglu_quick_split", &ggml_geglu_quick_split); + add("ggml_swiglu_oai", &ggml_swiglu_oai); + add("ggml_norm", &ggml_norm); + add("ggml_norm_inplace", &ggml_norm_inplace); + add("ggml_rms_norm", &ggml_rms_norm); + add("ggml_rms_norm_inplace", &ggml_rms_norm_inplace); + add("ggml_group_norm", &ggml_group_norm); + add("ggml_group_norm_inplace", &ggml_group_norm_inplace); + add("ggml_l2_norm", &ggml_l2_norm); + add("ggml_l2_norm_inplace", &ggml_l2_norm_inplace); + add("ggml_rms_norm_back", &ggml_rms_norm_back); + add("ggml_mul_mat", &ggml_mul_mat); + add("ggml_mul_mat_id", &ggml_mul_mat_id); + add("ggml_out_prod", &ggml_out_prod); + add("ggml_scale", &ggml_scale); + add("ggml_scale_inplace", &ggml_scale_inplace); + add("ggml_scale_bias", &ggml_scale_bias); + add("ggml_scale_bias_inplace", &ggml_scale_bias_inplace); + add("ggml_set", &ggml_set); + add("ggml_set_inplace", &ggml_set_inplace); + add("ggml_set_1d", &ggml_set_1d); + add("ggml_set_1d_inplace", &ggml_set_1d_inplace); + add("ggml_set_2d", &ggml_set_2d); + add("ggml_set_2d_inplace", &ggml_set_2d_inplace); + add("ggml_cpy", &ggml_cpy); + add("ggml_cast", &ggml_cast); + add("ggml_cont", &ggml_cont); + add("ggml_cont_1d", &ggml_cont_1d); + add("ggml_cont_2d", &ggml_cont_2d); + add("ggml_cont_3d", &ggml_cont_3d); + add("ggml_cont_4d", &ggml_cont_4d); + add("ggml_reshape", &ggml_reshape); + add("ggml_reshape_1d", &ggml_reshape_1d); + add("ggml_reshape_2d", &ggml_reshape_2d); + add("ggml_reshape_3d", &ggml_reshape_3d); + add("ggml_reshape_4d", &ggml_reshape_4d); + add("ggml_view_1d", &ggml_view_1d); + add("ggml_view_2d", &ggml_view_2d); + add("ggml_view_3d", &ggml_view_3d); + add("ggml_view_4d", &ggml_view_4d); + add("ggml_permute", &ggml_permute); + add("ggml_transpose", &ggml_transpose); + add("ggml_get_rows", &ggml_get_rows); + add("ggml_get_rows_back", &ggml_get_rows_back); + add("ggml_set_rows", &ggml_set_rows); + add("ggml_diag", &ggml_diag); + add("ggml_diag_mask_inf", &ggml_diag_mask_inf); + add("ggml_diag_mask_inf_inplace", &ggml_diag_mask_inf_inplace); + add("ggml_diag_mask_zero", &ggml_diag_mask_zero); + add("ggml_diag_mask_zero_inplace", &ggml_diag_mask_zero_inplace); + add("ggml_soft_max", &ggml_soft_max); + add("ggml_soft_max_inplace", &ggml_soft_max_inplace); + add("ggml_soft_max_ext", &ggml_soft_max_ext); + add("ggml_soft_max_ext_inplace", &ggml_soft_max_ext_inplace); + add("ggml_soft_max_ext_back", &ggml_soft_max_ext_back); + add("ggml_soft_max_ext_back_inplace", &ggml_soft_max_ext_back_inplace); + add("ggml_rope", &ggml_rope); + add("ggml_rope_inplace", &ggml_rope_inplace); + add("ggml_rope_ext", &ggml_rope_ext); + add("ggml_rope_ext_inplace", &ggml_rope_ext_inplace); + add("ggml_rope_custom", &ggml_rope_custom); + add("ggml_rope_custom_inplace", &ggml_rope_custom_inplace); + add("ggml_rope_ext_back", &ggml_rope_ext_back); + add("ggml_clamp", &ggml_clamp); + add("ggml_im2col", &ggml_im2col); + add("ggml_conv_1d", &ggml_conv_1d); + add("ggml_conv_1d_ph", &ggml_conv_1d_ph); + add("ggml_conv_1d_dw", &ggml_conv_1d_dw); + add("ggml_conv_1d_dw_ph", &ggml_conv_1d_dw_ph); + add("ggml_conv_transpose_1d", &ggml_conv_transpose_1d); + add("ggml_conv_2d", &ggml_conv_2d); + add("ggml_im2col_3d", &ggml_im2col_3d); + add("ggml_conv_3d", &ggml_conv_3d); + add("ggml_conv_2d_sk_p0", &ggml_conv_2d_sk_p0); + add("ggml_conv_2d_s1_ph", &ggml_conv_2d_s1_ph); + add("ggml_conv_2d_dw", &ggml_conv_2d_dw); + add("ggml_conv_2d_dw_direct", &ggml_conv_2d_dw_direct); + add("ggml_conv_2d_deform", &ggml_conv_2d_deform); + add("ggml_conv_transpose_2d_p0", &ggml_conv_transpose_2d_p0); + add("ggml_conv_2d_direct", &ggml_conv_2d_direct); + add("ggml_conv_3d_direct", &ggml_conv_3d_direct); + add("ggml_pool_1d", &ggml_pool_1d); + add("ggml_pool_2d", &ggml_pool_2d); + add("ggml_pool_2d_back", &ggml_pool_2d_back); + add("ggml_upscale", &ggml_upscale); + add("ggml_upscale_ext", &ggml_upscale_ext); + add("ggml_interpolate", &ggml_interpolate); + add("ggml_pad", &ggml_pad); + add("ggml_pad_circular", &ggml_pad_circular); + add("ggml_pad_ext", &ggml_pad_ext); + add("ggml_pad_ext_circular", &ggml_pad_ext_circular); + add("ggml_pad_reflect_1d", &ggml_pad_reflect_1d); + add("ggml_roll", &ggml_roll); + add("ggml_tri", &ggml_tri); + add("ggml_fill", &ggml_fill); + add("ggml_fill_inplace", &ggml_fill_inplace); + add("ggml_timestep_embedding", &ggml_timestep_embedding); + add("ggml_argsort", &ggml_argsort); + add("ggml_argsort_top_k", &ggml_argsort_top_k); + add("ggml_top_k", &ggml_top_k); + add("ggml_arange", &ggml_arange); + add("ggml_flash_attn_ext", &ggml_flash_attn_ext); + add("ggml_flash_attn_back", &ggml_flash_attn_back); + add("ggml_ssm_conv", &ggml_ssm_conv); + add("ggml_ssm_scan", &ggml_ssm_scan); + add("ggml_win_part", &ggml_win_part); + add("ggml_win_unpart", &ggml_win_unpart); + add("ggml_unary", &ggml_unary); + add("ggml_unary_inplace", &ggml_unary_inplace); + add("ggml_get_rel_pos", &ggml_get_rel_pos); + add("ggml_add_rel_pos", &ggml_add_rel_pos); + add("ggml_add_rel_pos_inplace", &ggml_add_rel_pos_inplace); + add("ggml_rwkv_wkv6", &ggml_rwkv_wkv6); + add("ggml_gated_linear_attn", &ggml_gated_linear_attn); + add("ggml_rwkv_wkv7", &ggml_rwkv_wkv7); + add("ggml_solve_tri", &ggml_solve_tri); + add("ggml_gated_delta_net", &ggml_gated_delta_net); + add("ggml_cross_entropy_loss", &ggml_cross_entropy_loss); + add("ggml_cross_entropy_loss_back", &ggml_cross_entropy_loss_back); + add("ggml_opt_step_adamw", &ggml_opt_step_adamw); + add("ggml_opt_step_sgd", &ggml_opt_step_sgd); + add("ggml_set_zero", &ggml_set_zero); + add("linear", &linear, {}); + add("layer_norm", &layer_norm, {Value(1e-05)}); + add("group_norm", &group_norm, {Value(1e-05)}); + add("pad_reflect_ext", &pad_reflect_ext, {}); + add("space_to_depth_quad", &space_to_depth_quad, {}); + add("permute_cwhn_to_whcn", &permute_cwhn_to_whcn, {}); + add("permute_whcn_to_cwhn", &permute_whcn_to_cwhn, {}); + add("as_f32", &as_f32, {}); + add("cwhn_to_contiguous_2d", &cwhn_to_contiguous_2d, {}); + add("whcn_to_contiguous_2d", &whcn_to_contiguous_2d, {}); + add("contiguous_2d_to_cwhn", &contiguous_2d_to_cwhn, {}); + add("contiguous_2d_to_whcn", &contiguous_2d_to_whcn, {}); + add("conv_2d", &conv_2d, {Value(1), Value(0), Value(1)}); + add("conv_2d_ex", &conv_2d_ex, {Value(1), Value(1)}); + add("conv_2d_grouped", &conv_2d_grouped, {Value(1), Value(0), Value(1), Value(1)}); + add("conv_2d_wt", &conv_2d_wt, {Value(1), Value(0), Value(1)}); + add("conv_2d_wt_dw", &conv_2d_wt_dw, {Value(1), Value(0), Value(1)}); + add("conv_2d_depthwise", &conv_2d_depthwise, {Value(1), Value(0), Value(1)}); + add("conv_2d_depthwise_ex", &conv_2d_depthwise_ex, {}); + add("conv_2d_deform", &conv_2d_deform, {}); + add("conv_transpose_2d", &conv_transpose_2d, {Value(0), Value(1), Value(0)}); + add("batch_norm_2d", &batch_norm_2d, {}); + add("prelu", &prelu, {}); + add("patch_embed", &patch_embed, {}); + add("attention", &attention, {}); + add("compute_graph_output", &compute_graph_output, {Value(std::string("output"))}); + add("slice", &slice, {Value::array({}), Value::array({}), Value::array({})}); + add("concat", &concat, {}); + add("interpolate", &interpolate, {}); + add("named", &named, {}); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f31792d..72f6107 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -13,6 +13,10 @@ target_link_options(test-vision PRIVATE ${VISP_LINK_OPTIONS}) target_link_libraries(test-vision PRIVATE visioncpp ${VISP_FMT_LINK}) add_test(NAME vision COMMAND test-vision -v) +add_executable(test-mmlab test-mmlab.cpp) +target_link_libraries(test-mmlab PRIVATE visioncpp) +add_test(NAME mmlab-sampling COMMAND test-mmlab) + # # Model tests (image comparisons) diff --git a/tests/test-mmlab.cpp b/tests/test-mmlab.cpp new file mode 100644 index 0000000..3d4f733 --- /dev/null +++ b/tests/test-mmlab.cpp @@ -0,0 +1,55 @@ +#include "visp/ml.h" + +#include +#include +#include + +// Internal builder helper. This test calls the implementation in visioncpp, +// rather than maintaining a second copy of the sampling graph. +namespace visp::mmlab { +tensor sample_tokens(model_ref, tensor, int64_t, int64_t, tensor, tensor); +} + +int main() { + using namespace visp; + auto graph = compute_graph_init(); + model_ref model(nullptr, &graph); + auto values = compute_graph_input(model, GGML_TYPE_F32, {2, 12, 2, 1}, "values"); + auto x = compute_graph_input(model, GGML_TYPE_F32, {7, 2, 1, 1}, "x"); + auto y = compute_graph_input(model, GGML_TYPE_F32, {7, 2, 1, 1}, "y"); + auto output = compute_graph_output(model, mmlab::sample_tokens(model, values, 4, 3, x, y)); + auto device = backend_init(backend_type::cpu); + if (!compute_graph_allocate(graph, device)) return 2; + std::array data{}; + for (int group = 0; group < 2; ++group) + for (int pixel = 0; pixel < 12; ++pixel) + for (int channel = 0; channel < 2; ++channel) + data[channel + 2 * pixel + 24 * group] = float(1 + channel + 2 * group); + constexpr std::array xs{-.25f, 0, .75f, 3, 3.25f, -2, 8}; + constexpr std::array ys{0, .5f, 1.25f, 2, 2.25f, 1, 1}; + // Independent expected zero-padding contribution for these coordinates. + constexpr std::array contribution{.75f, 1, 1, 1, .5625f, 0, 0}; + std::array coordinates_x{}, coordinates_y{}; + for (int group = 0; group < 2; ++group) + for (int i = 0; i < 7; ++i) { + coordinates_x[i + 7 * group] = xs[i]; + coordinates_y[i + 7 * group] = ys[i]; + } + transfer_to_backend(values, span{data}); + transfer_to_backend(x, span{coordinates_x}); + transfer_to_backend(y, span{coordinates_y}); + compute(graph, device); + std::array result{}; + transfer_from_backend(output, span{result}); + for (int group = 0; group < 2; ++group) + for (int i = 0; i < 7; ++i) + for (int channel = 0; channel < 2; ++channel) { + float expected = float(1 + channel + 2 * group) * contribution[i]; + float actual = result[channel + 2 * i + 14 * group]; + if (std::abs(actual - expected) > 1e-6f) { + std::fprintf(stderr, "group=%d point=%d channel=%d: %g != %g\n", group, i, channel, actual, expected); + return 1; + } + } + std::puts("MMLab CPU zero-padding sampling: PASS"); +} diff --git a/tests/test_yolov9t.py b/tests/test_yolov9t.py deleted file mode 100644 index c6cfcfe..0000000 --- a/tests/test_yolov9t.py +++ /dev/null @@ -1,170 +0,0 @@ -""" -cd /mnt/c/Users/x2199/Desktop/vision.cpp_ -cmake --build build -j 6 --target vision-workbench -LD_LIBRARY_PATH=build/lib pytest -q tests/test_yolov9t.py::test_dfl_layer -q -""" - -import torch -import numpy as np -import sys -import os -from torch.nn import Conv2d -from torch import nn -import math -from typing import List, Tuple -from torch import Tensor -from . import workbench -# workbench helpers -from .workbench import to_nchw, to_nhwc, input_tensor -import torch.nn.functional as F -from pathlib import Path - -# layout parameter used by the workbench C++ harness -nhwc_layout = dict(memory_layout="nhwc") -# Add the scripts directory to the path to import workbench -sys.path.append(os.path.dirname(os.path.abspath(__file__))) - - -import pytest - -class DFL(nn.Module): - """Distribution Focal Loss (DFL)""" - def __init__(self, c1=16): - super().__init__() - self.conv = Conv2d(c1, 1, 1, bias=False).requires_grad_(False) - x = torch.arange(c1, dtype=torch.float) - self.conv.weight.data[:] = nn.Parameter(x.view(1, c1, 1, 1)) - self.c1 = c1 - - def forward(self, x:torch.Tensor) -> torch.Tensor: - b, c, a = x.shape # batch, channels, anchors - return self.conv(x.view(b, 4, self.c1, a).transpose(2, 1).softmax(1)).view(b, 4, a) - -def test_dfl_layer(c1=16, reg_max=16, debug=False): - """ - parameter: - c1(channels 1D): 16 - reg_max(int): 16 - Input/Output: - input: torch.Size([1, 64, 8400]) - output: torch.Size([1, 4, 8400]) - """ - # Load the DFL conv weight from the provided checkpoint - # Expected key: 'detect.dfl.conv.weight' - ckpt_paths = [ - Path(__file__).parent.parent / 'scripts' / 'yolov9t_converted.pth', - Path.cwd() / 'scripts' / 'yolov9t_converted.pth', - Path(__file__).parent / '..' / 'scripts' / 'yolov9t_converted.pth', - ] - ckpt_file = None - for p in ckpt_paths: - if p.exists(): - ckpt_file = p - break - if ckpt_file is None: - raise FileNotFoundError( - 'Checkpoint ./scripts/yolov9t_converted.pth not found. Place the file at repo scripts/ directory.' - ) - - ckpt = torch.load(str(ckpt_file), map_location='cpu') - key = 'detect.dfl.conv.weight' - if isinstance(ckpt, dict) and key in ckpt: - weight = ckpt[key] - elif isinstance(ckpt, dict) and 'state_dict' in ckpt and key in ckpt['state_dict']: - weight = ckpt['state_dict'][key] - else: - # try to find a matching key - found = None - if isinstance(ckpt, dict): - for k in ckpt.keys(): - if k.endswith('detect.dfl.conv.weight') or k == key: - found = ckpt[k] - break - if found is None: - raise KeyError(f"Key '{key}' not found in checkpoint {ckpt_file}") - weight = found - - # Ensure weight is a torch tensor - if not isinstance(weight, torch.Tensor): - weight = torch.tensor(weight) - - # Map into state for workbench - state = {key: weight} - - # Use fixed input shape: (1, 64, 8400) # N, A, 4*c1 - box3 = input_tensor(1, 64, 8400) - # bool debug = p.get("debug", 0) != 0; - # int reg_max = p.get("reg_max", 16); - # tensor weight = m.weight("detect.dfl.conv.weight"); - params = dict(debug=int(bool(debug)), reg_max=reg_max) - # pass the state dict (mapping of weight name -> tensor) as the third arg - # The workbench harness expects a 4D NHWC tensor, so expand a spatial dim - box_4d = box3.unsqueeze(2) # shape (N, C, 1, anchors) - print(box_4d.shape) - - box_nhwc = to_nhwc(box_4d) # shape (N, anchors, 1, C) - result = workbench.invoke_test("dfl_forward", box_nhwc, state, params) - - # Compute expected using the DFL computation with the loaded weight - # Following the original DFL implementation: - # x.view(b, 4, c1, a).transpose(2,1) -> (b, c1, 4, a) - # softmax over c1 dim, then apply 1x1 conv with loaded weight (shape (1, c1, 1, 1)) - b, C, A = box3.shape - assert C == 4 * c1, f"expected C==4*c1 ({4*c1}), got {C}" - x_view = box3.view(b, 4, c1, A).transpose(2, 1) # (b, c1, 4, a) - soft = torch.softmax(x_view, dim=1) - # Ensure weight shape is (out_channels=1, in_channels=c1, 1, 1) - w = weight - if w.dim() == 1: - w = w.view(1, -1, 1, 1) - elif w.dim() == 4 and w.shape[0] != 1 and w.shape[1] == 1: - # some conversions store transposed shapes; try to reshape - w = w.permute(1, 0, 2, 3) - w = w.to(dtype=soft.dtype) - expected_conv = F.conv2d(soft, w) - expected = expected_conv.view(b, 4, A) - - # Normalize possible return types from workbench.invoke_test to shape (N,4,anchors) - if result is None: - raise AssertionError("workbench.invoke_test returned no output") - - if not isinstance(result, torch.Tensor): - raise AssertionError(f"Unexpected result type: {type(result)}") - - # If the C++ side returned a 4D tensor (NHWC), convert it back to NCHW and collapse spatial dims - if result.dim() == 4: - # convert NHWC -> NCHW - result_nchw = to_nchw(result) - N = result_nchw.size(0) - C = result_nchw.size(1) - H = result_nchw.size(2) - W = result_nchw.size(3) - # Common possibilities from the C++ side: - # 1) (N, 4, A1, A2) -> channels already in dim1 (C==4) - # 2) (N, 1, 4, anchors) -> channels collapsed into dim2 - # 3) (N, 1, H, W) where H*W == 4*anchors -> need to split - if C == 4: - result = result_nchw.view(N, 4, -1) - elif C == 1 and H == 4: - # (N,1,4,anchors) -> squeeze channel dim - result = result_nchw.squeeze(1) - elif C == 1 and (H * W) % 4 == 0: - anchors = (H * W) // 4 - result = result_nchw.view(N, 4, anchors) - else: - # fallback: collapse spatial dims into one dimension - result = result_nchw.view(N, C, -1) - elif result.dim() == 3: - # already (N, C, anchors) - pass - else: - raise AssertionError(f"Unexpected result dimensionality: {result.dim()}") - - assert result.shape == expected.shape, f"shape mismatch result={result.shape} expected={expected.shape}" - assert torch.allclose(result, expected) - - - - -if __name__ == "__main__": - test_dfl_layer() \ No newline at end of file diff --git a/tools/README.md b/tools/README.md index dfae01e..91eaaca 100755 --- a/tools/README.md +++ b/tools/README.md @@ -14,8 +14,8 @@ tools/ mmdet_to_pt.py CLI: config -> backbone.pt + .postproc.h frcnn_wrap.py two-stage / mask sub-graphs frcnn_to_pt.py CLI for the above - detect/ Framework-neutral head components, compiled into the runner - head.h head.cpp + detect/ Drawing helper shared by the verification runners + draw.h verify/ Runners and inspection, grouped by task backbone/ run_mmdet.cpp run_dump.cpp dense_head/ run_vfnet_head.cpp verify_heads.py @@ -32,7 +32,7 @@ control flow that depends on the data — suppression counts that are not known deformable offsets derived from an earlier prediction, proposal counts that vary per image — and tracing records only the path one input happened to take. -`detect/head.cpp` assembles the dense-head families with thirteen functions. RetinaNet, ATSS, +`src/visp/mmdet_head.cpp` assembles the dense-head families with thirteen functions. RetinaNet, ATSS, PAA, FCOS and GFL share one skeleton — two convolution towers and a few output convolutions — so they differ only through flags on `anchor_head_cfg`: a third `centerness` branch, per-level learnable scales, the FCOS bbox transform, and GFL's distribution decode. VFNet, RepPoints and @@ -43,7 +43,7 @@ Nothing here holds a table of layer names. The final classification and regressi are found by output channel count, and convolution padding comes from the kernel size stored in the weights, so a new family needs no edit to a lookup table. -`verify/dense_head/verify_heads.py` measures each family against `bbox_head` in PyTorch, one +`verify/mmdet/dense_head/verify_heads.py` measures each family against `bbox_head` in PyTorch, one tensor at a time, before decoding. It needs trained checkpoints: an untrained model leaves gamma at 1, beta at 0 and scale at 1, and an assembly that skips those terms still scores a perfect cosine. @@ -71,8 +71,9 @@ flag. The attention primitives it would build on already exist in `src/visp/nn.h Two arrangements keep framework knowledge from spreading: -- `detect/head.cpp` is not part of `libvisioncpp`. It is compiled together with the runner, - so detector-specific structure never enters the core library. +- `src/visp/mmdet_head.cpp` assembles the heads and `src/visp/mmdet_run.cpp` sequences a whole + detector. Both live in `libvisioncpp`, so `vision-cli` runs MMDetection models like any other + architecture and the verification runner shares the same code. - Decoding lives in the library, not in the frontend. `detect_anchor`, `roi_align` and `rpn_proposals` in `src/visp/postproc.h` take numbers, not configuration objects, which is why they are reusable for detectors that never went through MMDetection. @@ -90,12 +91,13 @@ python tools/frontend/mmdet/mmdet_to_pt.py \ # one input shape, and a graph built for another size aborts in ggml_can_repeat at run time. # The interface the generated code must satisfy is in docs/mmdet-detectors.md. -# 3. Build the runner: the generated graph, head.cpp and run_mmdet.cpp together. -# The parameters header is the one export wrote next to backbone.pt. -bash tools/build/build_mmdet_cpp.sh output/MMDetBackbone backbone.postproc.h +# 3. Register and build. The parameters header is the one export wrote next to backbone.pt. +python tools/install_arch.py output/MMDetBackbone --mmdet --postproc backbone.postproc.h +cmake --build build -j8 # 4. Run. -output/MMDetBackbone/run_mmdet output/MMDetBackbone/MMDetBackbone.gguf image.jpg detected.png 512 +build/bin/vision-cli mmdetbackbone -m output/MMDetBackbone/MMDetBackbone.gguf \ + -i image.jpg -o detected.png ``` Step 3 looks for the library in `build/`. If you configured elsewhere, name it: @@ -116,7 +118,9 @@ library's `preprocess()` driven by extracted values rather than anything hand-wr ## Looking at the output -The extension of the output path decides what `run_mmdet` writes. +`vision-cli` draws the boxes on the image. The verification runner `run_mmdet` is kept for two +things `vision-cli` does not do: it accepts an already pre-processed `.bin` tensor as input, and +`MMDET_DUMP_HEAD` dumps the tensors before decoding. Its output extension decides the format. ```sh run_mmdet model.gguf image.jpg detected.png 512 # the image, boxes drawn on it @@ -144,7 +148,7 @@ encoded as colour, which keeps a font out of the runner. | `VISP_DRAW_THRESHOLD` | Minimum score to draw. Default `0.3` | | `VISP_PRINT_DETS` | How many rows to print. `0` turns the table off | -`tools/verify/draw_boxes.py` draws a `.bin` that was written earlier, which is the way to look +`tools/verify/common/draw_boxes.py` draws a `.bin` that was written earlier, which is the way to look at a file kept for comparison. It adds class names and scores as text, which the C++ path does not. diff --git a/tools/build/build_mmdet_cpp.sh b/tools/build/build_mmdet_cpp.sh index dd0012e..eba6182 100755 --- a/tools/build/build_mmdet_cpp.sh +++ b/tools/build/build_mmdet_cpp.sh @@ -1,9 +1,12 @@ #!/usr/bin/env bash # -# build_mmdet_cpp.sh — g2c 가 생성한 mmdet 백본(output/.cpp)을 러너(verify/backbone/run_mmdet)· -# head 부품(detect/head.cpp)과 함께 컴파일해 libvisioncpp 에 링크한다. -# head.cpp 는 라이브러리가 아니라 여기서 러너와 함께 컴파일된다 → g2c output/.cpp 를 직접 -# 컴파일(arch/ 복사·cli REG 없음). +# build_mmdet_cpp.sh — g2c 가 생성한 mmdet 백본(output/.cpp)을 검증 하네스 +# (verify/mmdet/backbone/run_mmdet)와 함께 컴파일해 libvisioncpp 에 링크한다. +# head 조립과 디코드는 라이브러리 안에 있다(src/visp/mmdet_head.cpp · mmdet_run.cpp) — +# 여기서는 백본과 러너만 컴파일한다. +# +# 배포 경로는 이 러너가 아니라 vision-cli 다. 하네스는 `.bin` 입력과 중간 텐서 덤프 때문에 +# 남아 있다. # # usage: build_mmdet_cpp.sh [arch_name] # gen_dir = g2c --output 디렉토리 (예: output/MMDetBackbone) — .cpp/.h/.gguf 있음 @@ -20,7 +23,7 @@ set -e SELF="$(cd "$(dirname "$0")" && pwd)" # vision.cpp/tools/build (빌드 스크립트) V="$(cd "$SELF/../.." && pwd)" # vision.cpp (libvisioncpp 소스) -DETECT="$V/tools/detect" # 공용 head/decode 부품 (head.cpp/head.h) +DETECT="$V/tools/detect" # 하네스 전용 그리기 부품 (draw.h) RUN="$V/tools/verify" # E2E 검증 러너 # `--build ` 를 어디에 두든 받는다 — 위치 인자와 섞이지 않게 먼저 걷어낸다. ARG_BUILD="" @@ -70,13 +73,13 @@ INC="$GEN/inc" mkdir -p "$INC/visp/arch" cp "$GEN/$ARCH.h" "$INC/visp/arch/$ARCH.h" -# run_mmdet.cpp + head.cpp(러너와 함께 컴파일, 라이브러리 아님) + g2c 백본 output/.cpp +# run_mmdet.cpp + g2c 백본 output/.cpp g++ -std=c++20 -O2 $FMT_FLAGS \ -DARCH="$ARCH" -DVISP_ARCH_HEADER="\"visp/arch/$ARCH.h\"" \ -DMMDET_PARAMS_HEADER="\"$PARAMS\"" \ -I"$DETECT" -I"$INC" -I"$V/include" -I"$V/src" \ -I"$V/depend/llama/ggml/include" -I"$V/depend/llama/vendor" \ - "$RUN/backbone/run_mmdet.cpp" "$DETECT/head.cpp" "$GEN/$ARCH.cpp" \ + "$RUN/backbone/run_mmdet.cpp" "$GEN/$ARCH.cpp" \ -L"$LIB" -lvisioncpp -lggml -lggml-base -lggml-cpu \ -Wl,-rpath,"$LIB" \ -o "$GEN/run_mmdet" diff --git a/tools/build_gtx_smoke.sh b/tools/build_gtx_smoke.sh new file mode 100644 index 0000000..e27cf11 --- /dev/null +++ b/tools/build_gtx_smoke.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# build_routing_probe.sh — gtx_smoke 컴파일 (libggml + libggml-gtx 링크). +# 전제: vision.cpp 가 -DGGML_GTX=ON 으로 빌드돼 libggml-gtx.so 가 있을 것. +set -e +SELF="$(cd "$(dirname "$0")" && pwd)"; V="$(cd "$SELF/.." && pwd)" +BUILD="${VISP_BUILD:-$V/build}"; LIB="$BUILD/lib" +[ -f "$LIB/libggml-gtx.so" ] || { echo "libggml-gtx.so 없음: $LIB (cmake -DGGML_GTX=ON 필요)"; exit 1; } +g++ -std=c++20 -O2 -I"$V/depend/llama/ggml/include" -I"$V/depend/ggml-gtx/include" \ + "$SELF/gtx_smoke.cpp" \ + -L"$LIB" -lggml -lggml-base -lggml-cpu -lggml-gtx -Wl,-rpath,"$LIB" \ + -o "${1:-$BUILD/bin}/gtx_smoke" +echo "built: ${1:-$BUILD/bin}/gtx_smoke" diff --git a/tools/example_custom.sh b/tools/example_custom.sh index 6e79021..dcf348e 100755 --- a/tools/example_custom.sh +++ b/tools/example_custom.sh @@ -3,7 +3,7 @@ # # ./tools/example_custom.sh # -# 앞의 세 예제는 라이브러리가 모델을 준다(ultralytics·torchvision·mmdet). 이건 아니다 — +# 앞의 두 예제는 라이브러리가 모델을 준다(torchvision·mmdet). 이건 아니다 — # 클래스를 내가 쓰고, `.pt` 로 저장하고, g2c 에 그 `.pt` 를 준다. 다른 부서가 자기 모델을 # 얹을 때 밟는 경로가 이것이다. # diff --git a/tools/example_mmdet.sh b/tools/example_mmdet.sh index 894cf45..22f106b 100755 --- a/tools/example_mmdet.sh +++ b/tools/example_mmdet.sh @@ -5,7 +5,7 @@ # ./tools/example_mmdet.sh fcos # 다른 계열 # MMDET=~/mmbuild/mmdetection ./tools/example_mmdet.sh # -# yolo·torchvision 예제와 갈리는 지점: **mmdet head 는 트레이스를 통과하지 못한다.** +# torchvision 예제와 갈리는 지점: **mmdet head 는 트레이스를 통과하지 못한다.** # 그래서 `.pt` 로 나가는 것은 backbone+neck 뿐이고, head 는 C++ 부품이 조립한다. # `bbox_head` 는 속성으로 남아 state_dict(→GGUF)에 실린다 — 연산만 빼고 가중치는 남긴다. # 자세한 배경은 `docs/mmdet-detectors.md`. @@ -20,7 +20,7 @@ SIZE="${SIZE:-512}" VCPP="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" MMDET="${MMDET:-$HOME/mmbuild/mmdetection}" WORK="${WORK:-/tmp/visp-example-$FAM}" -DH="$VCPP/tools/verify/dense_head" +DH="$VCPP/tools/verify/mmdet/dense_head" # ⚠️ 이 예제만 **mmdet 이 설치된 인터프리터**가 필요하다. 앞의 세 예제와 갈리는 지점이다 — # 거긴 `uv run --project` 로 g2c 환경을 쓰면 됐지만, 여기선 하네스가 mmdet 을 import 한다. diff --git a/tools/example_torchvision.sh b/tools/example_torchvision.sh index 3c799e8..9b454e0 100755 --- a/tools/example_torchvision.sh +++ b/tools/example_torchvision.sh @@ -4,7 +4,7 @@ # ./tools/example_torchvision.sh # resnet18 # ./tools/example_torchvision.sh resnet50 # -# yolo 예제와 다른 점: 분류 모델은 박스가 없다. `vision-cli` 대신 생성된 `.py` 로 +# 검출 예제와 다른 점: 분류 모델은 박스가 없다. `vision-cli` 대신 생성된 `.py` 로 # ggml 커널에서 돌리고 **torch 와 값을 대조**한다 — 그게 이 갈래의 "끝까지" 다. # # ⚠️ `weights='DEFAULT'` 를 쓴다. 랜덤 초기화로 재지 마라 — 항등 초기값(γ=1·β=0)이 diff --git a/tools/example_yolo.sh b/tools/example_yolo.sh deleted file mode 100755 index a148d05..0000000 --- a/tools/example_yolo.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# example_yolo.sh — g2c 생성 arch 를 붙여 **끝까지** 돌리는 예제. -# -# ./tools/example_yolo.sh # yolo26m, 기본 테스트 이미지 -# ./tools/example_yolo.sh yolo26s # 다른 크기 -# ./tools/example_yolo.sh yolo26m my.jpg -# -# 생성물을 저장소에 커밋하지 않는 이유: `.pt` 는 ultralytics 가 받아주고 `.cpp`/`.gguf` 는 -# g2c 가 만든다 — 커밋해봐야 g2c 가 바뀌면 조용히 낡는다. 대신 **이 스크립트로 재현**한다. -set -euo pipefail - -MODEL="${1:-yolo26m}" -IMAGE="${2:-}" -SIZE="${SIZE:-640}" - -VCPP="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -G2C="${G2C_ROOT:-$(cd "$VCPP/.." && pwd)}" -BUILD="${VISP_BUILD:-$VCPP/build}" -WORK="${WORK:-/tmp/visp-example-$MODEL}" - -# 클래스명 첫 글자를 대문자로: yolo26m → Yolo26m (g2c --name 규약) -CLS="$(printf '%s' "${MODEL:0:1}" | tr '[:lower:]' '[:upper:]')${MODEL:1}" - -[ -z "$IMAGE" ] && IMAGE="$VCPP/tests/input/cat-and-hat.jpg" - -if [ ! -f "$G2C/shared/compile/pipeline.py" ]; then - echo "g2c 를 못 찾았다: $G2C" >&2 - echo "G2C_ROOT= 로 지정할 것." >&2 - exit 1 -fi - -echo "== 1/4 g2c 컴파일 ($MODEL → $WORK) ==" -# ⚠️ trace 는 thread 폭주로 hang 할 수 있다 — 한 개로 묶는다. -OMP_NUM_THREADS=1 PYTHONPATH="$G2C" \ - uv run --project "$G2C" python -m shared.compile.pipeline \ - --model "ultralytics.YOLO('$MODEL.pt')" --name "$CLS" \ - --output "$WORK" --input-shape "1,3,$SIZE,$SIZE" - -# 성공 판정은 종료코드가 아니라 **파일 유무**로 한다 — g2c 는 실패해도 exit 0 + "완료!" 를 낸다. -[ -f "$WORK/$CLS.gguf" ] || { echo "gguf 가 안 나왔다 — 위 로그를 볼 것" >&2; exit 2; } - -echo "== 2/4 arch 등록 ==" -python3 "$VCPP/tools/install_arch.py" "$WORK" --name "$CLS" --detect-yolo --size "$SIZE" - -echo "== 3/4 빌드 ==" -cmake --build "$BUILD" -j"$(nproc)" > /dev/null - -echo "== 4/4 실행 ==" -OUT="$WORK/detected.png" # vision-cli 는 확장자와 무관하게 PNG 로 쓴다 -"$BUILD/bin/vision-cli" "$MODEL" -m "$WORK/$CLS.gguf" -i "$IMAGE" -o "$OUT" - -echo -echo "결과: $OUT" diff --git a/tools/frontend/mmdet/append_head_weights.py b/tools/frontend/mmdet/append_head_weights.py index a370010..aab6c63 100644 --- a/tools/frontend/mmdet/append_head_weights.py +++ b/tools/frontend/mmdet/append_head_weights.py @@ -63,8 +63,13 @@ def append(pt_path: str, gguf_path: str, prefixes=None) -> int: """`gguf_path` 를 제자리에서 다시 써서 선언된 가중치를 덧붙인다. 추가된 개수를 돌려준다.""" import torch - gguf = _gguf() model = torch.load(pt_path, weights_only=False) + return append_model(model, gguf_path, prefixes) + + +def append_model(model, gguf_path: str, prefixes=None) -> int: + """Append declared weights from an already loaded model, preserving GGUF metadata.""" + gguf = _gguf() if prefixes is None: prefixes = tuple(getattr(model, "head_weight_prefixes", ()) or ()) if not prefixes: @@ -82,12 +87,19 @@ def append(pt_path: str, gguf_path: str, prefixes=None) -> int: extra = [] for key, tensor in model.state_dict().items(): - if key in have or not key.startswith(prefixes) or key.endswith(_SKIP_SUFFIX): + if short_name(key) in have or not key.startswith(prefixes) or key.endswith(_SKIP_SUFFIX): continue arr = tensor.detach().cpu().numpy() # 정수 버퍼를 fp16 으로 구우면 값이 뭉개진다 — 부동소수만 내린다. + # ⚠️ **본체가 어떤 정밀도로 구워졌는지 따라가야 한다.** g2c 의 `generate_gguf` 는 + # `GTX_GGUF_F32=1` 이면 fp32 로 굽는데 여기만 fp16 을 하드코딩하고 있었다. + # 한 gguf 안에 두 정밀도가 섞이면 러너가 **세그폴트로 죽는다**(실측: fp32 로 + # 구운 yolact·rpn 이 `run_mmdet` 진입 직후 SIGSEGV). + # 「정밀도냐 버그냐」를 가르는 fp32 대조가 one-stage 계열에서 **통째로 막혀 + # 있었다** — two-stage 는 head 를 덧붙이지 않아 안 걸렸다. if arr.dtype.kind == "f": - arr = arr.astype(np.float16) + arr = arr.astype(np.float32 + if os.environ.get("GTX_GGUF_F32") == "1" else np.float16) # ⚠️ **여기도 이름을 줄여야 한다.** g2c 가 굽는 쪽(`generate_gguf`)만 줄이면 # 나중에 덧붙는 head 가중치가 64자를 넘어 로드가 거부된다 # (`mask_head.mask_feature_head.convs_all_levels.…` 실측 64자). @@ -101,6 +113,16 @@ def append(pt_path: str, gguf_path: str, prefixes=None) -> int: # `Duplicated key name 'general.architecture'` 를 뱉는다. 값이 같아 산출물은 멀쩡한데 # 그 줄이 상위 하네스의 "실패 이유" 로 잡혀 원인을 오도한다. writer = gguf.GGUFWriter(tmp, arch=arch) + # Keep graph recipes and model configuration when rewriting the container. + # GGUF header counters are reader pseudo-fields, not serialised metadata. + for key, field in reader.fields.items(): + if key.startswith("GGUF.") or key == "general.architecture": + continue + writer.add_key_value(key, field.contents(), field.types[0], + sub_type=field.types[1] if len(field.types) > 1 else None) + extra_names = [name for name, _ in extra] + if len(set(extra_names)) != len(extra_names): + raise ValueError("Extra weight names collide after shortening") for name, arr in existing + extra: writer.add_tensor(name, arr) writer.write_header_to_file() diff --git a/tools/frontend/mmdet/frcnn_wrap.py b/tools/frontend/mmdet/frcnn_wrap.py index 87a0fed..fc6cd87 100755 --- a/tools/frontend/mmdet/frcnn_wrap.py +++ b/tools/frontend/mmdet/frcnn_wrap.py @@ -232,12 +232,41 @@ def num_bbox_stages(det): class MaskRCNN_SubC(nn.Module): - """mask RoIAlign feat (M,256,14,14) → mask_logits (M, num_classes, 28, 28). (Mask R-CNN).""" + """mask RoIAlign feat (M,256,14,14) → mask_logits (M, num_classes, 28, 28). (Mask R-CNN). + + ⚠️ **마스크 추출기도 `GenericRoIExtractor` 일 수 있다**(`groie`). 그때는 bbox 쪽과 + 똑같이 레벨을 고르지 않고 전 레벨에 RoIAlign 을 걸어 pre→합산→post 를 태운다. + 러너가 레벨별 결과를 **배치로 이어붙여** 넣고 여기서 가른다 — `FRCNN_SubB` 와 같은 수법. + 다른 점 하나: 러너는 마스크 head 를 **검출 하나씩(배치 1)** 돌리므로 레벨당 RoI 수가 + 항상 **1** 이다(bbox 쪽은 `rpn.max_per_img`). 그래서 여기 `mgroie_n` 은 상수 1 이다. + """ def __init__(self, det): super().__init__() self.mask_head = det.roi_head.mask_head + mext = getattr(det.roi_head, "mask_roi_extractor", None) + if isinstance(mext, nn.ModuleList): + mext = mext[0] + gen = mext is not None and type(mext).__name__ == "GenericRoIExtractor" + self.mgroie_pre = (getattr(mext, "pre_module", None) + if gen and getattr(mext, "with_pre", False) else None) + self.mgroie_post = (getattr(mext, "post_module", None) + if gen and getattr(mext, "with_post", False) else None) + self.mgroie_lvls = len(getattr(mext, "featmap_strides", []) or []) if gen else 0 def forward(self, mask_feat): + # groie: (L, C, 14, 14) 로 들어온다 → pre 한 번 → 레벨별로 갈라 합산 → post + # ⚠️ `getattr(..., 기본값)` 으로 읽는다 — 이 클래스는 절여져서 **다른 프로세스**가 + # 푼다. 새 속성을 그냥 읽으면 옛 코드가 절인 객체에서 AttributeError 가 난다. + L = getattr(self, "mgroie_lvls", 0) + if L: + if getattr(self, "mgroie_pre", None) is not None: + mask_feat = self.mgroie_pre(mask_feat) + acc = mask_feat[:1] # 러너가 검출 하나씩 넣는다 → 레벨당 1 + for i in range(1, L): + acc = acc + mask_feat[i:i + 1] + mask_feat = acc + if getattr(self, "mgroie_post", None) is not None: + mask_feat = self.mgroie_post(mask_feat) return self.mask_head(mask_feat) @@ -328,16 +357,22 @@ def frcnn_cfg(det, size=800): # RoIAlign 은 그걸 표현 못 한다 — 조용히 레벨 선택으로 떨어뜨리면 값만 틀린다. # GenericRoIExtractor(groie): **집계 경로는 지원한다** — 러너가 레벨별로 RoIAlign 을 # 걸어 배치로 이어붙이고, SubB 가 pre→합산→post 를 그래프 안에서 한다. - # ⚠️ 다만 `post_cfg` 가 `GeneralizedAttention` 이면 아직 못 굽는다. 위치 임베딩 - # broadcast 가 **5차원**인데 ggml 은 4D 까지라, 렌더러가 `ggml_cont` 로 떨어뜨리고 - # (생성 코드에 `repeat [N,6,7,4,42]` 주석이 남는다) 뒤의 add 가 - # `GGML_ASSERT(ggml_can_repeat)` 로 죽는다. 러너 크래시로 흘려보내지 말고 - # **여기서 이유를 말한다** — 렌더러 작업이지 하네스 작업이 아니다. - if type(ext).__name__ == "GenericRoIExtractor" and getattr(ext, "with_post", False): - if type(getattr(ext, "post_module", None)).__name__ == "GeneralizedAttention": - raise NotImplementedError( - "GenericRoIExtractor + GeneralizedAttention: 집계는 되지만 post 의 위치 임베딩 " - "broadcast 가 5차원이라 ggml(4D)로 못 편다 — g2c 렌더러 작업이다") + # `post_cfg` 가 `GeneralizedAttention` 인 경우(2026-08-24 진행 중): + # ✔ 5D 위치 임베딩 repeat — 렌더러가 4D 로 접는다(`shape/render.reduce_to_4d`) + # ✔ `aten::div_` — 렌더러 등록(없을 땐 나눗셈이 통째로 사라졌다) + # ✔ 비균일 상수(`dim_mat` 64원소) — 리터럴 상수를 항상 GGUF 로 굽는다 + # ✔ `div` 양방향 broadcast — `sub` 와 같은 처방(좌변 실체화) + # ✔ >4D `zeros` 가 1원소로 뭉개지던 것 — 같은 축소 규칙으로 접는다 + # ✗ **6D `energy` 를 4D 로 못 맞춘다.** `energy(n,heads,h,w,h_kv,w_kv)` 는 6D 이고 + # 위치항은 5D 인데, 바깥 축 병합은 **각자 다른 개수의 축을 접는다** — + # energy 는 (42000,7,4,4), 위치항은 (6000,7,7,4) 가 돼 broadcast 가 안 맞는다. + # 일반 규칙(인접 축 병합)으로는 여기까지다. **축을 피연산자끼리 맞추려면 + # 이 op 전용 로우어링**(예: (h,w)→hw, (h_kv,w_kv)→kv 로 접어 (n*heads, hw, kv)) + # 이 필요하다 — 그래프 전역 레이아웃 패스는 pvt 를 회귀시킨 전례가 있다. + # ⚠️ **op 이름으로 거절하는 게 아니다.** 위 둘은 실제로 남은 한계이고, 둘 다 고쳐지면 + # 이 분기를 지운다. 이름 목록은 실제 한계보다 항상 넓거나 좁다 — 이전 조건은 + # `GeneralizedAttention` 전체를 막았지만 `empirical_attention`(`0010`)은 진작 + # 통과하고 있었다. 막힌 건 `attention_type[1]/[3]`(기하 갈래)뿐이다. bh = det.roi_head.bbox_head # 캐스케이드는 bbox_head 도 ModuleList 다. 클래스 수·coder 종류는 단계 공통이라 # **마지막 단계**를 쓴다(최종 박스를 내는 단계라 디코드 규약이 거기 맞춰져 있다). @@ -381,6 +416,17 @@ def frcnn_cfg(det, size=800): "mask_finest_scale": int(getattr(mext, "finest_scale", 56)), "mask_thr_binary": float(rcnn_c.mask_thr_binary), } + # 마스크 로짓을 재려면 `MaskRCNN_SubC` 를 구울 수 있어야 한다. head 가 **하나일 + # 때만** 굽는다 — HTC·SCNet 은 `mask_head` 가 `ModuleList`(단계마다 하나)라 + # 통째로 태우면 `forward` 가 없어 export 에서 죽는다. + # ⚠️ 이 값이 **박스 판정을 바꾸면 안 된다.** 마스크는 더 재는 축이지 문턱이 아니다 + # (게이트는 `groie` 하나뿐 — `verify_postproc_roi.MASK_GATE`). + mask["mask_head_single"] = int( + not isinstance(det.roi_head.mask_head, nn.ModuleList)) + # 마스크 추출기도 레벨을 고르지 않을 수 있다 — 러너가 레벨마다 RoIAlign 을 걸어 + # 배치로 이어붙여야 한다(0 이면 평소대로 레벨 하나를 고른다). + mask["mask_groie_levels"] = (len(mext.featmap_strides) + if type(mext).__name__ == "GenericRoIExtractor" else 0) # Mask Scoring R-CNN: 마스크 IoU 로 점수를 다시 매긴다. 없으면 안 싣는다. if getattr(det.roi_head, "mask_iou_head", None) is not None: mih = det.roi_head.mask_iou_head @@ -476,7 +522,11 @@ def frcnn_cfg(det, size=800): # SparseR-CNN/QueryInst: 단계마다 query(object_feats)를 함께 나른다. # 0 이면 평범한 캐스케이드다. "sparse_stages": ns if type(det.roi_head).__name__ == "SparseRoIHead" else 0, - "num_proposals": int(getattr(det.rpn_head, "num_proposals", 0) or 0), + # ⚠️ **`det.rpn_head` 를 직접 읽지 마라.** `FastRCNN` 은 proposal 을 밖에서 받는 + # 것이 정의라 그 속성이 **아예 없다**(`AttributeError: 'FastRCNN' object has no + # attribute 'rpn_head'`). 위에서 `rh` 로 안전하게 받아 뒀는데 여기 한 줄만 + # 직접 읽고 있어서, 외부 proposal 경로를 다 갖춰 놓고도 이 줄에서 죽었다. + "num_proposals": int(getattr(rh, "num_proposals", 0) or 0), # groie: 러너가 레벨마다 RoIAlign 을 걸어 배치로 이어붙여야 한다(0 이면 평소대로). "groie_levels": len(getattr(ext, "featmap_strides", []) or []) if type(ext).__name__ == "GenericRoIExtractor" else 0, diff --git a/tools/frontend/mmdet/mmdet_compat.py b/tools/frontend/mmdet/mmdet_compat.py index f376dfc..5593bc6 100644 --- a/tools/frontend/mmdet/mmdet_compat.py +++ b/tools/frontend/mmdet/mmdet_compat.py @@ -156,6 +156,8 @@ def sigmoid_geometric_mean(x, y): _patch_carafe() _patch_swin_mask() + _patch_pointrend_coords() + _patch_san_attn_bias() _patch_sac_dilation() _patch_masked_conv() @@ -274,6 +276,198 @@ def forward(self, x): SAConv2d.forward = forward SAConv2d._visp_sac_patched = True + +import torch.nn as _nn + + +class _ScalarAffine(_nn.Module): + """`nn.Linear(1, 1)` 을 **matmul 없이** 낸다 — 스칼라 affine 이다. + + ⚠️ **모듈 스코프에 둔다.** 함수 안에 정의하면 `torch.save(model)` 가 + `Can't pickle local object` 로 죽는다(하네스가 모델을 `.pt` 로 저장한다). + """ + + def __init__(self, lin): + super().__init__() + self.register_buffer("w", lin.weight.detach().reshape(())) + b = (lin.bias.detach().reshape(()) if lin.bias is not None + else lin.weight.detach().new_zeros(())) + self.register_buffer("b", b) + + def forward(self, x): + return x * self.w + self.b + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + # Replacement happens during construction, before the learned Linear + # checkpoint is loaded. Preserve its values under the scalar buffers. + for source, target in (("weight", "w"), ("bias", "b")): + name = prefix + source + if name in state_dict: + value = state_dict.pop(name) + if value.numel() != 1: + error_msgs.append(f"{name}: expected one affine coefficient") + else: + state_dict[prefix + target] = value.reshape(()) + super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs) + + +def _patch_pointrend_coords(): + """PointRend 의 좌표 조립을 **슬라이스 대입 대신 `stack`** 으로 다시 쓴다. + + ⚠️ **trace 가 부분 슬라이스 대입을 삼킨다.** mmseg `point_head.py:357-366` 은 + + point_coords = torch.zeros(B, P, 2) + point_coords[:, :, 0] = w_step/2 + (idx %% width) * w_step + point_coords[:, :, 1] = h_step/2 + (idx // width) * h_step + + 로 좌표를 만드는데, 이 두 대입이 그래프에 안 실린다. 산술(`mul`·`sub`)은 실리는데 + **되쓰기만 빠진다** — 그래서 `zeros` 가 그대로 흘러가 **8196개 점이 전부 같은 + 자리를 샘플링**하고, 그 값을 서로 다른 위치에 되쓴다. + `unhandled op` 표시조차 안 남는다(`grep -c` 가 0 이다). + + 실측(mmseg point_rend, 512x1024): + 정제 없이 coarse 만 올린 것 vs torch 최종 = 2.44e-02 ← 통과선 안 + 우리 최종 vs torch 최종 = 8.24e-02 ← **정제가 값을 망친다** + 오차 분포도 산수가 맞는다 — 정제점 8196 x 업샘플 4 = 32,784 / 524,288 = 6.25%, + 실측 「>1.0 오차 6.27%」와 일치. + + ⚠️ swin 마스크(`_patch_swin_mask`)와 달리 **버퍼로 못 굽는다** — 좌표는 입력에 따라 + 변한다. 대신 **같은 수식을 `stack` 으로** 쓴다. 값이 동일하므로 torch 기준값 쪽이 + 이 패치를 지나도 결과가 같다. + """ + try: + from mmseg.models.decode_heads.point_head import PointHead + except Exception: + return # mmseg 가 없는 환경 — 조용히 넘어간다 + if getattr(PointHead, "_visp_coords_patched", False): + return + import torch + + def get_points_test(self, seg_logits, uncertainty_func, cfg): + num_points = cfg.subdivision_num_points + uncertainty_map = uncertainty_func(seg_logits) + batch_size, _, height, width = uncertainty_map.shape + # ⚠️ **파이썬 int 로 내린다.** trace 에서 `.shape` 은 동적 값으로 잡혀, + # `idx %% width` 의 나누는 수가 상수가 아니게 된다 → `remainder`·`floor_divide` + # 렌더러가 상수를 못 뽑아 `unhandled op` 으로 떨어진다. + # 입력 크기가 고정이라(계열별 crop) 이 둘은 실제로 상수다. + height, width = int(height), int(width) + h_step = 1.0 / height + w_step = 1.0 / width + uncertainty_map = uncertainty_map.view(batch_size, height * width) + num_points = min(height * width, num_points) + point_indices = uncertainty_map.topk(num_points, dim=1)[1] + # 원식과 같은 값 — 대입 대신 stack 으로 쌓는다. + # ⚠️ **정수 인덱스를 먼저 f32 로 올린 뒤 계산한다.** ggml 의 산술(`sub`·`mul`· + # `div`·`scale`·`concat`)은 정수 타입을 모르고 **abort** 한다. 정수인 채로 + # 두면 렌더러마다 하나씩 걸린다(실측: sub → scale → concat 순으로 세 번). + # 인덱스 최대가 height*width(=131072)라 f32(2^24 까지 정확)로 손실이 없다. + idxf = point_indices.float() + xs = w_step / 2.0 + (idxf % width) * w_step + ys = h_step / 2.0 + torch.floor(idxf / width) * h_step + point_coords = torch.stack([xs, ys], dim=2) + return point_indices, point_coords + + PointHead.get_points_test = get_points_test + PointHead._visp_coords_patched = True + + + +def _patch_san_attn_bias(): + """SAN 의 어텐션 바이어스를 **대입 대신 `cat`** 으로 조립한다. + + ⚠️ mmseg `san_head.py:401-409` 이 `new_zeros` 뒤 **대입 네 번**으로 마스크를 만든다: + + new_attn_bias = attn_bias.new_zeros(S+1+L, S+1+L) + new_attn_bias[:, :S] = -100 + new_attn_bias[arange(S), arange(S)] = 0 + new_attn_bias[:S, S] = -100 + new_attn_bias[..., :S, -L:] = attn_bias # ← 이것만 입력 의존 + + trace 가 이 대입들을 삼켜 마스크가 **전부 0** 이 된다. `zeros` 가 그대로 흘러가고 + `unhandled op 'index_put_inplace'` 하나만 남는다(`point_rend` 좌표와 같은 부류). + + 앞 셋은 **shape 만으로 정해지는 상수**이고 넷째만 입력에 의존한다. + 블록으로 갈라 `cat` 으로 쌓으면 값이 동일하고 trace 에 실린다. + + ┌─────────────┬────┬──────────┐ + │ -100·(1-I) │-100│ attn_bias│ S 행 + ├─────────────┼────┴──────────┤ + │ -100 │ 0 │ 1+L 행 + └─────────────┴───────────────┘ + """ + try: + from mmseg.models.decode_heads.san_head import RecWithAttnbias + except Exception: + return # mmseg/SAN 이 없는 환경 — 조용히 넘어간다 + if getattr(RecWithAttnbias, "_visp_bias_patched", False): + return + import torch + import torch.nn.functional as F + + def _build_attn_biases(self, attn_biases, target_shape): + formatted_attn_biases = [] + for attn_bias in attn_biases: + n, num_head, num_sos, h, w = attn_bias.shape + attn_bias = F.adaptive_max_pool2d( + attn_bias.reshape(n, num_head * num_sos, h, w), + output_size=target_shape) + attn_bias = attn_bias.reshape(n, num_head, num_sos, *target_shape) + true_num_head = self.num_heads + if num_head == 1: + attn_bias = attn_bias.repeat(1, true_num_head, 1, 1, 1) + attn_bias = attn_bias.reshape(n * true_num_head, num_sos, -1) + L = int(attn_bias.shape[-1]) + if self.cross_attn: + formatted_attn_biases.append(attn_bias) + else: + S = int(num_sos) + B = int(attn_bias.shape[0]) + dev, dt = attn_bias.device, attn_bias.dtype + eye = torch.eye(S, device=dev, dtype=dt) + tl = -100.0 * (1.0 - eye) # (S,S) 대각만 0 + tm = torch.full((S, 1), -100.0, device=dev, dtype=dt) + top_c = torch.cat([tl, tm], dim=-1).unsqueeze(0).expand(B, -1, -1) + top = torch.cat([top_c, attn_bias], dim=-1) # (B, S, S+1+L) + bl = torch.full((1 + L, S), -100.0, device=dev, dtype=dt) + br = torch.zeros(1 + L, 1 + L, device=dev, dtype=dt) + bot = torch.cat([bl, br], dim=-1).unsqueeze(0).expand(B, -1, -1) + formatted_attn_biases.append(torch.cat([top, bot], dim=-2)) + if len(formatted_attn_biases) == 1: + formatted_attn_biases = [ + formatted_attn_biases[0] for _ in range(self.num_layers) + ] + return formatted_attn_biases + + RecWithAttnbias._build_attn_biases = _build_attn_biases + RecWithAttnbias._visp_bias_patched = True + + # ── `bias_scaling` 은 `nn.Linear(1,1)` = **스칼라 affine** 이다 ──────────────── + # ⚠️ 호출부가 `bias_scaling(attn_bias[..., None]).squeeze(-1)`(san_head.py:91) 라 + # **크기-1 축을 만들어 matmul** 을 건다. 그 축은 `reduce_to_4d` 가 버리므로 + # `ggml_mul_mat` 이 `a->ne[0]=1` vs `b->ne[0]=32` 로 죽는다(`ggml.c:3203`). + # matmul 없이 곱셈+덧셈으로 내면 그 축 자체가 필요 없다 — 값은 동일하다. + try: + from mmseg.models.decode_heads.san_head import MLPMaskDecoder + except Exception: + return + if getattr(MLPMaskDecoder, "_visp_scale_patched", False): + return + import torch.nn as nn + + _orig_init = MLPMaskDecoder.__init__ + + def __init__(self, *a, **kw): + _orig_init(self, *a, **kw) + if isinstance(getattr(self, "bias_scaling", None), nn.Linear): + self.bias_scaling = _ScalarAffine(self.bias_scaling) + + MLPMaskDecoder.__init__ = __init__ + MLPMaskDecoder._visp_scale_patched = True + + def _patch_swin_mask(): """Swin 의 shifted-window 어텐션 마스크를 **버퍼로 미리 굽는다.** @@ -426,12 +620,18 @@ def install_loader_modules(out_path, *names): import os import shutil + import sys + here = os.path.dirname(os.path.abspath(__file__)) dst_dir = out_path if os.path.isdir(out_path) else os.path.dirname(os.path.abspath(out_path)) os.makedirs(dst_dir, exist_ok=True) copied = [] for name in (*names, "mmdet_compat"): - src = os.path.join(here, name + ".py") + # ⚠️ **이 폴더에만 있다고 보지 마라.** `mmseg_wrap`·`mmpose_wrap` 은 형제 폴더에 + # 있어서 `here` 로 찾으면 `FileNotFoundError` 가 난다(2026-09-01 실측). + # 이미 import 된 모듈이면 그 실제 위치를 쓴다. + mod = sys.modules.get(name) + src = getattr(mod, "__file__", None) or os.path.join(here, name + ".py") dst = os.path.join(dst_dir, name + ".py") if os.path.abspath(src) == os.path.abspath(dst): continue diff --git a/tools/frontend/mmdet/mmdet_to_pt.py b/tools/frontend/mmdet/mmdet_to_pt.py index 0198fe0..4e19e8f 100755 --- a/tools/frontend/mmdet/mmdet_to_pt.py +++ b/tools/frontend/mmdet/mmdet_to_pt.py @@ -56,7 +56,7 @@ def emit_params(cfg, config_name): f"// head_type: {h}\n", "#pragma once\n\n", "#include \n", - '#include "head.h"\n\n', + '#include "visp/mmdet_head.h"\n\n', "namespace visp {\n\n", "inline mmdet_cfg mmdet_params() {\n", " mmdet_cfg c;\n", @@ -198,9 +198,12 @@ def main(argv=None): ap.add_argument("--checkpoint", default=None, help="가중치(.pth) — 없으면 config init") ap.add_argument("--out", required=True, help="출력 .pt") ap.add_argument("--size", type=int, default=512) + # 텍스트 조건부 계열(GLIP)만 쓴다. 캡션이 고정이면 텍스트 임베딩이 상수라 + # 버퍼로 구워 그래프 입력을 이미지 하나로 만든다 — `MMDetTextCond` 주석 참고. + ap.add_argument("--caption", default=None, help="텍스트 조건부 계열의 고정 문구") a = ap.parse_args(argv) - m, shapes, cfg = build(a.config, a.checkpoint, a.size) + m, shapes, cfg = build(a.config, a.checkpoint, a.size, caption=a.caption) print(f" backbone/neck features: {shapes}") torch.save(m, a.out) n_head = sum(1 for k in m.state_dict() if k.startswith("bbox_head")) diff --git a/tools/frontend/mmdet/mmdet_wrap.py b/tools/frontend/mmdet/mmdet_wrap.py index 4e1b796..8966e35 100755 --- a/tools/frontend/mmdet/mmdet_wrap.py +++ b/tools/frontend/mmdet/mmdet_wrap.py @@ -183,6 +183,69 @@ def forward(self, x): return tuple(f) # FPN features (레벨별 텐서) +# 검증용 기본 캡션. **아무 문장이나 되는 건 아니다** — 토큰 수가 임베딩 shape 을 정하고 +# 그게 GGUF 에 구워지므로, 실제로 쓸 문구와 비슷한 길이여야 의미 있는 측정이 된다. +DEFAULT_CAPTION = "person. bicycle. car. dog." + + +def text_embedding(det, caption): + """텍스트 가지를 한 번 돌려 임베딩을 얻는다. `(1, max_tokens, 768)`. + + ⚠️ transformers 5.x 에서 `batch_encode_plus` 가 **삭제**됐다. mmdet 3.3.0 은 4.x API 를 + 기대한다(`language_models/bert.py`) — 같은 뜻인 `__call__` 로 이어 준다. + """ + tok = det.language_model.tokenizer + if not hasattr(tok, "batch_encode_plus"): + tok.batch_encode_plus = tok.__call__ + with torch.no_grad(): + d = det.language_model([caption]) + return d["embedded"].contiguous() + + +class MMDetTextCond(nn.Module): + """텍스트 조건부 검출기(GLIP)를 **이미지 하나만 받는** 모듈로 감싼다. + + 캡션이 고정이면 텍스트 임베딩은 입력과 무관한 **상수**다 → 버퍼로 구우면 그래프 입력이 + 이미지 하나가 되어 단일입력 codegen·러너를 그대로 쓴다. DETR 의 `pos_embed`·`enc_ref`, + Deformable 의 `enc_proposals` 를 굽는 것과 같은 수법이다. + (그렇게 안 하면 융합헤드는 입력이 6개인데 codegen 이 **모든 input 노드를 같은 `x` 에 + 묶어** 크래시 없이 전부 이미지가 들어간다. 자세한 사유는 g2c `DECISIONS.md`.) + + ⚠️ 이 래퍼로 재는 것은 **이미지 가지 + 융합헤드**다. BERT 자체가 아니다 — + 구운 임베딩은 torch 가 계산한 값이다(BERT 는 2026-08-26 에 따로 쟀다). + + 출력은 15텐서: 레벨 5 × {정렬점수 (1,HW,256) · 박스 (1,4,H,W) · centerness (1,1,H,W)}. + """ + + def __init__(self, det, caption=DEFAULT_CAPTION): + super().__init__() + self.backbone = det.backbone + self.neck = det.neck + self.bbox_head = det.bbox_head + self.register_buffer("text_embedded", text_embedding(det, caption)) + self.caption = caption + # `_no_box` 경로는 g2c 가 그래프에서 모은 가중치만 쓴다 — 헤드가 트레이스에 들어오므로 + # `append_head_weights` 가 필요 없다. 선언은 남겨 둔다(다른 경로가 읽는다). + self.head_weight_prefixes = ("text_embedded",) + + def forward(self, x): + f = tuple(self.neck(self.backbone(x))) + cls_logits, bbox_preds, centerness = self.bbox_head( + f, {"embedded": self.text_embedded}) + return tuple(cls_logits) + tuple(bbox_preds) + tuple(centerness) + + +def is_text_conditional(det): + """이 검출기가 `MMDetTextCond` 로 감쌀 수 있는가. + + GLIP 만이다. GroundingDINO 계열은 head 가 DETR 꼴이라 `forward` 규약이 다르다 — + 같은 래퍼로 감싸면 인자 개수에서 죽는다. 여기서 **명시적으로** 갈라 둔다. + """ + if getattr(det, "language_model", None) is None: + return False + return type(getattr(det, "bbox_head", None)).__name__ == "ATSSVLFusionHead" + + def _tolist(v): try: return list(v) @@ -525,6 +588,9 @@ def _last_conv(seq): img_mean = [float(v) for v in dp.mean.flatten().tolist()] if getattr(dp, "std", None) is not None: img_std = [float(v) for v in dp.std.flatten().tolist()] + # 채널 순서는 정규화 여부와 무관하다. YOLOX 처럼 mean/std 가 없는(정규화를 끈) + # 모델도 BGR 을 받으므로, 이 줄이 위 분기 안에 있으면 RGB 를 그대로 먹인다. + if dp is not None: to_rgb = not bool(getattr(dp, "_channel_conversion", False)) # ── 디코드 임계값: mmdet 의 test_cfg 가 정본이다 ──────────────────────────── @@ -640,7 +706,7 @@ def _tc(key, default): } -def build(config, checkpoint=None, size=512): +def build(config, checkpoint=None, size=512, caption=None): """mmdet config(.py) → (MMDetBackbone(eval), feature shapes, postproc cfg).""" from mmdet.apis import init_detector # mmdet 만 import (g2c 무관) trace_friendly_ops() # trace 가 삼키는 커스텀 op 을 등가 수식으로 @@ -652,6 +718,17 @@ def build(config, checkpoint=None, size=512): det = init_detector(config, checkpoint, device="cpu") det.eval() _unwrap_checkpoint_forward(det) + # 텍스트 조건부(GLIP)는 **융합헤드까지** 한 그래프에 넣는다 — 캡션이 고정이면 텍스트 + # 임베딩이 상수라 입력이 이미지 하나로 줄어든다(`MMDetTextCond` 주석 참고). + # 그러면 `_no_box` 경로의 "그래프 출력 대조" 가 곧 **헤드 검증**이 된다. + if is_text_conditional(det): + m = MMDetTextCond(det, caption or DEFAULT_CAPTION) + m.eval() + cfg = postproc_cfg(det) + cfg["img_size"] = int(size) + with torch.no_grad(): + outs = m(torch.randn(1, 3, size, size)) + return m, [tuple(o.shape) for o in outs], cfg m = MMDetBackbone(det) m.eval() cfg = postproc_cfg(det) diff --git a/tools/frontend/mmpose/mmpose_wrap.py b/tools/frontend/mmpose/mmpose_wrap.py new file mode 100644 index 0000000..f189298 --- /dev/null +++ b/tools/frontend/mmpose/mmpose_wrap.py @@ -0,0 +1,205 @@ +"""mmpose_wrap.py — mmpose 추정기를 '이미지 하나 → head 출력' nn.Module 로 감싸는 부품. + +**클래스 전용 import 모듈**(스크립트로 직접 실행 금지) — `mmseg_wrap`·`mmdet_wrap` 과 같은 규약. + +mmseg 와 같은 이유로 단순하다: 앵커도 NMS 도 없다. topdown 계열은 이미 잘린 사람 이미지를 +받아 **히트맵**(또는 SimCC 의 1D 벡터)을 내고, 디코드는 argmax + 소수점 보정뿐이라 그래프 +밖이다. `backbone → (neck) → head` 를 한 그래프로 컴파일하고 출력 텐서를 대조한다. +""" +import os +import sys + +import torch +import torch.nn as nn + +_FE = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(_FE, "mmdet")) +import mmdet_compat # noqa: E402 +import mmdet_wrap # noqa: E402 + +trace_friendly_ops = mmdet_compat.patch_ops + + +def _stub_xtcocotools(): + """`xtcocotools` 를 `pycocotools` 로 잇는다 — **없으면 모델 로드조차 못 한다.** + + `mmpose.apis.init_model` 은 레지스트리를 채우려고 `mmpose.datasets` 를 통째로 import + 하는데, 그 안의 COCO 계열 데이터셋이 `from xtcocotools.coco import COCO` 를 한다. + xtcocotools 는 C 확장이라 이 환경에서 빌드가 실패한다(gcc exit 1). + + ⚠️ **우리는 데이터셋을 안 쓴다.** 재는 것은 `backbone → head` 의 텐서뿐이고, 이 심은 + import 를 통과시키는 용도다. 평가지표(AP 등)를 계산하려 들면 **이 심으로는 안 된다** — + xtcocotools 는 crowdpose/wholebody 용으로 확장된 것이라 API 가 더 넓다. + 그때는 진짜 xtcocotools 를 빌드해야 한다. + """ + if "xtcocotools.coco" in sys.modules or _has_module("xtcocotools"): + return + import types + import pycocotools.coco + import pycocotools.cocoeval + import pycocotools.mask + pkg = types.ModuleType("xtcocotools") + pkg.__path__ = [] + sys.modules["xtcocotools"] = pkg + for sub in ("coco", "cocoeval", "mask"): + sys.modules[f"xtcocotools.{sub}"] = getattr(pycocotools, sub) + setattr(pkg, sub, getattr(pycocotools, sub)) + + +def _mmpose_pkg_dir(): + import mmpose + return os.path.dirname(os.path.abspath(mmpose.__file__)) + + +class _chdir: + """`with` 로 cwd 를 잠깐 바꾼다 (py3.11 의 `contextlib.chdir` 과 같다).""" + + def __init__(self, path): + self.path, self.prev = path, None + + def __enter__(self): + self.prev = os.getcwd() + os.chdir(self.path) + return self + + def __exit__(self, *exc): + os.chdir(self.prev) + return False + + +def _has_module(name): + import importlib.util + try: + return importlib.util.find_spec(name) is not None + except Exception: + return False + + +# ⚠️ **import 시점에 건다.** `torch.load` 로 `MMPoseWrap` 를 되살리는 쪽(g2c 서브프로세스)은 +# 피클이 이 모듈을 먼저 import 하는데, 거기서 mmpose 가 딸려 들어온다. 함수 안에서만 +# 걸면 그 경로는 심을 못 보고 `ModuleNotFoundError: xtcocotools` 로 컴파일이 죽는다. +_stub_xtcocotools() + + +class MMPoseWrap(nn.Module): + """`backbone → (neck) → head.forward` 까지. 디코드(argmax·보정)는 그래프 밖. + + `head.forward` 는 mmpose 의 **순수 forward** 다(`base.py:_forward` 가 부르는 것과 같다). + `predict` 는 flip TTA·디코드가 붙어 있어 쓰지 않는다 — 그건 후처리다. + + 출력은 head 마다 다르다: + · `HeatmapHead` → (1, K, H/4, W/4) 텐서 하나 + · `SimCCHead` → (x, y) 1D 로짓 **두 개** (튜플로 그대로 흘린다) + """ + + def __init__(self, pose): + super().__init__() + self.backbone = pose.backbone + self.neck = pose.neck if getattr(pose, "with_neck", False) else None + self.head = pose.head + + def forward(self, x): + f = self.backbone(x) + if self.neck is not None: + f = self.neck(f) + out = self.head.forward(f) + # 중첩 출력을 평탄화하며 단일 출력도 튜플로 반환한다. + flat = [] + + def _walk(o): + if isinstance(o, torch.Tensor): + flat.append(o) + elif isinstance(o, (list, tuple)): + for e in o: + _walk(e) + elif isinstance(o, dict): + for _k in sorted(o): + _walk(o[_k]) + # 그 외(None·스칼라)는 그래프 밖이라 버린다. + + _walk(out) + return tuple(flat) + + +def build(config, checkpoint=None, size=(256, 192)): + """mmpose config(.py) → (MMPoseWrap(eval), 출력 shape 목록). + + ⚠️ **입력이 정방이 아니다.** topdown 은 사람 박스를 256x192(H x W)로 잘라 넣는다 — + 정방으로 넣으면 히트맵 크기가 config 와 안 맞아 head 가 죽거나 조용히 다른 것을 잰다. + """ + _stub_xtcocotools() + from mmpose.apis import init_model # mmpose 만 import (g2c 무관) + trace_friendly_ops() + mmdet_wrap.allow_mmengine_checkpoint_globals() + # ⚠️ **mmpose 의 dataset metainfo 경로는 cwd 기준이다.** config 가 + # `from_file='configs/_base_/datasets/aic.py'` 같은 **상대경로**를 들고 있고, + # `parse_pose_metainfo` 는 그걸 cwd 에서 찾는다. 폴백은 `mmpose/.mim/configs/` 인데 + # editable 설치에는 그 폴더가 없다 → `FileNotFoundError` 로 모델 로드가 죽는다. + # 레포 루트에서 부르고 되돌린다(산출물은 원래 cwd 에 쓰인다). + with _chdir(os.path.dirname(_mmpose_pkg_dir())): + pose = init_model(config, checkpoint, device="cpu") + pose.eval() + m = MMPoseWrap(pose) + m.eval() + shape = tuple(int(v) for v in size) if len(size) != 2 else (1, 3, int(size[0]), int(size[1])) + with torch.no_grad(): + outs = m(torch.randn(*shape)) + if isinstance(outs, torch.Tensor): + outs = (outs,) + return m, [tuple(o.shape) for o in outs] + + +def input_shape(config): + """torch 입력 shape 전체를 돌려준다 — 이미지면 `(1, 3, H, W)`. + + ⚠️ **입력이 이미지가 아닌 계열이 있다.** `PoseLifter`(h36m 의 TCN)는 2D 키포인트 + `[N, K*2, T]` 를 받는다. 이미지 크기로 재면 `conv1d` 가 4D 입력을 받아 죽는다 — + 그건 컴파일러가 못 하는 게 아니라 **잘못된 것을 먹인 것**이다. + """ + _stub_xtcocotools() + from mmengine.config import Config + cfg = Config.fromfile(config) + model = cfg.get("model") or {} + if str(model.get("type", "")) == "PoseLifter": + bb = model.get("backbone") or {} + ch = bb.get("in_channels") + if ch: + return (1, int(ch), 1) # T=1 (image-pose-lift). 시계열 계열은 미확인(방향모름) + h, w = input_size(config) + return (1, 3, h, w) + + +def input_size(config): + """config 의 `codec.input_size` → (H, W). 없으면 topdown 관례값 (256, 192). + + codec 은 `(W, H)` 순서다 — **torch 와 반대다.** 뒤집어 쓰면 히트맵이 전치돼 나오는데 + 정사각이 아니면 shape 에서 죽고, 정사각이면 조용히 틀린다. + """ + _stub_xtcocotools() + from mmengine.config import Config + cfg = Config.fromfile(config) + codec = cfg.get("codec") + if isinstance(codec, (list, tuple)): + codec = codec[0] + sz = (codec or {}).get("input_size") if codec else None + if sz and len(sz) == 2: + return int(sz[1]), int(sz[0]) + return 256, 192 + + +def save(model, path): + """`.pt` 를 쓰고 **여는 데 필요한 모듈을 그 옆에 전부 복사한다.** + + `torch.save` 는 클래스를 `__module__` 이름으로 절이므로 여는 쪽이 그 이름을 import + 할 수 있어야 한다. g2c 는 `.pt` 가 있는 디렉터리를 `sys.path` 에 넣으므로, 모듈이 + 거기 **있기만 하면** `PYTHONPATH` 없이 열린다. + + ⚠️ **자기 파일 하나로는 부족하다.** 이 래퍼는 `mmdet_wrap`·`mmdet_compat` 을 import + 하므로 그것들도 같이 날라야 한다 — 안 그러면 컴파일이 + `ModuleNotFoundError: No module named 'mmdet_compat'` 에서 멈춘다(2026-09-01 실측). + mmdet 쪽 `install_loader_modules` 가 그 일을 이미 한다. + """ + import torch + + torch.save(model, path) + return mmdet_compat.install_loader_modules(path, "mmpose_wrap", "mmdet_wrap") diff --git a/tools/frontend/mmseg/mmseg_wrap.py b/tools/frontend/mmseg/mmseg_wrap.py new file mode 100644 index 0000000..0ad7ad4 --- /dev/null +++ b/tools/frontend/mmseg/mmseg_wrap.py @@ -0,0 +1,447 @@ +"""mmseg_wrap.py — mmseg 세그멘터를 '이미지 하나 → seg_logits' nn.Module 로 감싸는 부품. + +**클래스 전용 import 모듈**(스크립트로 직접 실행 금지). `torch.save` 는 클래스를 +`__module__` 로 피클하므로, CLI(`mmseg_to_pt.py`)와 로더가 **같은 이름으로 import** 해야 +동일 클래스로 복원된다(mmdet 쪽 `mmdet_wrap` 과 같은 규약). + +**mmdet 보다 훨씬 단순하다** — 앵커도 NMS 도 박스 디코드도 없다. head 를 C++ 로 조립할 +이유가 없어 `decode_head` 까지 통째로 g2c 로 컴파일한다(GLIP 융합헤드에서 통한 경로). +""" +import inspect +import os +import sys + +import torch +import torch.nn as nn + +# mmdet 프론트엔드의 trace 호환 패치를 그대로 쓴다 — mmcv custom op(CARAFE·DCN 등)은 +# 두 라이브러리가 같은 것을 쓴다. 없으면 그 op 들이 trace 에서 삼켜진다. +_FE = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(_FE, "mmdet")) +import mmdet_compat # noqa: E402 +import mmdet_wrap # noqa: E402 + +trace_friendly_ops = mmdet_compat.patch_ops + + +class MMSegWrap(nn.Module): + """`backbone → (neck) → decode_head` 까지. 출력은 **resize 전** seg_logits. + + mmseg 는 `encode_decode` 에서 head 출력을 입력 크기로 bilinear resize 하는데, 그건 + **후처리**라 그래프 밖에 둔다 — 러너가 원하는 크기로 올린다. 그래프에 넣으면 입력 크기가 + 바뀔 때마다 다시 구워야 한다. + + `auxiliary_head` 는 학습 전용이라 뺀다(`predict` 경로가 안 부른다). + """ + + def __init__(self, seg): + super().__init__() + # ⚠️ **`backbone` 이 없는 세그멘터가 있다.** `MultimodalEncoderDecoder`(san)는 + # `image_encoder` + `text_encoder` 로 되어 있어 `seg.backbone` 을 무조건 집으면 + # `AttributeError: ... has no attribute 'backbone'` 로 죽는다. 「범위 밖」이 + # 아니라 **하네스가 못 세운 것**이었다(2026-08-31). + self.multimodal = (not hasattr(seg, "backbone") + and hasattr(seg, "image_encoder") + and hasattr(seg, "text_encoder")) + if self.multimodal: + self.backbone = None + self.neck = None + self.image_encoder = seg.image_encoder + self.text_encoder = seg.text_encoder + # `encode_decode` 가 이미지 인코더에만 줄인 입력을 넣는다(san 은 0.5배). + self.asymetric_input = getattr(seg, "asymetric_input", False) + self.encoder_resolution = getattr(seg, "encoder_resolution", 1.0) + else: + self.backbone = seg.backbone + self.neck = seg.neck if getattr(seg, "with_neck", False) else None + self.decode_head = seg.decode_head + self.cascade = isinstance(seg.decode_head, nn.ModuleList) + # ⚠️ **cascade 라고 다 `forward(x, prev)` 로 끝나지 않는다.** `PointHead`(point_rend)의 + # `forward` 는 `(fine_grained_point_feats, coarse_point_feats)` 라 특징 튜플을 + # 그대로 넘기면 `torch.cat` 이 `expected Tensor ... but got tuple` 로 죽는다. + # mmseg 자신도 `cascade_encoder_decoder.py:135` 에 "TODO support PointRend + # tensor mode" 라고 적어 두었다 — 추론 경로는 마지막 단계만 `predict` 다 + # (`encode_decode`: 0..n-2 는 forward, n-1 은 predict). + self.cascade_predict = ( + self.cascade + and "prev_output" in inspect.signature( + seg.decode_head[-1].predict).parameters + and "fine_grained_point_feats" in inspect.signature( + seg.decode_head[-1].forward).parameters + ) + self.test_cfg = getattr(seg, "test_cfg", None) + # ⚠️ **MaskFormer 계열의 head 는 `forward(x)` 가 아니다** — + # `forward(x, batch_data_samples)` 라 그냥 부르면 + # `forward() missing 1 required positional argument` 로 죽는다. + # 두 번째 인자는 이름과 달리 **입력 크기를 넘기는 통로**일 뿐이고 + # (mmseg `maskformer_head.py:151` 주석이 그렇게 말한다), 실제 추론 경로는 + # `predict(x, batch_img_metas, test_cfg)` 다. 그쪽으로 부른다. + self.needs_metas = ( + not self.cascade + and not self.multimodal + and "batch_data_samples" in inspect.signature( + seg.decode_head.forward).parameters + ) + + def forward(self, x): + # **계열 무관 프로브.** 값이 틀릴 때 백본/넥/헤드 중 어디서 벌어지는지 먼저 가른다 — + # 계열마다 프로브를 새로 짜지 않으려고 여기 둔다. 튜플을 그대로 돌려주면 + # 하네스가 `out_i` 로 각각 덤프하고 `ref.shapes.txt` 도 여러 줄이 된다. + _probe = os.environ.get("VISP_SEG_PROBE") + if self.multimodal: + return self._forward_multimodal(x, _probe) + f = self.backbone(x) + if _probe == "bb": + return f if isinstance(f, torch.Tensor) else tuple(f) + if self.neck is not None: + f = self.neck(f) + if _probe == "neck": + return f if isinstance(f, torch.Tensor) else tuple(f) + if self.cascade: + # ⚠️ **`CascadeEncoderDecoder` 의 head 는 `nn.ModuleList` 다.** 그냥 부르면 + # `Module [ModuleList] is missing the required forward` 로 죽는다 — + # 「범위 밖」처럼 보이지만 **하네스가 못 부른 것**이다(ocrnet·point_rend). + # 단계 규약은 mmseg `cascade_encoder_decoder.py:78` 과 같다: 0단계는 + # `forward(x)`, 이후는 `forward(x, prev)`. 마지막 단계의 `predict` 는 + # `forward(x, prev)` 뒤 **resize** 만 하는데 그건 그래프 밖이라 뺀다. + n = len(self.decode_head) + out = self.decode_head[0](f) + # `cascade_predict` 면 마지막 단계는 forward 가 아니라 predict 로 부른다. + # 그 외(ocrnet 등)는 예전 그대로 끝까지 forward 다. + for i in range(1, (n - 1) if self.cascade_predict else n): + out = self.decode_head[i](f, out) + if self.cascade_predict: + h, w = int(x.shape[-2]), int(x.shape[-1]) + metas = [{"img_shape": (h, w), "batch_input_shape": (h, w)}] + out = self.decode_head[n - 1].predict(f, out, metas, self.test_cfg) + elif self.needs_metas: + # `predict` 는 마스크를 **입력 크기로** 올린 뒤 클래스 점수와 곱해 + # seg_logits 를 만든다(`einsum('bqc,bqhw->bchw')`). 다른 계열의 + # 「resize 는 그래프 밖」 규약과 달리 이건 head 안의 계산이라 그래프에 둔다 — + # 여기서 빼면 남는 게 마스크 로짓이라 비교 대상이 달라진다. + h, w = int(x.shape[-2]), int(x.shape[-1]) + metas = [{"img_shape": (h, w), "batch_input_shape": (h, w)}] + probe = os.environ.get("VISP_SEG_PROBE") + if probe == "cs": + # cumsum 그 자체만. 여기서 틀리면 렌더러, 맞으면 PE 의 뒷단 + # (stride-2 슬라이스·stack·view)이 범인이다. + feat = f[-1] + mask = feat.new_zeros( + (int(feat.shape[0]), int(feat.shape[2]), int(feat.shape[3]))) + not_mask = 1 - mask + # ⚠️ 배치축(dim=0)에 붙이면 안 된다 — 하네스가 `t[0]` 로 배치를 + # 벗기므로 절반만 비교된다. 채널축으로 붙인다. + out = (torch.stack([not_mask.cumsum(1), not_mask.cumsum(2)], dim=1) + + feat[:, 0:1, :, :] * 0) + elif probe in ("pe", "pe_n", "pe_d"): + # 위치인코딩만. `cumsum` 이 여기 있고 MHA(baddbmm)는 없다 — + # 둘 중 누구인지 가르는 자리. + # `pe_n`·`pe_d` 는 그 안을 다시 셋으로 가른다. `pe` 는 전체다. + # pe_n → cumsum + normalize(⚠️ **음수 인덱스 슬라이스** `[:, -1:, :]`) + # pe_d → 거기에 dim_t 브로드캐스트 나눗셈까지 + # pe → 거기에 stride-2 슬라이스 + stack(dim=4) + view 까지 + feat = f[-1] + mask = feat.new_zeros( + (int(feat.shape[0]), int(feat.shape[2]), int(feat.shape[3]))) + zero = feat[:, :1] * 0 + if probe == "pe": + out = self.decode_head.decoder_pe(mask) + zero + else: + out = _pe_stage(self.decode_head.decoder_pe, mask, probe) + zero + elif probe in ("pix", "pix_ms"): + # ⚠️ **head 안을 픽셀 디코더와 트랜스포머 디코더로 가른다.** + # `pixel_decoder`(MSDeformAttn) 는 head 의 앞단이다. 여기서 틀리면 + # deform-attn 이 범인이고, 여기가 맞으면 뒤의 디코더 층이다. + mask_features, multi_scale_memorys = self.decode_head.pixel_decoder(f) + out = (mask_features if probe == "pix" + else _flatten_tensors(multi_scale_memorys)) + elif probe and ((probe.startswith("am") and probe[2:].isdigit()) or ( + probe.startswith("ami") and probe[3:].isdigit())): + # `_forward_head` 가 만드는 **attn_mask** 자체를 그래프 출력으로 뺀다. + # 계단 함수(`sigmoid()<0.5`)라 여기가 한 비트만 틀려도 뒤가 크게 벌어진다 — + # 「자를 의심하기 전에 자가 맞는지 재라」에 해당하는 자리. + head = self.decode_head + want = int(probe[3:] if probe.startswith("ami") else probe[2:]) + grabbed = {} + orig = head._forward_head + + import torch.nn.functional as _F + logits = probe.startswith("ami") + + def _spy(decoder_out, mask_feature, attn_mask_target_size): + cls_pred, mask_pred, attn_mask = orig( + decoder_out, mask_feature, attn_mask_target_size) + # `ami` 는 **문턱 전 로짓**을 잡는다. 여기가 통과선 안이면 + # 마스크 비트 차이는 계단 함수의 민감도지 별개의 버그가 아니다. + grabbed.setdefault( + len(grabbed), + _F.interpolate(mask_pred, attn_mask_target_size, + mode="bilinear", align_corners=False) + if logits else attn_mask) + return cls_pred, mask_pred, attn_mask + + head._forward_head = _spy + try: + from mmseg.structures import SegDataSample + head(f, [SegDataSample(metainfo=metas[0])]) + finally: + head._forward_head = orig + am = grabbed[want] + # bool → float. 하네스는 f32 만 비교한다. + out = am.to(torch.float32).unsqueeze(0) + elif probe and probe.startswith("mask") and probe[4:].isdigit(): + # 디코더 **층별** 마스크 예측. `all_mask_preds[0]` 은 층을 하나도 안 거친 + # 예측이다 — 거기서 맞으면 범인은 디코더 층(마스크드 어텐션)이다. + from mmseg.structures import SegDataSample + _, all_mask_preds = self.decode_head( + f, [SegDataSample(metainfo=metas[0])]) + out = all_mask_preds[int(probe[4:])] + elif probe == "mask": + # 값이 어디서 벌어지는지 가르는 자리 — 디코더까지만 내고 + # `predict` 의 interpolate·softmax·einsum 은 뺀다. + from mmseg.structures import SegDataSample + _, all_mask_preds = self.decode_head( + f, [SegDataSample(metainfo=metas[0])]) + out = all_mask_preds[-1] + else: + out = self.decode_head.predict(f, metas, None) + else: + out = self.decode_head(f) + # ⚠️ **항상 튜플이다.** 예전엔 출력이 하나면 벌거벗은 Tensor 를 돌려줬는데, + # `build()` 는 shape 을 늘 리스트로 알려줘서 **호출자가 두 겹 벗기는 사고**가 + # 났다 — `outs[0][0]` 이 19장짜리 클래스 맵에서 1장만 남겼다(2026-09-01 실측). + # 형태를 하나로 고정해 그 사고 자체를 없앤다. 러너는 `out_i` 로 순서대로 덤프한다. + return (out,) if isinstance(out, torch.Tensor) else tuple(out) + + def _forward_multimodal(self, x, probe=None): + """`MultimodalEncoderDecoder`(san) 전용 경로. + + mmseg `multimodal_encoder_decoder.py:121` 의 `encode_decode` 와 **같은 순서**다: + (필요하면 입력을 줄여) `image_encoder` → `text_encoder()` → + `decode_head.predict([원본입력, 시각특징, 클래스임베딩], metas, test_cfg)`. + 다르게 쓰면 재는 대상이 달라지므로 순서를 바꾸지 않는다. + + `text_encoder()` 는 입력을 안 받는다 — 클래스 이름 임베딩이라 이미지와 무관하다. + `predict` 안에서 마스크를 입력 크기로 올려 클래스 점수와 곱하므로 + (`einsum('bqc,bqhw->bchw')`) 그 resize 는 head 안의 계산이고 그래프에 남긴다 + — maskformer 계열(`needs_metas`)과 같은 사정이다. + """ + import torch.nn.functional as F + clip_x = x + if self.asymetric_input: + clip_x = F.interpolate(x, scale_factor=self.encoder_resolution, + mode="bilinear") + feats = self.image_encoder(clip_x) + if probe == "bb": + # ⚠️ **san 의 image_encoder 는 리스트 안에 리스트를 낸다.** 그냥 `tuple(feats)` + # 로 내면 하네스가 `.shape` 을 찾다 죽는다(REF_FAIL 로 보여 「프로브가 + # 안 되는 계열」처럼 읽힌다 — 실제로는 평탄화만 하면 된다). + return _flatten_tensors(feats) + cls_embeds = self.text_encoder() + h, w = int(x.shape[-2]), int(x.shape[-1]) + metas = [{"img_shape": (h, w), "batch_input_shape": (h, w)}] + if probe in ("san_pe", "san_fuse"): + # `encode_feature` 의 **앞머리만** 재현한다(mmseg san_head.py:114-137 과 같은 식). + # san_pe : patch_embed + pos_embed(필요하면 bicubic resize) + query 결합까지 + # san_fuse : 거기에 첫 CLIP 융합(fuse_clip)까지 + # 블록을 돌기 전에 이미 틀렸는지부터 가른다. + from mmseg.models.utils import resize as _resize + san = self.decode_head.side_adapter_network + xx, hwshape = san.patch_embed(x) + ori_h, ori_w = san.patch_embed.init_out_size + pos_embed = san.pos_embed + if san.pos_embed.shape[1] != xx.shape[1]: + pos_embed = _resize( + san.pos_embed.reshape(1, ori_h, ori_w, -1).permute(0, 3, 1, 2), + size=hwshape, mode="bicubic", align_corners=False, + ).flatten(2).permute(0, 2, 1) + pos_embed = torch.cat( + [san.query_pos_embed.expand(pos_embed.shape[0], -1, -1), pos_embed], dim=1) + xx = torch.cat([san.query_embed.expand(xx.shape[0], -1, -1), xx], dim=1) + xx = xx + pos_embed + if probe == "san_pe": + return xx + L = hwshape[0] * hwshape[1] + if san.fusion_index[0] == 0: + xx = san.fuse_clip(0, xx, feats[0][0], hwshape, L) + return xx + if probe == "san_enc": + # SAN 안을 다시 가른다 — `encode_feature`(경량 ViT + CLIP 융합)까지만 잰다. + # 여기서 맞으면 범인은 `decode_feature`(MLPMaskDecoder)다. + san = self.decode_head.side_adapter_network + return _flatten_tensors(san.encode_feature(x, feats, [])) + if probe in ("san_mask", "san_cls", "san_head"): + # `predict` 는 `forward` → `predict_by_feat`(업샘플·softmax·einsum) 이다. + # 그 둘을 갈라야 어느 쪽이 틀렸는지 짚인다 — `predict` 전체만 재면 + # 마스크 오류와 클래스 오류가 한 숫자로 섞인다. + mask_props, mask_logits = self.decode_head.forward([x, feats, cls_embeds], []) + if probe == "san_mask": + return _flatten_tensors(mask_props[-1]) + if probe == "san_cls": + return _flatten_tensors(mask_logits[-1]) + return _flatten_tensors((mask_props[-1], mask_logits[-1])) + out = self.decode_head.predict([x, feats, cls_embeds], metas, self.test_cfg) + return out if isinstance(out, torch.Tensor) else tuple(out) + + + +def _flatten_tensors(obj): + """중첩 list/tuple 을 텐서 튜플로 편다(프로브 전용). 텐서 하나면 그대로 돌려준다.""" + if isinstance(obj, torch.Tensor): + return obj + out = [] + + def _walk(o): + if isinstance(o, torch.Tensor): + out.append(o) + elif isinstance(o, (list, tuple)): + for e in o: + _walk(e) + elif isinstance(o, dict): + # san 의 `encode_feature` 는 `{'query':…, 'x':…}` 리스트를 낸다. + # 안 걸으면 빈 리스트가 돼 `out[0]` 이 IndexError 로 죽는다. + for k in sorted(o): + _walk(o[k]) + + _walk(obj) + return out[0] if len(out) == 1 else tuple(out) + + +def _pe_stage(pe, mask, stage): + """`SinePositionalEncoding.forward` 를 단계별로 끊어 낸다(프로브 전용). + + 본체(mmdet `positional_encoding.py`)와 **같은 식**을 그대로 옮긴 것이다 — + 다르게 쓰면 재는 대상이 달라진다. 출력은 항상 `[B, C, H, W]` 로 맞춘다: + 하네스가 `t[0]` 로 배치를 벗기므로 채널축에 실어야 전부 비교된다. + """ + B, H, W = mask.size() + mask = mask.to(torch.int) + not_mask = 1 - mask + y_embed = not_mask.cumsum(1, dtype=torch.float32) + x_embed = not_mask.cumsum(2, dtype=torch.float32) + if pe.normalize: + # ⚠️ 음수 인덱스 슬라이스다. 마지막 행/열을 집는 이 두 줄이 g2c 에서 + # 어떻게 나가는지가 미확인이었다. + y_embed = (y_embed + pe.offset) / (y_embed[:, -1:, :] + pe.eps) * pe.scale + x_embed = (x_embed + pe.offset) / (x_embed[:, :, -1:] + pe.eps) * pe.scale + if stage == "pe_n": + return torch.stack([y_embed, x_embed], dim=1) + dim_t = torch.arange(pe.num_feats, dtype=torch.float32) + dim_t = pe.temperature ** (2 * (dim_t // 2) / pe.num_feats) + pos_x = x_embed[:, :, :, None] / dim_t + pos_y = y_embed[:, :, :, None] / dim_t + if stage == "pe_d": + return torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) + raise ValueError(stage) + +def crop_size(config): + """config 의 `data_preprocessor.size` → (H, W). 없으면 (512, 512). + + ⚠️ **계열마다 다르다** — cityscapes 는 512x1024, ade20k 는 512x512, beit 는 640x640, + cgnet 은 680x680, bisenetv2 는 1024x1024. 하나로 고정하면 **잰 것이 그 모델이 아니다.** + ViT 계열은 조용히 넘어가지도 않는다: `pos_embed` 토큰 수가 입력에 묶여 있어 + `The size of tensor a (1025) must match ...` 로 죽는다(512/16 → 1024+1 vs 640/16 → 1600+1). + `data_preprocessor.size` 는 학습·평가 crop 이라 그 모델이 실제로 보는 크기다. + """ + from mmengine.config import Config + cfg = Config.fromfile(config) + sz = (cfg.get("model") or {}).get("data_preprocessor", {}).get("size") + if sz and len(sz) == 2: + return int(sz[0]), int(sz[1]) + return 512, 512 + + + +def _pin_nmf_random_bases(): + """SegNeXt(`LightHamHead`)의 NMF2D 를 **결정적**으로 만든다. + + `NMF2D._build_bases` 는 추론마다 `torch.rand` 로 기저를 새로 뽑는다(`rand_init=True` 가 + config 의 실제 값이다). 그래서 **torch 두 번이 서로 어긋난다** — 실측 rel L1 1.09e-02 + (2026-08-28) · 1.275e-02 (2026-08-31). 기준값 자체가 흔들리면 컴파일러를 못 잰다. + + 게다가 `aten::rand` 는 ggml 로 낼 수 없다 — 렌더러가 없어 passthrough 로 떨어지고 + 「낼 수 없는 op 1개」로 컴파일이 멈춘다. + + 고정 시드로 한 번 뽑아 **모듈 버퍼로 등록**한다. 그러면 + · 기준값(ref.py)과 컴파일(trace)이 **같은 값**을 쓴다 — 같은 프로세스에서 만든 + `seg.pt` 한 벌을 양쪽이 읽기 때문이다 + · 그래프에서 난수가 사라지고 GGUF 가중치가 된다 + `build()` 의 워밍업 forward 가 `torch.save` 보다 앞서므로 저장 시점에 버퍼가 잡혀 있다. + + ⚠️ **이건 원 모델과 다른 구성이다.** 원 모델은 매 추론 기저를 다시 뽑는다. 여기서 + 재는 것은 「고정 기저에서 컴파일러가 맞게 번역하는가」이지 「난수 초기화까지 같은가」가 + 아니다. 수치를 적을 때 이 조건을 같이 적어라. + """ + try: + from mmseg.models.decode_heads.ham_head import NMF2D + import torch.nn.functional as _F + except Exception: + return + if getattr(NMF2D, "_visp_pinned", False): + return + + def _build_bases(self, B, S, D, R, device=None): + # ⚠️ **버퍼 이름을 shape 으로 만들지 마라.** trace 중에는 `D = C // S` 가 정수가 + # 아니라 그래프 텐서로 와서 이름이 매번 달라지고, 버퍼가 새로 등록돼 + # `torch.jit.trace` 가 「state_dict changed after running the tracer」로 죽는다. + # 한 모델 안에서 shape 은 고정이므로 이름 하나면 된다. + buf = getattr(self, "visp_bases", None) + if buf is None: + n = int(B) * int(S) + g = torch.Generator().manual_seed(0) + bases = _F.normalize(torch.rand((n, int(D), int(R)), generator=g), dim=1) + # persistent=True — state_dict 에 남아야 GGUF 로 나간다. + self.register_buffer("visp_bases", bases, persistent=True) + buf = getattr(self, "visp_bases") + return buf + + NMF2D._build_bases = _build_bases + NMF2D._visp_pinned = True + + +# ⚠️ **모듈 import 시점에 건다.** `build()` 안에서만 걸면 기준값 프로세스에만 먹고 +# **컴파일 프로세스에는 안 먹는다** — 거기서는 `seg.pt` 를 언피클할 뿐 `build()` 를 +# 부르지 않기 때문이다(언피클이 이 모듈을 import 하므로 여기 두면 양쪽 다 걸린다). +# 실측: build() 안에만 뒀을 때 생성물에 `aten::rand` 가 그대로 남았다. +_pin_nmf_random_bases() + + +def build(config, checkpoint=None, size=512): + """mmseg config(.py) → (MMSegWrap(eval), 출력 shape 목록). + + `size` 는 int(정방) 또는 (H, W) 다. + """ + from mmseg.apis import init_model # mmseg 만 import (g2c 무관) + trace_friendly_ops() + _pin_nmf_random_bases() + # PyTorch 2.6 부터 `torch.load` 의 `weights_only` 기본값이 True 라 mmengine 이 넣은 + # 학습 메타에 걸린다. mmdet 쪽과 같은 사정이므로 그 구현을 재사용한다. + mmdet_wrap.allow_mmengine_checkpoint_globals() + seg = init_model(config, checkpoint, device="cpu") + seg.eval() + m = MMSegWrap(seg) + m.eval() + h, w = (size, size) if isinstance(size, int) else size + with torch.no_grad(): + outs = m(torch.randn(1, 3, h, w)) + if isinstance(outs, torch.Tensor): + outs = (outs,) + return m, [tuple(o.shape) for o in outs] + + +def save(model, path): + """`.pt` 를 쓰고 **여는 데 필요한 모듈을 그 옆에 전부 복사한다.** + + `torch.save` 는 클래스를 `__module__` 이름으로 절이므로 여는 쪽이 그 이름을 import + 할 수 있어야 한다. g2c 는 `.pt` 가 있는 디렉터리를 `sys.path` 에 넣으므로, 모듈이 + 거기 **있기만 하면** `PYTHONPATH` 없이 열린다. + + ⚠️ **자기 파일 하나로는 부족하다.** 이 래퍼는 `mmdet_wrap`·`mmdet_compat` 을 import + 하므로 그것들도 같이 날라야 한다 — 안 그러면 컴파일이 + `ModuleNotFoundError: No module named 'mmdet_compat'` 에서 멈춘다(2026-09-01 실측). + mmdet 쪽 `install_loader_modules` 가 그 일을 이미 한다. + """ + import torch + + torch.save(model, path) + return mmdet_compat.install_loader_modules(path, "mmseg_wrap", "mmdet_wrap") diff --git a/tools/gtx_smoke.cpp b/tools/gtx_smoke.cpp new file mode 100644 index 0000000..9ce196a --- /dev/null +++ b/tools/gtx_smoke.cpp @@ -0,0 +1,261 @@ +#include +// gtx_smoke.cpp — NPU 경로를 **계층별로** 최소 비용 검증한다. +// +// 모델 전체(LPRNet ~1시간, ISS)를 돌려 실패 지점을 찾는 건 너무 비싸다. 아래 사다리는 +// 각 단계가 한 겹씩만 확인하므로 초 단위로 어디서 깨지는지 갈린다: +// +// L0 backend : GTX 백엔드 init (라이브러리·transport) +// L1 alloc : GTX 버퍼 할당 (device memory) +// L2 roundtrip: host→device→host 데이터 왕복 (**DDR 직접 채널** — CPY 트랩이 난 층) +// L3 compute : 커널 1개 실행(ADD) 후 수치 대조 (RPC launch + 커널) +// L4 cpy : CONT/CPY 경로 (LPRNet 이 죽은 op) +// +// 텐서는 아주 작게(기본 64x32) 잡아 ISS 부담과 로그 폭증을 피한다. +// +// 빌드: vision.cpp/tools/build_gtx_smoke.sh +// 실행: gtx_smoke [원소수] (기본 2048) + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +static int g_fail = 0; + +static void step(const char* name, bool ok, const std::string& detail = "") { + printf(" [%s] %-12s %s\n", ok ? "PASS" : "FAIL", name, detail.c_str()); + if (!ok) ++g_fail; +} + + +// ── 커널별 크기 한계 찾기 ──────────────────────────────────────────── +// ⚠️ **커널마다 감당하는 크기가 다르고, 넘으면 조용히 틀린 값을 낸다.** +// ADD 는 전송을 블록으로 쪼개 65535 까지 맞지만, unary 계열(relu·clamp·leaky_relu)은 +// 쪼개지 않고 폭을 `(uint16_t)(elems * 2)` 로 만들어 **32768 부터 감긴다.** +// 한 커널만 재고 나머지를 대표시키면 그 결함이 통과한다 — 실측으로 ResNet-18 이 +// argmax 282 → 228 로 틀렸는데 ADD 만 재던 스모크는 ALL PASS 였다. +// `gtx_smoke --limits` 로 커널마다 통과하는 최대 크기를 이분해 표로 낸다. + +using build_fn = ggml_tensor * (*)(ggml_context *, ggml_tensor *); +using want_fn = double (*)(double); + +static ggml_tensor * bld_relu(ggml_context * c, ggml_tensor * x) { return ggml_relu(c, x); } +static ggml_tensor * bld_clamp(ggml_context * c, ggml_tensor * x) { return ggml_clamp(c, x, -5.0f, 5.0f); } +static ggml_tensor * bld_leaky(ggml_context * c, ggml_tensor * x) { return ggml_leaky_relu(c, x, 0.1f, false); } +static ggml_tensor * bld_neg(ggml_context * c, ggml_tensor * x) { return ggml_neg(c, x); } +static ggml_tensor * bld_scale(ggml_context * c, ggml_tensor * x) { return ggml_scale(c, x, 2.0f); } + +static double want_relu(double v) { return v > 0 ? v : 0.0; } +static double want_clamp(double v) { return v < -5.0 ? -5.0 : (v > 5.0 ? 5.0 : v); } +static double want_leaky(double v) { return v > 0 ? v : v * 0.1; } +static double want_neg(double v) { return -v; } +static double want_scale(double v) { return v * 2.0; } + +// 한 크기에서 그 op 이 맞는 값을 내는지 본다. 계산 실패도 틀린 것으로 센다. +static bool op_ok_at(ggml_backend_t gtx, build_fn build, want_fn want, int64_t n) { + ggml_init_params ip = {}; + ip.mem_size = ggml_tensor_overhead() * 64 + ggml_graph_overhead(); + ip.no_alloc = true; + ggml_context * c = ggml_init(ip); + if (c == nullptr) return false; + ggml_tensor * x = ggml_new_tensor_1d(c, GGML_TYPE_F32, n); + ggml_set_input(x); + ggml_tensor * y = build(c, x); + ggml_set_output(y); + ggml_cgraph * g = ggml_new_graph_custom(c, 256, false); + ggml_build_forward_expand(g, y); + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(c, gtx); + if (buf == nullptr) { ggml_free(c); return false; } + std::vector hx((size_t) n); + for (int64_t i = 0; i < n; ++i) hx[(size_t) i] = float((i % 41) - 20); + ggml_backend_tensor_set(x, hx.data(), 0, (size_t) n * sizeof(float)); + bool ok = ggml_backend_graph_compute(gtx, g) == GGML_STATUS_SUCCESS; + if (ok) { + std::vector got((size_t) n); + ggml_backend_tensor_get(y, got.data(), 0, (size_t) n * sizeof(float)); + for (int64_t i = 0; i < n && ok; ++i) { + if (std::fabs(got[(size_t) i] - want(hx[(size_t) i])) > 0.5) ok = false; + } + } + ggml_backend_buffer_free(buf); + ggml_free(c); + return ok; +} + +// 통과하는 최대 크기를 이분한다. lo 는 통과가 확인된 값, hi 는 실패가 확인된 값. +static int64_t find_limit(ggml_backend_t gtx, build_fn build, want_fn want, int64_t hi_cap) { + if (!op_ok_at(gtx, build, want, 1024)) return 0; // 작은 것도 못 하면 0 + int64_t lo = 1024, hi = hi_cap; + if (op_ok_at(gtx, build, want, hi)) return hi; // 상한까지 다 된다 + while (hi - lo > 1) { + int64_t mid = lo + (hi - lo) / 2; + if (op_ok_at(gtx, build, want, mid)) lo = mid; else hi = mid; + } + return lo; +} + +static void run_limits(ggml_backend_t gtx) { + struct entry { const char * name; build_fn build; want_fn want; }; + const entry ops[] = { + {"relu", bld_relu, want_relu}, + {"clamp", bld_clamp, want_clamp}, + {"leaky_relu", bld_leaky, want_leaky}, + {"neg", bld_neg, want_neg}, + {"scale", bld_scale, want_scale}, + }; + printf("커널별 크기 한계 (원소 수, 통과하는 최대값)\n"); + for (const entry & e : ops) { + const int64_t lim = find_limit(gtx, e.build, e.want, 131072); + printf(" %-12s %8lld", e.name, (long long) lim); + if (lim == 0) printf(" 장치가 안 맡는다(또는 작은 크기부터 틀린다)"); + else if (lim >= 131072) printf(" 상한(131072)까지 통과"); + else printf(" %lld 부터 틀린다", (long long) (lim + 1)); + printf("\n"); + } +} + +int main(int argc, char** argv) { + const bool want_limits = argc > 1 && std::string(argv[1]) == "--limits"; + const int64_t N = (!want_limits && argc > 1) ? atoll(argv[1]) : 2048; + if (!want_limits) { + printf("GTX 스모크 사다리 (원소 %lld)\n", (long long) N); + } + + // ── L0: 백엔드 init ────────────────────────────────────────────── + // GTX 백엔드는 ggml 밖(depend/ggml-gtx)에서 빌드되므로 ggml 등록 목록에 직접 넣는다. + ggml_backend_register(ggml_backend_gtx_reg()); + ggml_backend_t gtx = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr); + if (!want_limits) { + step("L0 backend", gtx != nullptr, gtx ? ggml_backend_name(gtx) : "GPU 백엔드 없음"); + } + if (!gtx) return 1; + if (want_limits) { + run_limits(gtx); + return 0; + } + + ggml_init_params ip{}; + ip.mem_size = 32u * 1024 * 1024; + ip.no_alloc = true; + ggml_context* ctx = ggml_init(ip); + + ggml_tensor* a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, N); ggml_set_input(a); + ggml_tensor* b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, N); ggml_set_input(b); + + // ── L1: 디바이스 버퍼 할당 ──────────────────────────────────────── + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, gtx); + step("L1 alloc", buf != nullptr, buf ? "" : "GTX 버퍼 할당 실패"); + if (!buf) return 2; + + // ── L2: host→device→host 왕복 (DDR 직접 채널) ───────────────────── + std::vector ha(N), hb(N), back(N); + for (int64_t i = 0; i < N; ++i) { ha[i] = (float)(i % 97) - 48.0f; hb[i] = (float)(i % 13) - 6.0f; } + ggml_backend_tensor_set(a, ha.data(), 0, N * sizeof(float)); + ggml_backend_tensor_set(b, hb.data(), 0, N * sizeof(float)); + ggml_backend_tensor_get(a, back.data(), 0, N * sizeof(float)); + double worst = 0; + for (int64_t i = 0; i < N; ++i) worst = std::fmax(worst, std::fabs(back[i] - ha[i])); + { + char d[64]; snprintf(d, sizeof d, "max|Δ|=%.3g", worst); + step("L2 roundtrip", worst == 0.0, d); + } + + // ── L3: 커널 1개 실행 (ADD) ────────────────────────────────────── + ggml_tensor* sum = ggml_add(ctx, a, b); + ggml_set_output(sum); + ggml_cgraph* cg = ggml_new_graph_custom(ctx, 256, false); + ggml_build_forward_expand(cg, sum); + ggml_backend_buffer_t obuf = ggml_backend_alloc_ctx_tensors(ctx, gtx); + + enum ggml_status st = ggml_backend_graph_compute(gtx, cg); + if (st != GGML_STATUS_SUCCESS) { + step("L3 compute", false, "graph_compute 실패(status=" + std::to_string((int)st) + ")"); + } else { + std::vector got(N); + ggml_backend_tensor_get(sum, got.data(), 0, N * sizeof(float)); + double w = 0; + for (int64_t i = 0; i < N; ++i) w = std::fmax(w, std::fabs(got[i] - (ha[i] + hb[i]))); + char d[64]; snprintf(d, sizeof d, "max|Δ|=%.3g (fp16 오차 허용)", w); + step("L3 compute", w < 0.5, d); + } + + // ── L3b: UNARY(ReLU) ───────────────────────────────────────────── + // ⚠️ **L3 는 ADD 하나만 돌린다.** 그래서 다른 커널이 틀려도 ALL PASS 가 나온다 — + // 실측(2026-09-09): ResNet-18 이 장치에서 argmax 282 → 228 로 틀렸는데 + // 스모크는 통과했다. 원인은 ReLU 였다. 커널마다 한 단계씩 둔다. + { + ggml_init_params ip3 = ip; + ggml_context* c3 = ggml_init(ip3); + // 모델의 relu 는 4차원 WHCN 이다. 1차원만 재면 모양·stride 취급을 못 본다. + const char* shp = std::getenv("GTX_SMOKE_RELU_4D"); + ggml_tensor* x = shp ? ggml_new_tensor_4d(c3, GGML_TYPE_F32, 56, 56, 64, 1) + : ggml_new_tensor_1d(c3, GGML_TYPE_F32, N); + ggml_set_input(x); + const int64_t NR = ggml_nelements(x); + ggml_tensor* r = ggml_relu(c3, x); + ggml_set_output(r); + ggml_cgraph* g3 = ggml_new_graph_custom(c3, 256, false); + ggml_build_forward_expand(g3, r); + ggml_backend_buffer_t rbuf = ggml_backend_alloc_ctx_tensors(c3, gtx); + std::vector hx(NR); + for (int64_t i = 0; i < NR; ++i) hx[i] = float((i % 41) - 20); // 음수·0·양수를 고루 + ggml_backend_tensor_set(x, hx.data(), 0, NR * sizeof(float)); + enum ggml_status st3 = ggml_backend_graph_compute(gtx, g3); + if (st3 != GGML_STATUS_SUCCESS) { + step("L3b relu", false, "graph_compute 실패(status=" + std::to_string((int)st3) + ")"); + } else { + std::vector got(NR); + ggml_backend_tensor_get(r, got.data(), 0, NR * sizeof(float)); + double w = 0; + int64_t bad = -1; + for (int64_t i = 0; i < NR; ++i) { + double want = hx[i] > 0 ? hx[i] : 0.0; + double diff = std::fabs(got[i] - want); + if (diff > w) { w = diff; bad = i; } + } + char d[128]; + snprintf(d, sizeof d, "max|Δ|=%.3g @i=%lld (got %.3g, want %.3g)", w, + (long long)bad, bad >= 0 ? got[bad] : 0.0, + bad >= 0 ? (hx[bad] > 0 ? hx[bad] : 0.0) : 0.0); + step("L3b relu", w < 0.5, d); + } + ggml_backend_buffer_free(rbuf); + ggml_free(c3); + } + + // ── L4: CPY/CONT 경로 (LPRNet 이 죽은 지점) ─────────────────────── + { + ggml_init_params ip2 = ip; + ggml_context* c2 = ggml_init(ip2); + ggml_tensor* x = ggml_new_tensor_2d(c2, GGML_TYPE_F32, 64, N / 64); ggml_set_input(x); + ggml_tensor* t = ggml_cont(c2, ggml_transpose(c2, x)); // CONT → 실제 복사 발생 + ggml_set_output(t); + ggml_cgraph* g2 = ggml_new_graph_custom(c2, 256, false); + ggml_build_forward_expand(g2, t); + ggml_backend_buffer_t b2 = ggml_backend_alloc_ctx_tensors(c2, gtx); + if (!b2) { + step("L4 cpy", false, "버퍼 할당 실패"); + } else { + ggml_backend_tensor_set(x, ha.data(), 0, N * sizeof(float)); + enum ggml_status s2 = ggml_backend_graph_compute(gtx, g2); + step("L4 cpy", s2 == GGML_STATUS_SUCCESS, + s2 == GGML_STATUS_SUCCESS ? "CONT(transpose) 성공" + : "status=" + std::to_string((int)s2)); + ggml_backend_buffer_free(b2); + } + ggml_free(c2); + } + + printf("\nRESULT: %s (%d 실패)\n", g_fail ? "FAIL" : "ALL PASS", g_fail); + if (obuf) ggml_backend_buffer_free(obuf); + ggml_backend_buffer_free(buf); + ggml_free(ctx); + ggml_backend_free(gtx); + return g_fail ? 3 : 0; +} diff --git a/tools/install_arch.py b/tools/install_arch.py index e45a528..67bf946 100644 --- a/tools/install_arch.py +++ b/tools/install_arch.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """g2c 산출물을 vision.cpp 라이브러리에 붙인다 (arch 등록). - python tools/install_arch.py [--name Yolo26m] [--detect-yolo] + python tools/install_arch.py [--name ResNet] [--mmdet] 하는 일: 1. `.cpp` / `.h` → `src/visp/arch/` 복사 @@ -9,8 +9,8 @@ 3. 끝. **CMake 는 안 건드린다** — `arch/*.cpp` 를 glob 으로 모은다. 왜 g2c 가 아니라 여기서 하나: - g2c 는 main 의 source of truth 라 이 경로의 사정을 떠안기지 않는다. 그리고 검출 - 파라미터(클래스 수·stride·NMS-free)는 **gguf 에 없다** — 여기서 채워 넣는다. + g2c 는 main 의 source of truth 라 이 경로의 사정을 떠안기지 않는다. 그리고 실행 + 파라미터(입력 크기·전처리·mmdet head 설정)는 **gguf 에 없다** — 여기서 채워 넣는다. ⚠️ 등록은 **공유 라이브러리 전제**다. static 으로 링크하면 아무도 참조하지 않는 전역 객체를 링커가 버려 등록이 통째로 사라진다(`--whole-archive` 필요) → `DECISIONS.md`. @@ -39,12 +39,17 @@ "toothbrush", ] +# mmdet 검출기는 head 파라미터가 60여 개라 명령줄로 받지 않는다. `mmdet_to_pt.py` 가 이미 +# `.postproc.h` 에 C++ 값으로 구워 놓았으므로 그 헤더를 arch/ 로 옮겨 등록 단위에서 +# 부른다. 함수 이름은 arch 마다 바꾼다 — 그대로 두면 두 번째 검출기를 설치할 때 같은 이름이 +# 두 번 정의된다. REGISTER_TU = '''// GENERATED BY tools/install_arch.py — 손으로 고치지 마라(재설치하면 덮어쓴다). // // 이 파일은 **아무도 부르지 않는다.** 전역 객체 생성자가 프로그램 시작 시 스스로 // `arch_registry` 에 등록한다. 그래서 CLI 에 case 를 추가할 필요가 없다. #include "visp/arch/{header}" #include "visp/arch_registry.h" +{extra_include} namespace {{ @@ -66,6 +71,36 @@ ''' +def _find_postproc(out_dir, cls): + """mmpt 가 만든 `*.postproc.h` 를 찾는다. + + 파일 이름은 `mmpt --out` 이 정하므로 g2c 클래스명과 다를 수 있다. 하나만 있으면 그것을 + 쓰고, 여러 개면 어느 것인지 사람이 정해야 한다. + """ + found = [] + for root, _dirs, files in os.walk(out_dir): + found += [os.path.join(root, fn) for fn in files if fn.endswith(".postproc.h")] + if not found: + sys.exit(f"error: *.postproc.h 가 {out_dir} 에 없다. mmpt 로 다시 내보낸다") + if len(found) > 1: + sys.exit("error: postproc.h 가 여럿이다 — " + ", ".join(found)) + return found[0] + + +def _install_postproc(src, dst, cls): + """arch/ 로 옮기면서 include 경로와 함수 이름을 고친다. + + 생성 헤더는 이름이 `mmdet_params` 하나다. 검출기를 둘 이상 설치하면 같은 이름이 두 번 + 정의되어 링크가 깨지므로, 라이브러리에 들어가는 사본만 클래스 이름을 붙인다. + """ + with open(src, encoding="utf-8") as f: + text = f.read() + text = text.replace('#include "head.h"', '#include "visp/mmdet_head.h"') + text = text.replace("mmdet_params()", cls + "_mmdet_params()") + with open(dst, "w", encoding="utf-8") as f: + f.write(text) + + def find_outputs(out_dir, name): """생성 폴더에서 (클래스명, .cpp, .h) 를 찾는다.""" cpps = [f for f in os.listdir(out_dir) if f.endswith(".cpp")] @@ -95,22 +130,6 @@ def gguf_arch(out_dir, cls): return m.group(1) -def detect_shapes(out_dir, cls): - """생성 .cpp 의 출력 등록에서 (클래스 수, 앵커 수) 를 추정한다. - - `compute_graph_output(..., "out_i")` 앞의 텐서 shape 를 직접 읽을 수는 없으므로, - **weights_manifest 의 마지막 conv 출력 채널**로 클래스 수를 잡는다. 실패하면 80. - """ - nc = 80 - wt = os.path.join(out_dir, cls + ".weights.txt") - if os.path.exists(wt): - for line in open(wt, encoding="utf-8"): - m = re.search(r"cv3.*?\[(\d+),", line) - if m: - nc = int(m.group(1)) - return nc - - def traced_size(out_dir, cls, fallback): """생성 `.py` 에서 trace 입력 크기를 읽는다. @@ -122,24 +141,25 @@ def traced_size(out_dir, cls, fallback): try: src = open(path, encoding="utf-8").read() except OSError: - return fallback + return fallback, fallback m = re.search(r"np\.random\.randn\(\s*\d+\s*,\s*\d+\s*,\s*(\d+)\s*,\s*(\d+)", src) if not m: - return fallback + return fallback, fallback h, w = int(m.group(1)), int(m.group(2)) if h != w: - print(f" ⚠ 입력이 정사각이 아니다({h}x{w}) — {h} 로 등록한다. 필요하면 --size 로 덮어써라") - return h + print(f" · 입력이 정사각이 아니다 — {h}x{w} 를 그대로 등록한다") + return h, w def main(): ap = argparse.ArgumentParser() ap.add_argument("out_dir", help="g2c --output 폴더") - ap.add_argument("--name", help="클래스명(예: Yolo26m). 생략하면 .cpp 가 하나일 때 자동") - ap.add_argument("--detect-yolo", action="store_true", - help="YOLO dense 검출기로 등록(디코드+박스 그리기)") - ap.add_argument("--classes", type=int, default=0, help="클래스 수(0=자동)") - ap.add_argument("--strides", default="8,16,32") + ap.add_argument("--name", help="클래스명(예: ResNet). 생략하면 .cpp 가 하나일 때 자동") + ap.add_argument("--mmdet", action="store_true", + help="mmdet 검출기로 등록. mmpt 가 만든 .postproc.h 를 함께 설치해 " + "head 조립과 디코드 파라미터를 등록 단위에 싣는다") + ap.add_argument("--postproc", + help="mmpt 가 만든 <이름>.postproc.h 경로. 생략하면 출력 폴더에서 찾는다") ap.add_argument("--size", type=int, default=0, help="입력 해상도. 생략하면 생성 .py 에서 읽는다") ap.add_argument("--mean", default="0,0,0", @@ -147,9 +167,6 @@ def main(): "'123.675,116.28,103.53'") ap.add_argument("--std", default="255,255,255", help="전처리 표준편차. torchvision 분류기는 '58.395,57.12,57.375'") - ap.add_argument("--score-thr", type=float, default=0.25) - ap.add_argument("--nms", action="store_true", - help="NMS 를 건다(기본은 NMS-free — YOLOv10/26 은 one2one 이라 필요 없다)") a = ap.parse_args() cls, cpp, hdr = find_outputs(a.out_dir, a.name) @@ -161,7 +178,10 @@ def main(): shutil.copy(cpp, os.path.join(ARCH_DIR, cls + ".cpp")) shutil.copy(hdr, os.path.join(ARCH_DIR, header)) - size = a.size or traced_size(a.out_dir, cls, 640) + ih, iw = traced_size(a.out_dir, cls, 640) + if a.size: + ih = iw = a.size + size = ih # 옛 필드 호환 — 정사각이면 이 값 하나로 충분하다 def _triple(spec, what): v = [x.strip() for x in spec.split(",") if x.strip()] @@ -172,31 +192,46 @@ def _triple(spec, what): norm = [f" t.mean = {_triple(a.mean, 'mean')};", f" t.stdv = {_triple(a.std, 'std')};"] - if a.detect_yolo: - nc = a.classes or detect_shapes(a.out_dir, cls) - strides = [s.strip() for s in a.strides.split(",") if s.strip()] - names = COCO80 if nc == 80 else [] + extra_include = "" + if a.mmdet: + src = a.postproc or _find_postproc(a.out_dir, cls) + if not os.path.isfile(src): + sys.exit(f"error: {src} 가 없다") + pp = cls + ".postproc.h" + _install_postproc(src, os.path.join(ARCH_DIR, pp), cls) + extra_include = f'#include "visp/arch/{pp}"' body = [ - " t.kind = visp::arch_kind::detect_yolo;", - f" t.num_classes = {nc};", - " t.strides = {" + ", ".join(f"{s}.0f" for s in strides) + "};", - f" t.nms_free = {'false' if a.nms else 'true'};", - f" t.score_thr = {a.score_thr}f;", + " t.kind = visp::arch_kind::detect_mmdet;", + f" t.mmdet = visp::{cls}_mmdet_params();", f" t.input_size = {size};", - ] + norm - if names: - body.append(" t.class_names = {") - for i in range(0, len(names), 6): - body.append(" " + ", ".join(f'"{n}"' for n in names[i:i + 6]) + ",") - body.append(" };") + f" t.input_w = {iw};", + f" t.input_h = {ih};", + # 전처리 값은 mmdet 설정에서 이미 읽어 헤더에 들어 있다. 명령줄 --mean/--std 로 + # 다시 받으면 두 벌이 갈려 조용히 어긋난다. + " for (int i = 0; i < 3; ++i) {", + " t.mean[i] = t.mmdet.img_mean[i];", + " t.stdv[i] = t.mmdet.img_std[i];", + " }", + " t.to_rgb = t.mmdet.to_rgb;", + " t.num_classes = t.mmdet.det.num_classes;", + ] + body.append(" if (t.mmdet.det.num_classes == 80) {") + body.append(" t.class_names = {") + for i in range(0, len(COCO80), 6): + body.append(" " + ", ".join(f'"{n}"' for n in COCO80[i:i + 6]) + ",") + body.append(" };") + body.append(" }") else: # 검출기가 아니어도 크기는 실어야 한다 — 러너가 그 크기로 입력을 만든다. body = [" t.kind = visp::arch_kind::raw;", - f" t.input_size = {size};"] + norm + f" t.input_size = {size};", + f" t.input_w = {iw};", + f" t.input_h = {ih};"] + norm reg_path = os.path.join(ARCH_DIR, arch + "_register.cpp") with open(reg_path, "w", encoding="utf-8") as f: f.write(REGISTER_TU.format(header=header, cls=cls, arch=arch, + extra_include=extra_include, task_body="\n".join(body))) print(f"설치 완료: {cls} → arch '{arch}'") diff --git a/tools/verify/backbone/run_mmdet.cpp b/tools/verify/backbone/run_mmdet.cpp deleted file mode 100755 index 628a735..0000000 --- a/tools/verify/backbone/run_mmdet.cpp +++ /dev/null @@ -1,709 +0,0 @@ -// run_mmdet.cpp — mmdet 검출기 러너 (main 의 run_yolo_cpp.cpp 패턴). -// -// 백본 = g2c 가 생성한 output/.cpp (그대로 컴파일) → _forward -// head = tools/detect/head.cpp 부품 (러너와 함께 컴파일, 라이브러리 아님) -// decode+NMS = src/visp/postproc.cpp detect_anchor (라이브러리) -// cfg = .postproc.h (mmdet_params() from mmdet_to_pt.py, compiled in) -// -// 백본을 arch/ 로 복사하거나 cli REG 에 등록하지 않는다 — output/.cpp 를 직접 컴파일해 -// libvisioncpp 와 링크(build_mmdet_cpp.sh). run_yolo_cpp 와 동일한 -DARCH 매크로 방식. -// -// 컴파일: -DARCH=<클래스명> -DVISP_ARCH_HEADER='"/.h"' -// run: run_mmdet [size=512] -// an output ending in .bin holds raw f32 for comparison; anything else is an image - -#include VISP_ARCH_HEADER // 백본: _forward / _params / _detect_params -#include "head.h" - -#include // head 부품: anchor_head_forward (같은 폴더) -#include "draw.h" // draws detections onto the image (same folder) -#include MMDET_PARAMS_HEADER // generated mmdet_params(); values live in the binary -#include "visp/image.h" // image_load (이미지 입력 pre) -#include "visp/ml.h" -#include "visp/postproc.h" // detect_anchor, preprocess, det_params, detection - -#include - -#include -#include -#include -#include -#include -#include -#include - -using namespace visp; - -#define CAT_(a, b) a##b -#define CAT(a, b) CAT_(a, b) -#define FWD CAT(ARCH, _forward) -#define PARAMS_T CAT(ARCH, _params) -#define DETECT_PARAMS CAT(ARCH, _detect_params) - -static std::vector load_bin(const char* path, size_t n) { - std::vector v(n); - FILE* f = fopen(path, "rb"); - if (!f) { fprintf(stderr, "cannot open %s\n", path); exit(1); } - size_t got = fread(v.data(), sizeof(float), n, f); - fclose(f); - if (got != n) { fprintf(stderr, "short read %s: %zu/%zu\n", path, got, n); exit(1); } - return v; -} - -static std::vector to_vec(tensor t) { - int64_t ne = ggml_nelements(t); - std::vector d((size_t)ne); - transfer_from_backend(t, std::span(d.data(), d.size())); - return d; -} - -// An image input goes through preprocess() with the generated constants; -// a .bin is taken as an already pre-processed CWHN f32 tensor. -static bool has_ext(std::string const& s, const char* e) { - size_t n = std::strlen(e); - return s.size() >= n && s.compare(s.size() - n, n, e) == 0; -} - -static bool is_image_path(std::string const& s) { - return has_ext(s, ".jpg") || has_ext(s, ".jpeg") || has_ext(s, ".png") || has_ext(s, ".bmp"); -} - -// A non-empty `source` means the input was an image, so the result can be drawn on it. -static std::vector load_input(const char* path, int SZ, mmdet_cfg const& c, - image_data* source) { - std::string s(path); - if (is_image_path(s)) { - image_data img = image_load(path); - int iw = img.extent[0], ih = img.extent[1]; - int ic = n_channels(img.format); // stbi_load(...,0)=네이티브 채널수 (JPEG=3, PNG+α=4) - float const (&mean)[3] = c.img_mean; - float const (&sd)[3] = c.img_std; - bool to_rgb = c.to_rgb; - printf("- preprocess: image %dx%dx%d → %dx%d (mean %.1f,%.1f,%.1f std %.1f,%.1f,%.1f to_rgb=%d)\n", - iw, ih, ic, SZ, SZ, mean[0], mean[1], mean[2], sd[0], sd[1], sd[2], (int)to_rgb); - auto tensor_data = preprocess(img.data.get(), ih, iw, ic, SZ, mean, sd, to_rgb); - if (source) { - *source = std::move(img); - } - return tensor_data; - } - return load_bin(path, (size_t)3 * SZ * SZ); -} - -int main(int argc, char** argv) { - if (argc < 4) { - fprintf(stderr, - "usage: %s [size=512]\n" - " output ending in .bin holds raw float32 detections;\n" - " any other extension is an image with the boxes drawn on it\n", argv[0]); - return 1; - } - const char* gguf = argv[1]; - const char* inp = argv[2]; - const char* outp = argv[3]; - const int SZ = argc > 4 ? atoi(argv[4]) : 512; - - // 1) 가중치 (백본 + head 전부 이 gguf 에) - backend_device backend = backend_init(); - model_file file = model_load(gguf); - model_weights weights = model_init(file.n_tensors()); - model_transfer(file, weights, backend, backend.preferred_float_type(), file.tensor_layout()); - - compute_graph graph = compute_graph_init(131072); - model_ref m(weights, graph); - PARAMS_T p = DETECT_PARAMS(file); - - // 2) 백본 (g2c output/.cpp) — FPN features(out_0..L-1) 를 그래프에 조립 - tensor input = compute_graph_input(m, GGML_TYPE_F32, {3, SZ, SZ, 1}, "x"); - ggml_build_forward_expand(graph, input); - tensor bb = FWD(m, input, p); - ggml_build_forward_expand(graph, bb); - - // 3) Configuration -- constants fixed at compile time. No file is read. - mmdet_cfg cfg = mmdet_params(); - anchor_head_cfg& hc = cfg.head; - det_params& dp = cfg.det; - dp.input_w = SZ; - dp.input_h = SZ; - int L = (int)dp.strides.size(); - if (L == 0) { - fprintf(stderr, "no FPN strides — this config's head was not recognised at export\n"); - return 1; - } - - // 4) 백본 features(out_0..L-1) 를 잡아 head 부품 조립 - std::vector feats; - for (int l = 0; l < L; ++l) { - tensor f = ggml_graph_get_tensor(graph.graph, ("out_" + std::to_string(l)).c_str()); - if (!f) { fprintf(stderr, "백본 출력 out_%d 없음 (백본 .cpp 출력 규약 확인)\n", l); return 3; } - feats.push_back(f); - } - // deformable 을 쓰는 계열(vfnet/reppoints)은 고정 3×3 격자가 필요하다. mmdet 의 - // dcn_base_offset 과 같은 값 — [y,x] 가 번갈아 놓인 18개다. 계열이 안 쓰면 그냥 남는다. - // ⚠️ 안 쓰는 계열에서 만들면 그래프에 안 실려 버퍼가 없고, add_buffer 가 - // "tensor buffer not set" 으로 죽는다. 쓰는 계열에서만 만든다. - tensor dcn_base = nullptr; - if (hc.kind == head_kind::vfnet || hc.kind == head_kind::reppoints) { - dcn_base = compute_graph_input(m, GGML_TYPE_F32, {18, 1, 1, 1}, "dcn_base"); - tensor_data d = tensor_alloc(dcn_base); - std::span v = d.as_f32(); - for (int i = 0; i < 9; ++i) { - v[2 * i] = float(i / 3 - 1); // y: -1,-1,-1, 0,0,0, 1,1,1 - v[2 * i + 1] = float(i % 3 - 1); // x: -1,0,1 반복 - } - m.add_buffer(std::move(d)); - } - - // ── DDQ 만 다중 패스다 ────────────────────────────────────────────────── - // decoder 층마다 호스트 NMS 로 중복 query 를 가려야 해서 한 그래프로 못 돈다(head.h). - // 패스 0(백본+encoder) → NMS → 패스 1..N(층 하나씩, 사이마다 NMS). - if (hc.kind == head_kind::ddq) { - std::vector lv; - ddq_levels(feats, lv); - ddq_stage s0; - ddq_encode_forward(m, feats, hc, s0); - for (tensor x : {s0.mem, s0.om, s0.ecls, s0.ecoord, s0.qmap}) - ggml_build_forward_expand(graph, x); - compute_graph_allocate(graph, backend); - image_data src0; - auto in0 = load_input(inp, SZ, cfg, &src0); - transfer_to_backend(input, std::span(in0.data(), in0.size())); - compute(graph, backend); - - const int64_t T = s0.mem->ne[1], E = s0.mem->ne[0], NC = s0.ecls->ne[0]; - const int64_t NQ = hc.num_queries; - std::vector mem_v = to_vec(s0.mem), qm_v = to_vec(s0.qmap); - std::vector ec_v = to_vec(s0.ecls), eb_v = to_vec(s0.ecoord); - - // 호스트: 점수 = sigmoid(클래스 최댓값), 박스 = cxcywh→xyxy(sigmoid(coord)). - auto sigm = [](float v) { return 1.0f / (1.0f + std::exp(-v)); }; - std::vector dets((size_t)T); - for (int64_t i = 0; i < T; ++i) { - float best = -1e30f; - for (int64_t k = 0; k < NC; ++k) best = std::max(best, ec_v[i * NC + k]); - float cx = sigm(eb_v[i * 4 + 0]), cy = sigm(eb_v[i * 4 + 1]); - float w = sigm(eb_v[i * 4 + 2]), h = sigm(eb_v[i * 4 + 3]); - dets[i] = {cx - w * 0.5f, cy - h * 0.5f, cx + w * 0.5f, cy + h * 0.5f, sigm(best), 0}; - } - // 이미 있는 NMS 를 그대로 쓴다(postproc.cpp — "mmcv nms 동일"). - std::vector keep = nms(dets, hc.ddq_iou_thr); - if ((int64_t)keep.size() > NQ) keep.resize((size_t)NQ); - if ((int64_t)keep.size() < NQ) { - fprintf(stderr, "ddq: NMS 후 %zu 개 < num_queries %lld — 그래프 모양이 안 맞는다\n", - keep.size(), (long long)NQ); - return 5; - } - // DDQ 는 query 내용을 학습 embedding 이 아니라 **선택된 위치의 feature** 로 채운다. - std::vector q_v((size_t)E * NQ), r_v((size_t)4 * NQ); - for (int64_t i = 0; i < NQ; ++i) { - const int64_t t0 = keep[(size_t)i]; - std::copy_n(&qm_v[(size_t)t0 * E], E, &q_v[(size_t)i * E]); - for (int k = 0; k < 4; ++k) r_v[(size_t)i * 4 + k] = sigm(eb_v[(size_t)t0 * 4 + k]); - } - fprintf(stderr, "[ddq] NMS 후보 %lld → 유지 %zu (임계 %.2f)\n", - (long long)T, keep.size(), hc.ddq_iou_thr); - std::vector mask_v((size_t)NQ * NQ, 0.0f); // 가산 마스크(0 = 통과) - std::vector alive((size_t)NQ, 1); // 0층은 전부 살아있다 - - std::vector> cls_out, box_out; - for (int li = 0; li < hc.dec_layers; ++li) { - compute_graph g = compute_graph_init(65536); - model_ref mi(weights, g); - tensor memT = compute_graph_input(mi, GGML_TYPE_F32, {E, T, 1, 1}, "mem"); - tensor qT = compute_graph_input(mi, GGML_TYPE_F32, {E, NQ, 1, 1}, "q"); - tensor rT = compute_graph_input(mi, GGML_TYPE_F32, {4, NQ, 1, 1}, "ref"); - tensor kT = compute_graph_input(mi, GGML_TYPE_F32, {NQ, NQ, 1, 1}, "mask"); - for (tensor x : {memT, qT, rT, kT}) ggml_build_forward_expand(g, x); - ddq_stage si; - ddq_layer_forward(mi, lv, hc, li, memT, qT, rT, kT, si); - for (tensor x : {si.query, si.ref, si.cls, si.box}) ggml_build_forward_expand(g, x); - compute_graph_allocate(g, backend); - transfer_to_backend(memT, std::span(mem_v.data(), mem_v.size())); - transfer_to_backend(qT, std::span(q_v.data(), q_v.size())); - transfer_to_backend(rT, std::span(r_v.data(), r_v.size())); - transfer_to_backend(kT, std::span(mask_v.data(), mask_v.size())); - compute(g, backend); - - cls_out.push_back(to_vec(si.cls)); - box_out.push_back(to_vec(si.box)); - q_v = to_vec(si.query); - r_v = to_vec(si.ref); - if (li + 1 < hc.dec_layers) { - // 다음 층 마스크. mmdet 규약 두 가지를 그대로 따른다: - // ① NMS 는 **지금까지 살아남은 query 끼리만** 돈다(집합이 단조 감소한다). - // ② 셀 (i,j) 는 **행이나 열 중 하나라도** 살아있으면 통과, 아니면 가린다. - // (mmdet 주석: "if it requires to keep index i, then all cells in row - // or column i should be kept") - std::vector ori; - for (int64_t i = 0; i < NQ; ++i) if (alive[(size_t)i]) ori.push_back((int)i); - std::vector dq(ori.size()); - for (size_t n = 0; n < ori.size(); ++n) { - const int64_t i = ori[n]; - float best = -1e30f; - for (int64_t k = 0; k < NC; ++k) - best = std::max(best, cls_out.back()[(size_t)i * NC + k]); - float cx = r_v[(size_t)i * 4 + 0], cy = r_v[(size_t)i * 4 + 1]; - float w = r_v[(size_t)i * 4 + 2], h = r_v[(size_t)i * 4 + 3]; - dq[n] = {cx - w * 0.5f, cy - h * 0.5f, cx + w * 0.5f, cy + h * 0.5f, - sigm(best), 0}; - } - std::vector next((size_t)NQ, 0); - for (int k : nms(dq, hc.ddq_iou_thr)) next[(size_t)ori[(size_t)k]] = 1; - alive.swap(next); - const float NEG = -1e30f; - for (int64_t i = 0; i < NQ; ++i) - for (int64_t j = 0; j < NQ; ++j) - mask_v[(size_t)i * NQ + j] = - (alive[(size_t)i] || alive[(size_t)j]) ? 0.0f : NEG; - } - } - if (const char* pre = std::getenv("MMDET_DUMP_HEAD")) { - auto dump = [&](const char* kind, size_t l, std::vector const& v) { - std::string path = std::string(pre) + "." + kind + "." + std::to_string(l) + ".bin"; - if (FILE* f = fopen(path.c_str(), "wb")) { - fwrite(v.data(), sizeof(float), v.size(), f); fclose(f); - } - }; - for (size_t l = 0; l < cls_out.size(); ++l) dump("cls", l, cls_out[l]); - for (size_t l = 0; l < box_out.size(); ++l) dump("box", l, box_out[l]); - printf("- ddq: %d 패스 (호스트 NMS %d 회)\n", hc.dec_layers + 1, hc.dec_layers); - } - - // ── 최종 디코드 — mmdet `DDQDETRHead.predict_by_feat` ───────────────── - // 마지막 층 출력에서 **살아남은(distinct) query 만** 골라 DETR top-k 를 탄다. - // 최종 NMS 는 없다 — 층 사이 NMS 가 이미 중복을 걸렀고, mmdet 도 - // `_predict_by_feat_single`(Deformable 상속)로 top-k 만 한다. - // ⚠️ 박스는 `box_out.back()`(= s.box, 마지막 층 hidden 에서 낸 sigmoid cxcywh)다. - // r_v(s.ref)는 **다음 층 참조점**이라 미묘하게 다르다 — 섞으면 값만 틀린다. - // ⚠️ alive 는 마지막 층에 들어간 마스크 그대로다(mmdet 의 - // `distinct_query_mask[-1]` 과 같다 — 갱신이 li+1 fcls, fbox; - std::vector const& lc = cls_out.back(); - std::vector const& lb = box_out.back(); - int n_alive = 0; - for (int64_t i = 0; i < NQ; ++i) { - if (!alive[(size_t)i]) continue; - fcls.insert(fcls.end(), lc.begin() + i * NC, lc.begin() + (i + 1) * NC); - fbox.insert(fbox.end(), lb.begin() + i * 4, lb.begin() + (i + 1) * 4); - ++n_alive; - } - detr_params qp; - qp.num_queries = n_alive; - qp.num_classes = dp.num_classes; // 배경 제외 수 (프론트엔드 규약) - qp.use_sigmoid = true; // deformable 계열 — 항상 sigmoid+focal - qp.max_per_img = dp.max_per_img; - qp.input_w = SZ; - qp.input_h = SZ; - std::vector dq = detect_detr(fcls.data(), fbox.data(), qp); - - std::string out_s0(outp); - bool raw0 = has_ext(out_s0, ".bin") || src0.extent[0] == 0; - if (raw0) { - FILE* f = fopen(outp, "wb"); - if (!f) { fprintf(stderr, "cannot write %s\n", outp); return 1; } - for (detection const& d : dq) { - float rec[6] = {d.x1, d.y1, d.x2, d.y2, d.score, (float)d.label}; - fwrite(rec, sizeof(float), 6, f); - } - fclose(f); - } else { - float thr0 = 0.3f; - if (const char* e = std::getenv("VISP_DRAW_THRESHOLD")) thr0 = (float)atof(e); - float sx = float(src0.extent[0]) / float(SZ); - float sy = float(src0.extent[1]) / float(SZ); - draw_detections(src0, dq, sx, sy, thr0); - image_save(src0, outp); - } - printf("- detect(ddq): 생존 query %d → %zu boxes → %s\n", - n_alive, dq.size(), outp); - } - return 0; - } - - std::vector dets; - head_outputs ho; - mmdet_head_forward(m, feats, hc, dcn_base, ho); - std::vector&cls_t = ho.cls, &box_t = ho.box; - for (tensor t : cls_t) ggml_build_forward_expand(graph, t); - for (tensor t : box_t) ggml_build_forward_expand(graph, t); - for (tensor t : ho.ctr) ggml_build_forward_expand(graph, t); - // ⚠️ extra 갈래도 **그래프에 넣어야** 버퍼가 생긴다. 안 넣고 읽으면 - // `GGML_ASSERT(buf != NULL && "tensor buffer not set")` 로 죽는다. - for (auto const& e : ho.extra) - for (tensor t : e.second) ggml_build_forward_expand(graph, t); - printf("- mmdet runner: 백본 out_0..%d + C++ head(%d convs, %s)\n", - L - 1, hc.stacked_convs, hc.cls_head.c_str()); - - // 5) 계산 (입력: 이미지면 preprocess, .bin 이면 전처리된 텐서) - compute_graph_allocate(graph, backend); - image_data source; - auto in_data = load_input(inp, SZ, cfg, &source); - if (const char* dp = std::getenv("MMDET_DUMP_PRE")) { // 디버그: 전처리 텐서 덤프 - FILE* f = fopen(dp, "wb"); if (f) { fwrite(in_data.data(), sizeof(float), in_data.size(), f); fclose(f); } - } - if (in_data.size() != (size_t)3 * SZ * SZ) { - fprintf(stderr, "입력 크기 %zu != %d (전처리/크기 확인)\n", in_data.size(), 3 * SZ * SZ); - return 2; - } - transfer_to_backend(input, std::span(in_data.data(), in_data.size())); - compute(graph, backend); - - // 5b) head 검증용 원시 덤프. **디코드 전** 값을 그대로 내보내 torch 의 bbox_head 출력과 - // 직접 대조한다 — NMS 를 거치면 어디가 틀렸는지 못 짚는다. - if (const char* pre = std::getenv("MMDET_DUMP_HEAD")) { - auto dump = [&](const char* kind, size_t l, tensor t) { - std::vector v = to_vec(t); - std::string path = std::string(pre) + "." + kind + "." + std::to_string(l) + ".bin"; - if (FILE* f = fopen(path.c_str(), "wb")) { - fwrite(v.data(), sizeof(float), v.size(), f); - fclose(f); - } - }; - // ⚠️ **head 입력(neck 출력)도 같이 낸다.** head 출력만 보면 "조립이 틀렸다" 와 - // "백본·neck 이 이미 틀렸다" 를 못 가른다 — head 를 고쳐도 안 낫는 쪽인지 - // 여기서 바로 갈린다(efficientnet·nas_fpn 이 실제로 neck 쪽이었다). - for (size_t l = 0; l < feats.size(); ++l) dump("feat", l, feats[l]); - for (size_t l = 0; l < cls_t.size(); ++l) dump("cls", l, cls_t[l]); - for (size_t l = 0; l < box_t.size(); ++l) dump("box", l, box_t[l]); - for (size_t l = 0; l < ho.ctr.size(); ++l) dump("ctr", l, ho.ctr[l]); - // 갈래가 셋을 넘는 계열(CornerHead 6갈래). 이름은 조립기가 정한다. - for (auto const& e : ho.extra) - for (size_t l = 0; l < e.second.size(); ++l) dump(e.first.c_str(), l, e.second[l]); - printf("- head raw dump: %s.{cls,box%s}..bin\n", pre, - ho.ctr.empty() ? "" : ",ctr"); - } - - // ── CornerNet / CentripetalNet ───────────────────────────────────────── - // 앵커가 없다 — 코너 heatmap 두 장에서 top-k 를 뽑아 쌍을 만든다. 갈래가 6~8개라 - // out.extra 를 쓰고, **마지막 레벨만** 쓴다(mmdet 도 `tl_heats[-1]` 이다). - bool corner_done = false; - if (hc.kind == head_kind::cornernet) { - if (ho.cls.empty() || ho.box.empty() || ho.ctr.empty() || ho.extra.empty()) { - fprintf(stderr, "corner head 출력이 비었다\n"); - return 4; - } - auto last = [](std::vector const& v) { return v.back(); }; - auto named = [&](const char* tag) -> std::vector { - for (auto const& e : ho.extra) - if (e.first == tag && !e.second.empty()) return to_vec(e.second.back()); - return {}; - }; - tensor t_cls = last(ho.cls); - const int cfh = (int)t_cls->ne[2], cfw = (int)t_cls->ne[1]; - corner_params cp; - cp.num_classes = dp.num_classes; - cp.topk = hc.corner_topk; - cp.local_max_kernel = hc.local_max_kernel; - cp.distance_threshold = hc.corner_distance_thr; - cp.score_thr = dp.score_thr; - cp.nms_thr = dp.nms_thr; - cp.max_per_img = dp.max_per_img; - cp.input_w = SZ; - cp.input_h = SZ; - cp.centripetal = hc.corner_centripetal; - dets = detect_corner(to_vec(t_cls), to_vec(last(ho.box)), - to_vec(last(ho.ctr)), named("brof"), - named("tlemb"), named("bremb"), - named("tlcs"), named("brcs"), - cfh, cfw, cp); - corner_done = true; // 아래 레벨별 수집·디스패치를 건너뛴다 - } - - // ── CenterNet ────────────────────────────────────────────────────────── - // 앵커가 없고 레벨도 하나다. heatmap 의 국소 최대점이 중심이고, 같은 자리의 - // wh/offset 이 상자를 만든다. heat 는 head 안에서 이미 sigmoid 를 거쳤다. - if (hc.kind == head_kind::centernet) { - if (ho.cls.empty() || ho.box.empty() || ho.ctr.empty()) { - fprintf(stderr, "centernet head 출력이 비었다(heatmap/wh/offset 필요)\n"); - return 4; - } - tensor t = ho.cls.back(); - centernet_params cp; - cp.num_classes = dp.num_classes; - cp.topk = hc.corner_topk; // test_cfg.topk (헤더가 같은 칸을 쓴다) - cp.local_max_kernel = hc.local_max_kernel; - cp.input_w = SZ; - cp.input_h = SZ; - dets = detect_centernet(to_vec(t), to_vec(ho.box.back()), to_vec(ho.ctr.back()), - (int)t->ne[2], (int)t->ne[1], cp); - corner_done = true; // 레벨별 수집·디스패치를 건너뛴다(같은 이유) - } - - // DETR 계열은 출력 인덱스가 **decoder 층**이라 레벨 수와 무관하다 — 아래 레벨 검사와 - // 레벨별 수집을 건너뛴다(head.h:172-175). - const bool is_detr = hc.kind == head_kind::detr || - hc.kind == head_kind::conditional_detr || - hc.kind == head_kind::dab_detr || - hc.kind == head_kind::deformable_detr || - hc.kind == head_kind::dino; - - // YOLOv3 는 레벨당 **한 갈래**다(na×(5+nc) 한 텐서에 box·obj·cls 가 다 들어 있다). - // box 갈래가 없는 게 정상이라 아래 검사에서 빼야 한다 — 안 빼면 조립은 됐는데 - // "레벨에 못 미친다" 로 멈춘다. - const bool single_branch = hc.kind == head_kind::yolo; - - // 조립기가 레벨 수를 못 채우면 아래 인덱싱이 널을 읽는다. 여기서 말한다. - if (!corner_done && !is_detr && !single_branch && - ((int)cls_t.size() < L || (int)box_t.size() < L)) { - fprintf(stderr, "head 출력이 %d 레벨에 못 미친다 (cls %zu, box %zu)\n", - L, cls_t.size(), box_t.size()); - return 4; - } - - // 6) raw cls/box(+ctr) → 계열별 디코드 - // 코너 계열은 위에서 이미 디코드했다 — 레벨별 수집을 돌 필요가 없다. - const int NL = (is_detr || corner_done) ? 0 : L; - std::vector> cls_v(NL), box_v(NL), ctr_v; - std::vector> feat_hw(NL); - for (int l = 0; l < NL; ++l) { - feat_hw[l] = { (int)cls_t[l]->ne[2], (int)cls_t[l]->ne[1] }; // (fh, fw) - cls_v[l] = to_vec(cls_t[l]); - // 한 갈래 계열(YOLOv3)은 box 텐서가 없다 — 읽으면 널 참조다. - if (!single_branch) box_v[l] = to_vec(box_t[l]); - } - // centerness 갈래가 있으면 score factor 로 넘긴다(ATSS·PAA·DDOD·FCOS…). mmdet 은 이걸 - // top-k 뒤에 곱한다 — 안 넘기면 **박스는 맞고 점수만** 높게 나온다(실측 Δ0.071). - // ⚠️ YOLACT 의 coeff 갈래는 점수가 아니다 — `ctr_tanh` 로 가른다. - if (!corner_done && (int)ho.ctr.size() >= L && !hc.ctr_tanh) { - ctr_v.resize(L); - for (int l = 0; l < L; ++l) ctr_v[l] = to_vec(ho.ctr[l]); - } - - // 디코드는 **조립과 다른 축**이다. 조립은 타워 모양(head_kind)으로 갈리지만, 디코드는 - // 박스가 무엇이냐로 갈린다 — 앵커 대비 delta 냐, 격자점 대비 거리냐. - // · anchor(Delta 코더) → detect_anchor - // · fcos·gfl·vfnet → detect_fcos (조립기가 이미 픽셀 거리로 만들어 놨다) - // · anchor 인데 거리 예측 → 같은 거리 경로 (RTMDet 의 bbox_mul_stride 가 그 표시) - // 앵커 파라미터가 안 실린 계열은 `octave_scales` 가 비어 있어 detect_anchor 가 - // 후보 0개를 낸다 — 그건 "조용히 틀린 박스" 가 아니라 안전한 정지다. - // RepPoints 도 이 경로를 탄다 — 격자점을 깔고 top-k·NMS 하는 부분이 같다. - // 박스 해석만 다르다(`box_xyxy_offset`). - const bool distance_box = hc.kind == head_kind::fcos || hc.kind == head_kind::gfl || - hc.kind == head_kind::vfnet || hc.kind == head_kind::reppoints || - hc.bbox_mul_stride; - - if (corner_done) { - // 위에서 채웠다 — 아무것도 안 한다. - } else if (is_detr) { - // ⚠️ DETR 계열은 out.cls/out.box 의 인덱스가 **FPN 레벨이 아니라 decoder 층**이다 - // (head.h:172-175). mmdet 도 `all_layers_*[-1]` 만 쓴다 — 앞 층을 쓰면 shape 가 - // 맞아 안 죽고 **더 거친 박스**가 나온다. 그림만 보면 알 수 없다. - if (ho.cls.empty() || ho.box.empty()) { - fprintf(stderr, "DETR head 출력이 비었다\n"); - return 4; - } - std::vector cls_last = to_vec(ho.cls.back()); - std::vector box_last = to_vec(ho.box.back()); - detr_params qp; - qp.num_queries = hc.num_queries; - qp.num_classes = dp.num_classes; // 배경을 뺀 수 (프론트엔드가 그렇게 싣는다) - qp.use_sigmoid = dp.use_sigmoid; // 고전 DETR=softmax, Deformable/DINO=sigmoid - qp.max_per_img = dp.max_per_img; - qp.input_w = dp.input_w; - qp.input_h = dp.input_h; - // query 수는 실제 텐서에서 다시 확인한다 — 설정과 어긋나면 조용히 잘못 읽는다. - const int q_actual = (int)ho.box.back()->ne[1]; - if (q_actual > 0 && q_actual != qp.num_queries) { - fprintf(stderr, "query 수가 설정(%d)과 텐서(%d)에서 다르다 — 텐서를 따른다\n", - qp.num_queries, q_actual); - qp.num_queries = q_actual; - } - dets = detect_detr(cls_last.data(), box_last.data(), qp); - } else if (hc.kind == head_kind::tood) { - // TOOD 는 조립기가 이미 `distance2bbox` 까지 그래프로 펴서 **격자 좌표계 xyxy** 를 - // 낸다(head.cpp:1260) — stride 만 곱하면 픽셀이다. - // ⚠️ cls 는 `sqrt(σ(logits)·σ(align))` 라 **이미 확률**이다(head.cpp:1218). - // 시그모이드를 또 걸면 박스는 그대로인데 점수만 낮아진다 — 눈에 안 띈다. - std::vector cand; - for (int l = 0; l < L; ++l) { - const int fh = feat_hw[l].first, fw = feat_hw[l].second, npos = fh * fw; - const float stride = dp.strides[l]; - float const* cs = cls_v[l].data(); - float const* bx = box_v[l].data(); - for (int pos = 0; pos < npos; ++pos) { - float const* c1 = cs + (size_t)pos * dp.num_classes; - int best = 0; - for (int j = 1; j < dp.num_classes; ++j) - if (c1[j] > c1[best]) best = j; - if (c1[best] <= dp.score_thr) continue; - float const* b1 = bx + (size_t)pos * 4; - detection d{b1[0] * stride, b1[1] * stride, b1[2] * stride, b1[3] * stride, - c1[best], best}; - if (dp.input_w > 0) { - d.x1 = std::min(std::max(d.x1, 0.0f), (float)dp.input_w); - d.x2 = std::min(std::max(d.x2, 0.0f), (float)dp.input_w); - d.y1 = std::min(std::max(d.y1, 0.0f), (float)dp.input_h); - d.y2 = std::min(std::max(d.y2, 0.0f), (float)dp.input_h); - } - cand.push_back(d); - } - } - int maxlabel = 0; - for (auto const& c : cand) maxlabel = std::max(maxlabel, c.label); - for (int lab = 0; lab <= maxlabel; ++lab) { - std::vector per; - for (auto const& c : cand) if (c.label == lab) per.push_back(c); - if (per.empty()) continue; - for (int k : nms(per, dp.nms_thr)) dets.push_back(per[k]); - } - std::sort(dets.begin(), dets.end(), - [](detection const& a, detection const& b) { return a.score > b.score; }); - if ((int)dets.size() > dp.max_per_img) dets.resize(dp.max_per_img); - } else if (hc.kind == head_kind::rpn) { - // 제안 생성. mmdet 은 클래스별이 아니라 **레벨별**로 NMS 를 걸고(level_ids), - // pre-NMS 점수 임계값이 없다 — `detect_anchor` 로 두면 둘 다 어긋난다. - rpn_params rp; - rp.strides = dp.strides; - rp.octave_base_scale = dp.octave_base_scale; - rp.octave_scales = dp.octave_scales; - rp.ratios = dp.ratios; - for (int k = 0; k < 4; ++k) { rp.means[k] = dp.means[k]; rp.stds[k] = dp.stds[k]; } - rp.nms_pre = dp.nms_pre; - rp.nms_thr = dp.nms_thr; - rp.max_per_img = dp.max_per_img; - rp.input_w = dp.input_w; - rp.input_h = dp.input_h; - dets = detect_rpn(cls_v, box_v, feat_hw, rp); - // `label` 은 레벨 번호로 돌아온다. 밖에서는 클래스 자리라 0(유일한 클래스)으로 둔다. - for (auto& d : dets) d.label = 0; - } else if (hc.kind == head_kind::sabl) { - // 변마다 버킷 분류 + 오프셋. 두 번째 box 갈래는 `extra[0]`("bcls") 로 온다. - sabl_params sp; - sp.strides = dp.strides; - sp.anchor_scale = dp.anchor_scale; - sp.num_buckets = dp.num_buckets; - sp.bucket_scale = dp.bucket_scale; - sp.num_classes = dp.num_classes; - sp.score_thr = dp.score_thr; - sp.nms_thr = dp.nms_thr; - sp.nms_pre = dp.nms_pre; - sp.max_per_img = dp.max_per_img; - sp.input_w = dp.input_w; - sp.input_h = dp.input_h; - std::vector> bcls_v(L); - if (!ho.extra.empty() && (int)ho.extra[0].second.size() >= L) - for (int l = 0; l < L; ++l) bcls_v[l] = to_vec(ho.extra[0].second[l]); - dets = detect_sabl(cls_v, bcls_v, box_v, feat_hw, sp); - } else if (hc.kind == head_kind::yolo) { - // YOLOv3: 레벨당 **한 갈래**(na×(5+nc))라 조립기가 `out.cls` 에만 담는다. - // 앵커가 (w,h) 쌍이고 objectness 로 먼저 거른다 — 전용 디코더로 보낸다. - yolov3_params yp; - yp.strides = dp.strides; - yp.base_sizes = dp.base_sizes; - yp.num_classes = dp.num_classes; - yp.conf_thr = dp.conf_thr; - yp.score_thr = dp.score_thr; - yp.nms_thr = dp.nms_thr; - yp.nms_pre = dp.nms_pre; - yp.max_per_img = dp.max_per_img; - yp.input_w = dp.input_w; - yp.input_h = dp.input_h; - dets = detect_yolov3(cls_v, feat_hw, yp); - } else if (hc.kind == head_kind::yolox) { - // 격자 단위 (dx,dy,log w,log h) + 별도 objectness. 코더가 없어 `c.det` 의 앵커 - // 파라미터가 안 실리고, 그대로 두면 detect_anchor 가 후보 0개를 낸다. - // ⚠️ obj 는 `out.ctr` 로 온다(프론트엔드가 conv_obj 를 centerness_head 로 싣는다). - // 비면 점수가 cls 만 남아 **박스는 맞고 점수만** 높게 나온다. - yolox_params yp; - yp.strides = dp.strides; - yp.prior_offset = hc.center_offset; // YOLOX 의 MlvlPointGenerator 는 offset=0 - yp.num_classes = dp.num_classes; - yp.score_thr = dp.score_thr; - yp.nms_thr = dp.nms_thr; - yp.max_per_img = dp.max_per_img; - yp.input_w = dp.input_w; - yp.input_h = dp.input_h; - if ((int)ctr_v.size() < L) { - fprintf(stderr, "YOLOX 인데 objectness 갈래가 %zu 레벨뿐이다 (필요 %d)\n", - ctr_v.size(), L); - return 4; - } - dets = detect_yolox(cls_v, box_v, ctr_v, feat_hw, yp); - } else if (distance_box) { - fcos_params fp; - fp.strides = dp.strides; - fp.num_classes = dp.num_classes; - fp.score_thr = dp.score_thr; - fp.nms_thr = dp.nms_thr; - fp.nms_pre = dp.nms_pre; - fp.max_per_img = dp.max_per_img; - fp.input_w = dp.input_w; - fp.input_h = dp.input_h; - fp.base_edge = dp.base_edge; // FoveaBox 만 채워져 온다 - // FCOS 만 MlvlPointGenerator(0.5) 다. 나머지는 AnchorGenerator(center_offset=0). - // ⚠️ **kind 로 0.5 를 박으면 안 된다.** 같은 `fcos` kind 라도 격자 오프셋이 다르다 — - // FCOS·NAS-FCOS·FoveaBox 는 `MlvlPointGenerator` 기본값 0.5 지만 **AutoAssign 은 - // `offset=0`** 으로 만든다(autoassign_head.py:171). 프론트엔드가 생성기에서 읽어 - // 실어 주므로(`_center_offset`) 그 값을 그대로 쓴다. - // 박으면 0.5·stride 만큼 밀리는데, 최대 stride 128 에서 **64px** 이다(실측 64.30px). - fp.point_offset = hc.center_offset; - fp.box_xyxy_offset = hc.kind == head_kind::reppoints; - dets = detect_fcos(cls_v, box_v, ctr_v, feat_hw, fp); - } else if (dp.score_voting) { - // PAA·LAD — 점수가 sqrt(cls×iou) 라 iou 갈래(ctr) 없이는 못 디코드한다. - if ((int)ctr_v.size() < L) { - fprintf(stderr, "PAA 인데 iou 갈래가 %zu 레벨뿐이다 (필요 %d)\n", ctr_v.size(), L); - return 4; - } - dets = detect_paa(cls_v, box_v, ctr_v, feat_hw, dp); - } else if (dp.fast_nms) { - // YOLACT — softmax + fast NMS. coeff 갈래(ctr_tanh)는 박스 검증에서 안 쓴다. - dets = detect_yolact(cls_v, box_v, feat_hw, dp); - } else { - dets = detect_anchor(cls_v, box_v, feat_hw, dp, ctr_v.empty() ? nullptr : &ctr_v); - } - - // An image by default, as with every other entry point here. Raw numbers on request. - std::string out_s(outp); - bool want_raw = has_ext(out_s, ".bin"); - if (!want_raw && source.extent[0] == 0) { - fprintf(stderr, "- input was a tensor, so there is no image to draw on; writing raw\n"); - want_raw = true; - } - - if (want_raw) { - FILE* f = fopen(outp, "wb"); - if (!f) { fprintf(stderr, "cannot write %s\n", outp); return 1; } - for (detection const& d : dets) { - float rec[6] = { d.x1, d.y1, d.x2, d.y2, d.score, (float)d.label }; - fwrite(rec, sizeof(float), 6, f); - } - fclose(f); - printf("- detect(anchor): %zu boxes → %s (x1,y1,x2,y2,score,label f32*6)\n", - dets.size(), outp); - } else { - float thr = 0.3f; - if (const char* e = std::getenv("VISP_DRAW_THRESHOLD")) { - thr = (float)atof(e); - } - // Coordinates are in the square input, so scale them back to the original resolution. - float sx = float(source.extent[0]) / float(SZ); - float sy = float(source.extent[1]) / float(SZ); - int drawn = draw_detections(source, dets, sx, sy, thr); - image_save(source, outp); - printf("- detect(anchor): %zu boxes, %d drawn at score >= %.2f → %s\n", - dets.size(), drawn, thr, outp); - } - - // The highest-scoring detections, so a run says something without opening the output. - // VISP_PRINT_DETS sets how many; 0 turns it off. - int n_print = 10; - if (const char* e = std::getenv("VISP_PRINT_DETS")) { - n_print = atoi(e); - } - n_print = std::min(n_print, (int)dets.size()); - if (n_print > 0) { - printf("\n %5s %8s %8s %8s %8s %8s %6s\n", "#", "x1", "y1", "x2", "y2", "score", "label"); - for (int i = 0; i < n_print; ++i) { - detection const& d = dets[i]; - printf(" %5d %8.1f %8.1f %8.1f %8.1f %8.3f %6d\n", - i, d.x1, d.y1, d.x2, d.y2, d.score, d.label); - } - if ((int)dets.size() > n_print) { - printf(" %5s (%zu more)\n", "...", dets.size() - n_print); - } - printf("\n"); - } - return 0; -} diff --git a/tools/verify/draw_boxes.py b/tools/verify/common/draw_boxes.py similarity index 97% rename from tools/verify/draw_boxes.py rename to tools/verify/common/draw_boxes.py index 0bf53a6..f3079ed 100644 --- a/tools/verify/draw_boxes.py +++ b/tools/verify/common/draw_boxes.py @@ -5,7 +5,7 @@ comparing against a reference implementation needs. This turns such a file into something to look at. - python tools/verify/draw_boxes.py image.jpg boxes.bin -o annotated.png + python tools/verify/common/draw_boxes.py image.jpg boxes.bin -o annotated.png Boxes are in the coordinate space of the resized square input the detector ran on, so the script scales them back to the original image. Pass --size if the detector ran at something diff --git a/tools/verify/backbone/run_dump.cpp b/tools/verify/common/run_dump.cpp similarity index 61% rename from tools/verify/backbone/run_dump.cpp rename to tools/verify/common/run_dump.cpp index f4a9a07..a4c27ca 100755 --- a/tools/verify/backbone/run_dump.cpp +++ b/tools/verify/common/run_dump.cpp @@ -28,13 +28,18 @@ static std::vector load_bin(const char* path, size_t n) { int main(int argc, char** argv) { if (argc < 4) { - fprintf(stderr, "usage: %s [size=512]\n", argv[0]); + fprintf(stderr, + "usage: %s [W=512] [H=W]\n", argv[0]); return 1; } const char* gguf = argv[1]; const char* inb = argv[2]; std::string pref = argv[3]; - const int SZ = argc > 4 ? atoi(argv[4]) : 512; + // ⚠️ **정방이 아닌 입력이 있다.** mmpose topdown 은 사람 박스를 256x192(HxW)로 잘라 + // 넣는다. 크기를 하나만 받으면 정방으로 굳어 히트맵 크기가 config 와 어긋난다. + // `H` 를 안 주면 예전처럼 정방이다 — 기존 호출자는 그대로다. + const int W = argc > 4 ? atoi(argv[4]) : 512; + const int H = argc > 5 ? atoi(argv[5]) : W; backend_device backend = backend_init(); model_file file = model_load(gguf); @@ -45,13 +50,31 @@ int main(int argc, char** argv) { model_ref m(weights, graph); auto p = DETECT_PARAMS(file); - tensor input = compute_graph_input(m, GGML_TYPE_F32, {3, SZ, SZ, 1}, "x"); + // ⚠️ **입력이 이미지가 아닌 모델이 있다.** mmpose 의 pose-lift(TCN)는 2D 키포인트 + // [N, K*2, T] 를 받는다. 그런 계열은 `VISP_IN_NE="ne0,ne1,ne2,ne3"` 로 입력 + // ne 를 직접 준다. 안 주면 예전처럼 이미지 cwhn 이다 — 기존 호출자는 그대로다. + int64_t ne[4] = {3, W, H, 1}; + size_t n_in = (size_t)3 * W * H; + if (const char* env = getenv("VISP_IN_NE")) { + std::string sv(env); + size_t pos = 0; + for (int i = 0; i < 4; ++i) { + size_t comma = sv.find(',', pos); + ne[i] = atoll(sv.substr(pos, comma == std::string::npos ? comma : comma - pos).c_str()); + if (ne[i] <= 0) ne[i] = 1; + if (comma == std::string::npos) { for (int j = i + 1; j < 4; ++j) ne[j] = 1; break; } + pos = comma + 1; + } + n_in = (size_t)ne[0] * ne[1] * ne[2] * ne[3]; + } + tensor input = compute_graph_input(m, GGML_TYPE_F32, + {ne[0], ne[1], ne[2], ne[3]}, "x"); // cwhn ggml_build_forward_expand(graph, input); tensor last = FWD(m, input, p); // 내부에서 compute_graph_output 이 out_i 를 그래프에 등록 ggml_build_forward_expand(graph, last); compute_graph_allocate(graph, backend); - auto in = load_bin(inb, (size_t)3 * SZ * SZ); + auto in = load_bin(inb, n_in); transfer_to_backend(input, std::span(in.data(), in.size())); compute(graph, backend); diff --git a/tools/verify/deploy_compare.py b/tools/verify/deploy_compare.py new file mode 100644 index 0000000..409aab4 --- /dev/null +++ b/tools/verify/deploy_compare.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""vision-cli 출력과 torch 레퍼런스를 상대 L1·L2 로 댄다. + + python3 deploy_compare.py --cli --ref --tol 0.05 + +cosine 을 쓰지 않는다. 스케일 불변이라 값이 배로 틀려도 1.0 이 나온다. +종료 코드 0 이면 통과, 1 이면 값 불일치, 2 면 비교 자체가 불가능한 상태다. +""" +import argparse +import glob +import json +import os +import sys + +import numpy as np + + +def load(prefix): + out = [] + for p in sorted(glob.glob(prefix + ".*.bin")): + out.append((p, np.fromfile(p, dtype=np.float32))) + return out + + +def main(): + ap = argparse.ArgumentParser(prog="deploy_compare") + ap.add_argument("--cli", required=True) + ap.add_argument("--ref", required=True) + ap.add_argument("--tol", type=float, default=0.05) + a = ap.parse_args() + + cli, ref = load(a.cli), load(a.ref) + if not cli: + print("vision-cli 출력이 없다") + return 2 + if not ref: + print("레퍼런스가 없다") + return 2 + if len(cli) != len(ref): + print(f"출력 개수가 다르다: cli {len(cli)} vs ref {len(ref)}") + return 2 + + worst_l1 = worst_l2 = 0.0 + lines = [] + for i, ((cp, c), (rp, r)) in enumerate(zip(cli, ref)): + if c.size != r.size: + print(f"out_{i} 원소 수가 다르다: cli {c.size} vs ref {r.size}") + return 2 + c64, r64 = c.astype(np.float64), r.astype(np.float64) + denom_l1 = np.abs(r64).sum() + denom_l2 = np.sqrt((r64 ** 2).sum()) + l1 = float(np.abs(c64 - r64).sum() / denom_l1) if denom_l1 else float("inf") + l2 = float(np.sqrt(((c64 - r64) ** 2).sum()) / denom_l2) if denom_l2 else float("inf") + worst_l1, worst_l2 = max(worst_l1, l1), max(worst_l2, l2) + lines.append(f"out_{i} n={c.size} relL1={l1:.4e} relL2={l2:.4e}") + + ok = worst_l1 <= a.tol and worst_l2 <= a.tol + for l in lines: + print(l) + print(f"worst relL1={worst_l1:.4e} relL2={worst_l2:.4e} tol={a.tol} " + f"{'PASS' if ok else 'FAIL'}") + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/verify/deploy_reference.py b/tools/verify/deploy_reference.py new file mode 100644 index 0000000..dbad247 --- /dev/null +++ b/tools/verify/deploy_reference.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""배포 경로 스윕의 레퍼런스를 만든다 — 원본 체크포인트를 torch 로 돌린 값. + + python3 deploy_reference.py --kind seg --config --checkpoint \ + --image --size 1024x512 --out + +g2c 산출물을 쓰지 않는다. 변환이 틀렸을 때 레퍼런스까지 같이 틀리면 검증이 성립하지 +않기 때문이다. 전처리는 vision-cli 와 같은 규약으로 맞춘다 — 단순 bilinear 리사이즈 뒤 +config 의 mean/std 로 정규화한다(letterbox 없음). + +출력: ..bin (fp32) + .shapes.json +""" +import argparse +import json +import os +import sys + +import numpy as np +import torch +from PIL import Image + +V = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")) + + +def build(kind, cfg, ckpt, size): + sys.path.insert(0, V + "/tools/frontend/" + ("mmseg" if kind == "seg" else "mmpose")) + sys.path.insert(0, V + "/tools/frontend/mmdet") + if kind == "seg": + import mmseg_wrap as wrap + else: + import mmpose_wrap as wrap + h, w = size[1], size[0] + model, _ = wrap.build(cfg, ckpt, (h, w)) + return model, wrap + + +def preprocess(image_path, size, mean, std): + """vision-cli 와 같은 규약 — 단순 bilinear 리사이즈 + mean/std 정규화.""" + w, h = size + img = Image.open(image_path).convert("RGB").resize((w, h), Image.BILINEAR) + x = np.asarray(img, dtype=np.float32) # HWC, 0~255 + x = (x - np.asarray(mean, dtype=np.float32)) / np.asarray(std, dtype=np.float32) + x = np.ascontiguousarray(x.transpose(2, 0, 1))[None] # NCHW + return torch.from_numpy(x) + + +def norm_values(cfg, kind): + """config 의 data_preprocessor 에서 mean/std 를 읽는다. 없으면 ImageNet 값.""" + try: + from mmengine.config import Config + c = Config.fromfile(cfg) + dp = c.get("model", {}).get("data_preprocessor", {}) or {} + mean = dp.get("mean") or [123.675, 116.28, 103.53] + std = dp.get("std") or [58.395, 57.12, 57.375] + return list(mean), list(std) + except Exception: + return [123.675, 116.28, 103.53], [58.395, 57.12, 57.375] + + +def main(): + ap = argparse.ArgumentParser(prog="deploy_reference") + ap.add_argument("--kind", choices=["seg", "pose"], required=True) + ap.add_argument("--config", required=True) + ap.add_argument("--checkpoint", required=True) + ap.add_argument("--image", required=True) + ap.add_argument("--size", required=True, help="WxH") + ap.add_argument("--out", required=True) + a = ap.parse_args() + + w, h = (int(v) for v in a.size.lower().split("x")) + model, _ = build(a.kind, a.config, a.checkpoint, (w, h)) + model.eval() + mean, std = norm_values(a.config, a.kind) + x = preprocess(a.image, (w, h), mean, std) + + with torch.no_grad(): + outs = model(x) + if isinstance(outs, torch.Tensor): + outs = (outs,) + outs = tuple(outs) + + shapes = [] + for i, t in enumerate(outs): + arr = t[0].detach().cpu().numpy().astype("float32") + # ⚠️ **torch 는 CHW 로 내고 `vision-cli` 는 HWC 로 덤프한다.** 돌려놓지 않으면 + # 원소 수가 같아 비교가 그냥 통과하고 값만 무상관이 된다. 무상관인 두 텐서의 + # 상대 L2 는 √2 라 계열마다 1.41 근처가 나오고, 모델별 결함처럼 보인다. + # 실측(ann): 그대로 대면 relL1 1.3817, HWC 로 돌리면 3.41e-02 로 통과한다. + if arr.ndim == 3: + arr = arr.transpose(1, 2, 0) + shapes.append(list(arr.shape)) + np.ascontiguousarray(arr).tofile(f"{a.out}.{i}.bin") + json.dump({"shapes": shapes, "layout": "HWC", "mean": mean, "std": std, + "size": [w, h]}, + open(a.out + ".shapes.json", "w")) + print(f"REF_OK outputs={len(outs)} size={w}x{h} mean={mean}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/verify/deploy_sweep.py b/tools/verify/deploy_sweep.py new file mode 100644 index 0000000..9def74a --- /dev/null +++ b/tools/verify/deploy_sweep.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""배포 경로 스윕 — 사용자가 치는 명령 그대로 계열을 돌려 값을 판정한다. + + python3 tools/verify/deploy_sweep.py seg --limit 5 + python3 tools/verify/deploy_sweep.py pose --limit 5 --skip 5 + +`run_dump` 하네스와 다른 점이 세 가지다. + +1. 입력이 난수 텐서가 아니라 **샘플 이미지**다. 난수는 실제 이미지가 만드는 활성값 + 분포를 재현하지 않는다. +2. 실행기가 `vision-cli` 다. 이미지 로드·리사이즈·정규화·등록 경로가 검사 범위에 들어온다. +3. 레퍼런스를 g2c 산출물에서 만들지 않는다. 원본 체크포인트를 torch 로 돌린 값과 댄다. + +판정은 상대 L1 으로 한다. cosine 은 스케일 불변이라 값이 배로 틀려도 1.0 이 나온다. +""" +import argparse +import json +import os +import subprocess +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) # tools/verify +V = os.path.abspath(os.path.join(HERE, "..", "..")) # vision.cpp +G2C = os.path.abspath(os.path.join(V, "..")) # g2c 루트 +PY = G2C + "/.venv/bin/python" +BUILD = os.environ.get("VISP_BUILD", V + "/build") +GGUF_PY = V + "/depend/llama/gguf-py" +IMAGE = V + "/tests/input/cat-and-hat.jpg" +WORK = os.environ.get("SWEEP_WORK", "/tmp/visp-deploy-sweep") +TOL = 0.05 # 상대 L1 통과선. mmdet 하네스와 같은 값을 쓴다 +TIMEOUT = 900 + +MM = { + "seg": os.path.expanduser("~/mmbuild/mmsegmentation"), + "pose": os.path.expanduser("~/mmbuild/mmpose"), +} + + +def run(cmd, cwd=None, env=None, timeout=TIMEOUT): + e = dict(os.environ) + e.setdefault("PYTHONPATH", GGUF_PY) + if env: + e.update(env) + p = subprocess.run(cmd, cwd=cwd, env=e, capture_output=True, text=True, timeout=timeout) + return p.returncode, (p.stdout or "") + (p.stderr or "") + + +def families(kind): + """metafile 에서 (계열, config, 체크포인트) 를 읽는다. 손목록을 쓰지 않는다.""" + import glob + import yaml + root = MM[kind] + ck_dir = root + "/checkpoints" + out = [] + for p in sorted(glob.glob(root + "/configs/*/metafile.y*ml")): + fam = os.path.basename(os.path.dirname(p)) + try: + models = yaml.safe_load(open(p)).get("Models") or [] + except Exception: + continue + if not models: + continue + m = models[0] + cfg = m["Config"] + cfg = cfg if cfg.startswith("/") else root + "/" + cfg + w = m.get("Weights", "") + ck = os.path.join(ck_dir, os.path.basename(w)) if w else "" + out.append((fam, cfg, ck)) + return out + + +def one(kind, fam, cfg, ck): + """계열 하나를 배포 경로로 돌린다. (판정, 상세) 를 돌려준다.""" + d = os.path.join(WORK, kind, fam) + os.makedirs(d, exist_ok=True) + name = "Fam" + fam.replace("-", "").replace("_", "").title()[:20] + arch = name.lower() + + if not ck or not os.path.exists(ck): + return "CKPT_NONE", "체크포인트 없음" + + # 1) mmpt — config 가 요구하는 크기로 trace 한 .pt 를 만든다 + rc, log = run([PY, "-m", "shared.frontend.mmpt", cfg, "--checkpoint", ck, + "--name", name, "--out", d], cwd=G2C) + if rc != 0: + return "EXPORT_FAIL", tail(log) + pt = os.path.join(d, name + ".pt") + size_txt = os.path.join(d, name + ".size.txt") + W, H = read_size(size_txt, log) + if not W: + return "SIZE_UNKNOWN", tail(log) + + # 2) g2c 변환 + rc, log = run([PY, "-m", "shared.compile.pipeline", "--model", pt, "--name", name, + "--input-shape", f"1,3,{H},{W}", "--output", os.path.join(d, "out")], + cwd=G2C) + if rc != 0 or "→ gguf:" not in log: + return "COMPILE_FAIL", tail(log) + gguf = os.path.join(d, "out", name + ".gguf") + + # 3) 등록 + 빌드 + # 정규화 값을 넘기지 않으면 install_arch 가 0-1 스케일로 등록한다. mmlab 모델은 + # ImageNet 통계를 전제하므로 config 의 data_preprocessor 값을 그대로 박는다. + mean, std = norm_values(cfg) + rc, log = run([PY, V + "/tools/install_arch.py", os.path.join(d, "out"), "--name", name, + "--mean", ",".join(str(v) for v in mean), + "--std", ",".join(str(v) for v in std)], cwd=G2C) + if rc != 0: + return "INSTALL_FAIL", tail(log) + rc, log = run(["cmake", "--build", BUILD, "-j", str(os.cpu_count() or 4)]) + if rc != 0: + return "BUILD_FAIL", tail(log) + + # 4) vision-cli — 사용자가 치는 그 명령 + out_prefix = os.path.join(d, "cli") + rc, log = run([BUILD + "/bin/vision-cli", arch, "-m", gguf, "-i", IMAGE, + "-o", out_prefix, "-b", "cpu"]) + if rc != 0: + return "CLI_FAIL", tail(log) + + # 5) torch 레퍼런스 — 같은 이미지, config 의 크기와 정규화 + rc, log = run([PY, HERE + "/deploy_reference.py", "--kind", kind, "--config", cfg, + "--checkpoint", ck, "--image", IMAGE, "--size", f"{W}x{H}", + "--out", os.path.join(d, "ref")], cwd=G2C) + if rc != 0: + return "REF_FAIL", tail(log) + + # 6) 판정 + rc, log = run([PY, HERE + "/deploy_compare.py", "--cli", out_prefix, + "--ref", os.path.join(d, "ref"), "--tol", str(TOL)]) + verdict = "PASS" if rc == 0 else "VALUE_FAIL" + return verdict, log.strip().splitlines()[-1] if log.strip() else "" + + +def norm_values(cfg): + """config 의 data_preprocessor 에서 mean/std 를 읽는다. 없으면 ImageNet 값.""" + try: + from mmengine.config import Config + dp = (Config.fromfile(cfg).get("model") or {}).get("data_preprocessor", {}) or {} + return list(dp.get("mean") or [123.675, 116.28, 103.53]), \ + list(dp.get("std") or [58.395, 57.12, 57.375]) + except Exception: + return [123.675, 116.28, 103.53], [58.395, 57.12, 57.375] + + +def read_size(path, log): + if os.path.exists(path): + try: + w, h = open(path).read().split() + return int(w), int(h) + except Exception: + pass + import re + m = re.search(r"Input:\s*1,3,(\d+),(\d+)", log) + if m: + return int(m.group(2)), int(m.group(1)) + return None, None + + +def tail(log, n=3): + lines = [l for l in log.strip().splitlines() if l.strip()] + return " / ".join(lines[-n:])[:300] + + +def main(): + ap = argparse.ArgumentParser(prog="deploy_sweep") + ap.add_argument("kind", choices=["seg", "pose"]) + ap.add_argument("--limit", type=int, default=5, help="이번에 돌릴 계열 수") + ap.add_argument("--skip", type=int, default=0, help="앞에서 건너뛸 계열 수") + ap.add_argument("--only", nargs="*", help="계열 이름을 직접 지정한다") + ap.add_argument("--list", action="store_true") + a = ap.parse_args() + + fams = families(a.kind) + if a.list: + for i, (f, _, ck) in enumerate(fams): + print(f"{i:3d} {f:24s} {'ckpt' if os.path.exists(ck) else 'NO-CKPT'}") + return 0 + if a.only: + fams = [f for f in fams if f[0] in a.only] + else: + fams = fams[a.skip:a.skip + a.limit] + + os.makedirs(WORK, exist_ok=True) + results = [] + for fam, cfg, ck in fams: + t0 = time.time() + try: + verdict, detail = one(a.kind, fam, cfg, ck) + except subprocess.TimeoutExpired: + verdict, detail = "TIMEOUT", f"{TIMEOUT}s 초과" + except Exception as e: + verdict, detail = "ERROR", str(e)[:200] + dt = time.time() - t0 + print(f"[{verdict:12s}] {fam:24s} {dt:6.1f}s {detail}", flush=True) + results.append({"family": fam, "verdict": verdict, "detail": detail, "sec": round(dt, 1)}) + + path = os.path.join(WORK, f"{a.kind}_result.json") + prev = json.load(open(path)) if os.path.exists(path) else [] + json.dump(prev + results, open(path, "w"), ensure_ascii=False, indent=1) + ok = sum(1 for r in results if r["verdict"] == "PASS") + print(f"\n{ok}/{len(results)} PASS → {path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/verify/backbone/run_frcnn.cpp b/tools/verify/mmdet/backbone/run_frcnn.cpp similarity index 89% rename from tools/verify/backbone/run_frcnn.cpp rename to tools/verify/mmdet/backbone/run_frcnn.cpp index e2b1506..d6bf555 100644 --- a/tools/verify/backbone/run_frcnn.cpp +++ b/tools/verify/mmdet/backbone/run_frcnn.cpp @@ -723,6 +723,46 @@ int main(int argc, char** argv) { // 박스는 마지막 단계가 이미 정제해 `rois` 에 들어 있다(단계마다 갱신된다). // NMS 경로로 흘리면 임계값이 0 이라 전부 걸러져 **0건**이 나온다(실측). if (SPARSE > 0) { + // ⚠️ **마지막 단계의 박스 정제가 빠져 있었다.** 위 정제 루프는 `st + 1 < NS` 라 + // 마지막 단계를 건너뛴다 — 캐스케이드는 그 단계의 delta 를 `detect_roi` 가 + // 디코드하므로 맞지만, SparseR-CNN 은 NMS 경로를 안 타서 **아무도 디코드하지 + // 않는다.** 그래서 박스만 한 단계 뒤처졌다. 점수는 마지막 단계 cls 라 정확해서 + // "라벨·개수·점수는 맞는데 박스만 몇 px 밀린" 모습으로 나온다 — 실측 4.02px, + // C++ 박스가 마지막 단계 **입력** 박스와 0.000px 로 같았다. + { + const bool agn2 = J.num("class_agnostic", 0.0f) != 0.0f; + if (!agn2) { + fprintf(stderr, "[sparse] class_agnostic 이 아니다 — 마지막 정제를 건너뛴다\n"); + } else { + const int lb = NS - 1; + const bool hs = st_stds.size() >= (size_t)(lb + 1) * 4; + const bool hm = st_means.size() >= (size_t)(lb + 1) * 4; + const float MAXR = 4.13516655f; // |log(16/1000)| — 루프와 같다 + std::vector nb((size_t)M * 4); + for (int i = 0; i < M_real; ++i) { + float d[4]; + for (int k = 0; k < 4; ++k) { + d[k] = box_st[lb][(size_t)i * 4 + k]; + if (hs) d[k] *= st_stds[(size_t)lb * 4 + k]; + if (hm) d[k] += st_means[(size_t)lb * 4 + k]; + } + d[2] = std::min(std::max(d[2], -MAXR), MAXR); + d[3] = std::min(std::max(d[3], -MAXR), MAXR); + const float x1 = rois[(size_t)i * 4 + 0], y1 = rois[(size_t)i * 4 + 1]; + const float x2 = rois[(size_t)i * 4 + 2], y2 = rois[(size_t)i * 4 + 3]; + const float pw = x2 - x1, ph = y2 - y1; + const float cx = x1 + pw * 0.5f + d[0] * pw, cy = y1 + ph * 0.5f + d[1] * ph; + const float w = pw * std::exp(d[2]), h = ph * std::exp(d[3]); + const float bx1 = cx - w * 0.5f, by1 = cy - h * 0.5f; + const float bx2 = cx + w * 0.5f, by2 = cy + h * 0.5f; + nb[(size_t)i * 4 + 0] = RCLIP ? std::max(0.0f, bx1) : bx1; + nb[(size_t)i * 4 + 1] = RCLIP ? std::max(0.0f, by1) : by1; + nb[(size_t)i * 4 + 2] = RCLIP ? std::min((float)SZ, bx2) : bx2; + nb[(size_t)i * 4 + 3] = RCLIP ? std::min((float)SZ, by2) : by2; + } + rois.swap(nb); + } + } const int NCLS2 = (int)(cls_st[NS - 1].size() / M) ; // sigmoid head — 배경 없음 const int KMAX = (int)J.num("rcnn_max", 100.0f); struct cand { float sc; int q, c; }; @@ -743,12 +783,19 @@ int main(int argc, char** argv) { } } -#if defined(ARCH_C) && defined(ARCH_D) - // ── 패스 2: 마스크 IoU 로 점수를 다시 매긴다 (Mask Scoring R-CNN) ─────── +#ifdef ARCH_C + // ── 패스 2: 마스크 head 를 돌린다 ─────────────────────────────────────── + // 두 가지를 겸한다. + // ⓐ **마스크 로짓을 낸다**(`.mlogit.bin`) — 마스크 축 대조용. 모든 마스크 계열. + // ⓑ Mask Scoring R-CNN 이면 그 로짓으로 **점수를 다시 매긴다**(SubD 가 있을 때만). // mmdet 은 `score * mask_iou[label]` 로 점수를 낮춘다(maskiou_head.predict_by_feat). - // 이걸 빼면 **박스는 맞는데 점수만 틀린다** — 실측 박스 0.07px / 점수 0.163. + // ⓑ 를 빼면 **박스는 맞는데 점수만 틀린다** — 실측 박스 0.07px / 점수 0.163. // 게다가 점수가 임계값을 넘나들어 개수까지 달라진다(mmdet 4건 vs C++ 6건). // + // ⚠️ **ⓐ 가 실패해도 박스 판정이 바뀌면 안 된다.** 마스크는 더 재는 축이지 문턱이 + // 아니다 — 여기서 죽으면 이미 맞은 박스까지 못 쓰게 된다. 그래서 SubC 가 없으면 + // (`gc` 가 비었으면) 통째로 건너뛴다. + // // ⚠️⚠️ **두 그래프를 검출 하나씩(배치 1) 돌린다. 묶어 넣으면 안 된다.** // `ggml_compute_forward_conv_transpose_2d` 는 **배치 축을 안 돈다** — src1 을 // 풀 때 i12(채널)·i11(행)만 돌고 i13(배치)이 없고, 출력도 `dst->data + i2*nb2` @@ -756,7 +803,8 @@ int main(int argc, char** argv) { // 계산되고 나머지 19행은 bias 만 남는다.** 크래시도 경고도 없다 — // 실측: 행0 rel_L1 6.0e-04, 행1..4 는 전부 1.01 이고 |x| 평균이 서로 똑같았다 // (0.0870 = bias 뿐). mask head 의 `upsample`(14→28 deconv)이 그 경로다. - if (J.num("has_mask_iou", 0.0f) != 0.0f && gc && gd && !dets.empty()) { + const bool want_miou = J.num("has_mask_iou", 0.0f) != 0.0f && gd && *gd; + if (J.num("has_mask", 0.0f) != 0.0f && gc && *gc && !dets.empty()) { const int MD = (int)dets.size(); // ⚠️ **박스를 원본 해상도로 되돌리지 않는다.** mmdet 은 마스크 분기가 있으면 // bbox 쪽 rescale 을 끈다(`base_roi_head.py`: @@ -779,15 +827,36 @@ int main(int argc, char** argv) { std::vector> mfeats(feats.begin(), feats.begin() + ML); std::vector> mhw(feat_hw.begin(), feat_hw.begin() + ML); const int MO = mp.output_size; - std::vector mfeat = roi_align(mfeats, mhw, mbox.data(), MD, mp); + // ⚠️ **마스크 추출기도 레벨을 고를 수도, 안 고를 수도 있다.** `GenericRoIExtractor` + // (`groie`)면 전 레벨에 RoIAlign 을 걸어 pre→합산→post 를 SubC 안에서 태운다 — + // bbox 쪽과 같은 수법이다(위 `groie_levels` 참고). 이걸 빼고 레벨 하나만 넣으면 + // **크래시 없이 로짓 크기가 통째로 작아진다**(실측 |x| 0.45 vs 2.00, rel L1 1.00). + const int mgroie_lv = (int)J.num("mask_groie_levels", 0.0f); + const int MLV = mgroie_lv > 0 ? mgroie_lv : 1; // SubC 한 번에 넣는 배치 크기 + std::vector mfeat; + if (mgroie_lv > 0) { + // 레벨별 (MD,C,MO,MO) 를 이어붙인다 — 러너가 행 하나씩 꺼내 쓴다. + for (int l = 0; l < mgroie_lv; ++l) { + roi_align_params lp = mp; + lp.force_level = l; + std::vector one = roi_align(mfeats, mhw, mbox.data(), MD, lp); + mfeat.insert(mfeat.end(), one.begin(), one.end()); + } + } else { + mfeat = roi_align(mfeats, mhw, mbox.data(), MD, mp); + } // 모델은 한 번만 올린다. 행마다 바뀌는 것은 그래프뿐이다. model_file fc = model_load(gc); model_weights wc = model_init(fc.n_tensors()); model_transfer(fc, wc, backend, backend.preferred_float_type(), fc.tensor_layout()); +#ifdef ARCH_D + // SubD 는 Mask Scoring R-CNN 에만 컴파일된다 — 하네스가 SubD 를 넘길 때만 + // `ARCH_D` 를 세우므로 여기 오면 파일이 있다. model_file fd = model_load(gd); model_weights wd = model_init(fd.n_tensors()); model_transfer(fd, wd, backend, backend.preferred_float_type(), fd.tensor_layout()); +#endif std::vector logit_all, din_all, miou(MD, 1.0f); int MH = 0, MW = 0, NCLS_M = 0; @@ -798,17 +867,23 @@ int main(int argc, char** argv) { { compute_graph g2 = compute_graph_init(65536); model_ref mc(wc, g2); - tensor min_ = compute_graph_input(mc, GGML_TYPE_F32, {C, MO, MO, 1}, "mroi"); + tensor min_ = compute_graph_input(mc, GGML_TYPE_F32, {C, MO, MO, MLV}, "mroi"); ggml_build_forward_expand(g2, min_); ggml_build_forward_expand(g2, FWD_C(mc, min_, PRM_C(fc))); compute_graph_allocate(g2, backend); // roi_align 은 NCHW flat 을 낸다. 생성 코드는 cwhn 규약이다. - std::vector cw((size_t)C * MO * MO); - for (int c = 0; c < C; ++c) - for (int y = 0; y < MO; ++y) - for (int x = 0; x < MO; ++x) - cw[((size_t)y * MO + x) * C + c] = - mfeat[(((size_t)n * C + c) * MO + y) * MO + x]; + // groie 면 레벨 L 개를 **배치로** 쌓아 넣는다 — 레벨 l 의 행 n 은 + // `mfeat` 안에서 `(l * MD + n)` 번째다(레벨별로 통째로 이어붙였다). + std::vector cw((size_t)MLV * C * MO * MO); + for (int l = 0; l < MLV; ++l) { + const size_t row = (size_t)l * MD + n; + float* dstl = cw.data() + (size_t)l * C * MO * MO; + for (int c = 0; c < C; ++c) + for (int y = 0; y < MO; ++y) + for (int x = 0; x < MO; ++x) + dstl[((size_t)y * MO + x) * C + c] = + mfeat[((row * C + c) * MO + y) * MO + x]; + } transfer_to_backend(min_, std::span(cw.data(), cw.size())); compute(g2, backend); std::vector> hw; @@ -821,6 +896,11 @@ int main(int argc, char** argv) { MH = hw[0].first; MW = hw[0].second; NCLS_M = (int)(mo0.size() / ((size_t)MH * MW)); } + logit_all.insert(logit_all.end(), mo0.begin(), mo0.end()); +#ifdef ARCH_D + if (!want_miou) continue; + // ⚠️ 아래 「28 → 14」 전제는 **SubD 를 먹이는 쪽만** 필요하다. 로짓만 낼 때는 + // 아무 크기나 상관없으므로 여기서 막으면 마스크 축을 통째로 못 잰다. const int PH = MH / 2, PW = MW / 2; // 28 → 14 if (PH != MO || PW != MO) { fprintf(stderr, "mask pool %dx%d != roi %dx%d — 이어붙이기 불가\n", @@ -866,20 +946,27 @@ int main(int argc, char** argv) { const int NIOU = (int)od[0].size(); miou[n] = od[0][std::min(std::max(mlab[n], 0), NIOU - 1)]; } - logit_all.insert(logit_all.end(), mo0.begin(), mo0.end()); din_all.insert(din_all.end(), din.begin(), din.end()); +#endif // ARCH_D } - for (int i = 0; i < MD; ++i) { - dets[i].score *= miou[i]; - } + if (want_miou) + for (int i = 0; i < MD; ++i) dets[i].score *= miou[i]; // ⚠️ **재정렬하지 않는다.** mmdet 도 보정 뒤 정렬하지 않는다 // (`predict_by_feat` 는 `results.scores` 를 제자리에서 곱할 뿐이다). // 여기서 정렬하면 양쪽 순서가 갈려 인덱스로 짝짓는 대조가 어긋난다. // 중간 텐서도 낸다 — 값이 틀렸을 때 **어느 단계**인지 torch 와 대조한다. dump_bin(pref + ".mfeat.bin", mfeat); // 마스크 RoIAlign (NCHW) - dump_bin(pref + ".mlogit.bin", logit_all); // SubC 출력 (행별 cwhn) - dump_bin(pref + ".miouin.bin", din_all); // SubD 입력 (행별 cwhn, 257ch) - dump_bin(pref + ".maskiou.bin", miou); // 라벨 채널의 IoU 예측 + dump_bin(pref + ".mlogit.bin", logit_all); // SubC 출력 (행별 cwhn: ((y*MW+x)*NCLS+k)) + // ⚠️ 축을 파일 이름으로 못 적으니 **따로 적는다.** 행수·클래스수·H·W 를 모르면 + // 파이썬 쪽이 크기로 짐작하게 되고, 짐작은 계열이 바뀌면 조용히 틀린다. + { + const std::vector dims = {(float)MD, (float)NCLS_M, (float)MH, (float)MW}; + dump_bin(pref + ".mlogit.dims.bin", dims); + } + if (want_miou) { + dump_bin(pref + ".miouin.bin", din_all); // SubD 입력 (행별 cwhn, 257ch) + dump_bin(pref + ".maskiou.bin", miou); // 라벨 채널의 IoU 예측 + } } #endif diff --git a/tools/verify/mmdet/backbone/run_mmdet.cpp b/tools/verify/mmdet/backbone/run_mmdet.cpp new file mode 100755 index 0000000..06393ff --- /dev/null +++ b/tools/verify/mmdet/backbone/run_mmdet.cpp @@ -0,0 +1,161 @@ +// run_mmdet.cpp — mmdet 검증 하네스. +// +// 배포 경로는 `vision-cli -m -i -o out.png` 하나다. 이 러너는 그 +// 경로에 없는 두 가지를 위해 남아 있다. +// · 전처리를 마친 `.bin` 텐서를 그대로 입력으로 받는다 — torch 와 같은 입력으로 맞춰 +// 비교하려면 이미지 디코딩 차이를 빼야 한다. +// · `MMDET_DUMP_HEAD` 로 디코드 전 중간 텐서를 덤프한다. +// 백본은 arch/ 에 설치하지 않고 g2c 산출 `.cpp` 를 그대로 컴파일한다(build_mmdet_cpp.sh). +// +// 조립·디코드는 라이브러리의 `mmdet_run` 이 한다 — vision-cli 와 같은 코드다. +// +// 컴파일: -DARCH=<클래스명> -DVISP_ARCH_HEADER='"/.h"' +// 실행: run_mmdet [size=512] + +#include VISP_ARCH_HEADER // 백본: _forward / _detect_params +#include MMDET_PARAMS_HEADER // 생성된 mmdet_params() + +#include "draw.h" +#include "visp/image.h" +#include "visp/ml.h" +#include "visp/mmdet_run.h" +#include "visp/postproc.h" + +#include +#include +#include +#include +#include +#include + +using namespace visp; + +#define CAT_(a, b) a##b +#define CAT(a, b) CAT_(a, b) +#define FWD CAT(ARCH, _forward) +#define DETECT_PARAMS CAT(ARCH, _detect_params) + + +static tensor backbone(model_ref m, tensor x, model_file const& f) { + return FWD(m, x, DETECT_PARAMS(f)); +} + +static std::vector load_bin(const char* path, size_t n) { + std::vector v(n); + FILE* f = fopen(path, "rb"); + if (!f) { fprintf(stderr, "cannot open %s\n", path); exit(1); } + size_t got = fread(v.data(), sizeof(float), n, f); + fclose(f); + if (got != n) { fprintf(stderr, "short read %s: %zu/%zu\n", path, got, n); exit(1); } + return v; +} + +static bool has_ext(std::string const& s, const char* e) { + size_t n = std::strlen(e); + return s.size() >= n && s.compare(s.size() - n, n, e) == 0; +} + +static bool is_image_path(std::string const& s) { + return has_ext(s, ".jpg") || has_ext(s, ".jpeg") || has_ext(s, ".png") || has_ext(s, ".bmp"); +} + +// 이미지는 생성된 상수로 preprocess 를 거치고, `.bin` 은 이미 전처리된 CWHN f32 로 본다. +static std::vector load_input(const char* path, int SZ, mmdet_cfg const& c, + image_data* source) { + std::string s(path); + if (!is_image_path(s)) { + return load_bin(path, (size_t)3 * SZ * SZ); + } + image_data img = image_load(path); + int iw = img.extent[0], ih = img.extent[1]; + int ic = n_channels(img.format); // stbi_load(...,0)=네이티브 채널수 (JPEG=3, PNG+α=4) + printf("- preprocess: image %dx%dx%d → %dx%d (mean %.1f,%.1f,%.1f std %.1f,%.1f,%.1f to_rgb=%d)\n", + iw, ih, ic, SZ, SZ, c.img_mean[0], c.img_mean[1], c.img_mean[2], + c.img_std[0], c.img_std[1], c.img_std[2], (int)c.to_rgb); + auto data = preprocess(img.data.get(), ih, iw, ic, SZ, c.img_mean, c.img_std, c.to_rgb); + if (source) *source = std::move(img); + return data; +} + +int main(int argc, char** argv) { + if (argc < 4) { + fprintf(stderr, + "usage: %s [size=512]\n" + " output ending in .bin holds raw float32 detections;\n" + " any other extension is an image with the boxes drawn on it\n", argv[0]); + return 1; + } + const char* gguf = argv[1]; + const char* inp = argv[2]; + const char* outp = argv[3]; + const int SZ = argc > 4 ? atoi(argv[4]) : 512; + + backend_device backend = backend_init(); + model_file file = model_load(gguf); + model_weights weights = model_init(file.n_tensors()); + model_transfer(file, weights, backend, backend.preferred_float_type(), file.tensor_layout()); + + mmdet_cfg cfg = mmdet_params(); + image_data source; + std::vector in_data = load_input(inp, SZ, cfg, &source); + if (const char* dp = std::getenv("MMDET_DUMP_PRE")) { + if (FILE* f = fopen(dp, "wb")) { + fwrite(in_data.data(), sizeof(float), in_data.size(), f); + fclose(f); + } + } + + std::vector dets; + try { + dets = mmdet_run(backend, weights, file, &backbone, cfg, + std::span(in_data.data(), in_data.size()), SZ, SZ); + } catch (std::exception const& e) { + fprintf(stderr, "%s\n", e.what()); + return 4; + } + + std::string out_s(outp); + bool want_raw = has_ext(out_s, ".bin"); + if (!want_raw && source.extent[0] == 0) { + fprintf(stderr, "- input was a tensor, so there is no image to draw on; writing raw\n"); + want_raw = true; + } + + if (want_raw) { + FILE* f = fopen(outp, "wb"); + if (!f) { fprintf(stderr, "cannot write %s\n", outp); return 1; } + for (detection const& d : dets) { + float rec[6] = { d.x1, d.y1, d.x2, d.y2, d.score, (float)d.label }; + fwrite(rec, sizeof(float), 6, f); + } + fclose(f); + printf("- detect: %zu boxes → %s (x1,y1,x2,y2,score,label f32*6)\n", dets.size(), outp); + } else { + float thr = 0.3f; + if (const char* e = std::getenv("VISP_DRAW_THRESHOLD")) thr = (float)atof(e); + // 좌표는 정사각 입력 기준이라 원본 해상도로 되돌린다. + float sx = float(source.extent[0]) / float(SZ); + float sy = float(source.extent[1]) / float(SZ); + int drawn = draw_detections(source, dets, sx, sy, thr); + image_save(source, outp); + printf("- detect: %zu boxes, %d drawn at score >= %.2f → %s\n", + dets.size(), drawn, thr, outp); + } + + // 출력 파일을 열지 않아도 한 번의 실행이 무엇을 냈는지 보이게 상위 몇 개를 찍는다. + // VISP_PRINT_DETS 가 개수를 정하고, 0 이면 끈다. + int n_print = 10; + if (const char* e = std::getenv("VISP_PRINT_DETS")) n_print = atoi(e); + n_print = std::min(n_print, (int)dets.size()); + if (n_print > 0) { + printf("\n %5s %8s %8s %8s %8s %8s %6s\n", "#", "x1", "y1", "x2", "y2", "score", "label"); + for (int i = 0; i < n_print; ++i) { + detection const& d = dets[i]; + printf(" %5d %8.1f %8.1f %8.1f %8.1f %8.3f %6d\n", + i, d.x1, d.y1, d.x2, d.y2, d.score, d.label); + } + if ((int)dets.size() > n_print) printf(" %5s (%zu more)\n", "...", dets.size() - n_print); + printf("\n"); + } + return 0; +} diff --git a/tools/verify/mmdet/dense_head/box_all.py b/tools/verify/mmdet/dense_head/box_all.py new file mode 100644 index 0000000..49005f2 --- /dev/null +++ b/tools/verify/mmdet/dense_head/box_all.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""box_all.py — one-stage **박스 축**을 계열별로 전수로 돌린다. + + python box_all.py # 빌드된 계열 전부 + python box_all.py yolact rpn # 골라서 + python box_all.py --gen-root ~/work/box-full + python box_all.py --results /tmp/box-onestage.results.json + +`verify_postproc.py` 한 계열을 부르는 얇은 드라이버다. **판정 규약도 짝 결정도 여기서 +새로 만들지 않는다** — 전부 하네스 것을 그대로 쓴다. + +왜 있나 +------ +이 드라이버가 **없어서 one-stage 전수를 한 번 통째로 재현 못 했다.** 2026-08-25 에 +같은 이름의 스크립트로 44계열을 돌린 기록이 위키에 있는데, 그게 세션 로컬(`/tmp`)에만 +있었고 같이 사라졌다. 산출물(`<계열>/out/run_mmdet`)은 멀쩡히 남아 있었는데도 +**부를 방법이 없었다.** + +→ 위키 규칙 그대로다: **파일에 안 적힌 것은 다음 세션에 존재하지 않는다.** + 그래서 `/tmp` 가 아니라 저장소 안에 둔다. + +⚠️ **짝은 `used.json` 에서 받는다 — 손으로 고르지 마라.** + `verify_heads.py` 가 구울 때 실제로 쓴 config·체크포인트를 그 파일에 남긴다. + 래퍼 계열(KD·트래커)은 언랩한 config 를 `cfg.py` 로 덤프해 굽기 때문에, mmdet 원본 + config 를 주면 **「짝이 다르다」 경고가 항상 뜨고** 기준값도 다른 모델에서 나온다. + `ld`·`strongsort` 가 그래서 「값 못 냄」으로 잘못 적혔다(둘 다 멀쩡했다). + → wiki `pitfall/짝을-손으로-고르면-남의-가중치를-잰다.md` + +⚠️ **빌드는 여기서 안 한다.** `<계열>/out/run_mmdet` 이 없으면 `BUILD_NONE` 으로 적고 + 넘어간다 — 굽는 것은 `verify_heads.py` 몫이고, 그쪽이 export→g2c→가중치→빌드까지 + 한다. 두 도구가 각자 구우면 **어느 쪽 산출물을 쟀는지** 알 수 없게 된다. + +⚠️ **못 잰 것을 실패로 적지 마라.** `BUILD_NONE`·`PAIR_NONE`·`EMPTY` 는 실패가 아니라 + 측정 실패다. 표에서 따로 세고, PASS 수와 나란히 놓지 않는다. +""" +import argparse +import concurrent.futures as _fut +import json +import os +import re +import subprocess +import sys + +sys.stdout.reconfigure(line_buffering=True) # 파이프로 보내도 진행이 보이게 + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import mmdet_families as MF # noqa: E402 +import vconfig # noqa: E402 + +CFG, ARGS = vconfig.load() +V = os.path.normpath(os.path.join(HERE, "..", "..", "..")) # vision.cpp/tools +VISP = os.path.normpath(os.path.join(V, "..")) # vision.cpp +POSTPROC = os.path.join(HERE, "verify_postproc.py") +DEFAULT_IMAGE = os.path.join(VISP, "tests", "input", "cat-and-hat.jpg") + +# 판정 요약 줄. `verify_postproc.py` 가 찍는 형식과 **한 군데서만** 묶여 있다. +RE_WORST = re.compile( + r"최대: 박스 ([\d.]+)px · 점수 ([\d.]+) · 라벨 불일치 (\d+)건 · 개수차 (\d+)건") +RE_DIST = re.compile(r"분포: 중앙값 ([\d.]+)px · 95%tile ([\d.]+)px · 1px 이내 (\d+)/(\d+)건") +RE_BAND = re.compile(r"임계값 [\d.]+~[\d.]+ 구간: mmdet (\d+)건 · C\+\+ (\d+)건") +RE_COUNT = re.compile(r"mmdet (\d+)건 · run_mmdet (\d+)건") + + +def _rebase(path, fam, gen_root): + """`used.json` 에 박힌 절대경로를 지금 `gen_root` 로 되돌린다. + + ⚠️ **경로는 트리를 옮기면 죽는다.** `used.json` 은 구울 때의 절대경로를 적는데, + 2026-08-28 홈 재편으로 `~/work/box-full` 이 `~/company/artifacts/box-full` 로 + 옮겨가면서 래퍼 계열의 `cfg.py` 를 통째로 못 찾게 됐다. 그런데 `pair_for` 는 + **조용히 원본 mmdet config 로 떨어진다** — 트래커는 `model.backbone` 이 없어서 + `'ConfigDict' object has no attribute 'backbone'`, `ld` 는 선생 config 를 상대경로로 + 가리켜 `FileNotFoundError` 가 난다. 넷 다 「실패」로 보이지만 **짝을 잘못 준 것**이다. + + 산출물 디렉터리 안의 파일(`cfg.py`·`ckpt.pth`)이면 파일명만 살려 지금 자리에서 찾는다. + """ + if not path: + return path + if os.path.exists(path): + return path + here = os.path.join(gen_root, fam, os.path.basename(path)) + return here if os.path.exists(here) else path + + +def pair_for(fam, gen_root): + """(config, checkpoint) — **구울 때 쓴 짝**이 정본이다. `used.json` 하나만 본다. + + ⛔ **원본 mmdet config 로 떨어지지 않는다.** 예전엔 `used.json` 을 못 읽으면 조용히 + `resolve_pair` 로 내려갔는데, 그러면 **남의 짝으로 재고도 실패처럼 보인다.** + 실측 2026-09-01: 홈 재편으로 `used.json` 의 절대경로가 죽자 래퍼 4계열이 + 원본 config 로 떨어져 `'ConfigDict' object has no attribute 'backbone'` · + `FileNotFoundError` 를 냈다. 넷 다 **모델은 멀쩡했다** — 짝이 틀렸을 뿐이다. + + 못 잰 것은 못 잤다고 적는다. 실패 칸에 넣지 않는다. + """ + u = os.path.join(gen_root, fam, "used.json") + if not os.path.exists(u): + print(f" ⚠️ {fam}: used.json 이 없다 — verify_heads.py 로 먼저 구워라") + return None, None + try: + d = json.load(open(u, encoding="utf-8")) + except Exception as e: + print(f" ⚠️ {fam}: used.json 을 못 읽는다 — {type(e).__name__}: {e}") + return None, None + c = _rebase(d.get("config"), fam, gen_root) + k = _rebase(d.get("checkpoint"), fam, gen_root) + if c and k and os.path.exists(c) and os.path.exists(k): + return c, k + for name, v in (("config", c), ("checkpoint", k)): + if not v or not os.path.exists(v): + print(f" ⚠️ {fam}: used.json 의 {name} 이 없다 — {v}") + return None, None + + +def one(fam, gen_root, size, image, verbose): + gen = os.path.join(gen_root, fam, "out") + if not os.path.exists(os.path.join(gen, "run_mmdet")): + return dict(family=fam, status="BUILD_NONE", + note=f"{gen}/run_mmdet 없음 — verify_heads.py 로 먼저 굽는다") + cfg, ckpt = pair_for(fam, gen_root) + if not cfg or not ckpt: + return dict(family=fam, status="PAIR_NONE", note="config·체크포인트를 못 찾았다") + + img = MF.test_image(fam, image) + r = subprocess.run([sys.executable, POSTPROC, gen, cfg, ckpt, img, str(size)], + capture_output=True, text=True, + env={**os.environ, "OMP_NUM_THREADS": "1"}) + out = (r.stdout or "") + (r.stderr or "") + m = RE_WORST.search(out) + if not m: + # 판정 줄이 없다 = 비교까지 못 갔다. **왜** 인지를 그대로 옮긴다. + why = "한쪽이 비어 비교 불가" if "한쪽이 비어" in out else \ + next((l.strip() for l in reversed(out.splitlines()) + if l.strip() and not l.startswith(" ")), "출력 없음") + return dict(family=fam, status="EMPTY" if "한쪽이 비어" in out else "RUN_FAIL", + note=why[:110], stdout=out if verbose else None) + + box, score, bad_label, gap = float(m[1]), float(m[2]), int(m[3]), int(m[4]) + ok = box < 2.0 and score < 0.05 and bad_label == 0 and gap == 0 + res = dict(family=fam, status="PASS" if ok else "FAIL", + box=box, score=score, bad_label=bad_label, gap=gap, + image=os.path.basename(img), + note=f"박스 {box:.2f}px · 점수 {score:.3f} · 라벨 {bad_label} · 개수차 {gap}") + if (c := RE_COUNT.search(out)): + res["n_ref"], res["n_got"] = int(c[1]), int(c[2]) + # ⚠️ **박스가 수백 개인 계열은 최대값만 보면 오해한다**(rpn: 최대 133px 인데 중앙값 + # 0.09px · 182/185건이 1px 이내). 분포가 있으면 반드시 같이 싣는다. + if (d := RE_DIST.search(out)): + res["median"], res["p95"] = float(d[1]), float(d[2]) + res["within_1px"], res["n_rows"] = int(d[3]), int(d[4]) + res["note"] += f" · 중앙값 {float(d[1]):.2f}px · 1px 이내 {d[3]}/{d[4]}" + # 개수차가 컷 근처에서 났는지 — 「임계 경계」와 「진짜 결함」을 가르는 첫 단서다. + if (b := RE_BAND.search(out)): + res["band_ref"], res["band_got"] = int(b[1]), int(b[2]) + if "짝이 다르다" in out: + # 래퍼 계열에서 **거짓으로도** 뜬다(basename 비교라 `cfg.py` 와 늘 어긋난다). + # 판정을 바꾸지 않고 표시만 한다 — 수치가 크게 틀릴 때만 의심하면 된다. + res["pair_warn"] = True + res["note"] += " · ⚠️짝 경고" + if verbose: + res["stdout"] = out + return res + + +def main(): + ap = argparse.ArgumentParser(add_help=True) + ap.add_argument("families", nargs="*", help="비우면 빌드된 계열 전부") + ap.add_argument("--gen-root", default=CFG.workdir, + help=f"계열별 산출물 루트 (<루트>/<계열>/out). 기본 {CFG.workdir}") + ap.add_argument("--size", type=int, default=CFG.size) + ap.add_argument("--image", default=DEFAULT_IMAGE, + help="계열별 권장 이미지가 있으면 그쪽이 우선한다") + ap.add_argument("--results", default=None, help="results.json 을 쓸 경로") + ap.add_argument("--workers", type=int, default=CFG.workers) + ap.add_argument("-v", "--verbose", action="store_true", help="계열별 원본 출력을 남긴다") + a = ap.parse_args(ARGS) + + root = os.path.expanduser(a.gen_root) + fams = a.families or sorted( + d for d in os.listdir(root) + if os.path.exists(os.path.join(root, d, "out", "run_mmdet"))) + if not fams: + print(f"돌릴 계열이 없다 — {root}/<계열>/out/run_mmdet 이 하나도 없다.\n" + f" verify_heads.py 로 먼저 굽는다 (--set paths.workdir={root}).") + return 2 + + print(CFG.banner() if hasattr(CFG, "banner") else "") + print(f"gen-root={root} · size={a.size} · 판정: 박스<2.0px 점수<0.05 라벨0 개수차0") + print(f"{len(fams)}계열: {' '.join(fams)}\n") + + rows = [] + with _fut.ThreadPoolExecutor(max_workers=max(1, a.workers)) as ex: + futs = {ex.submit(one, f, root, a.size, a.image, a.verbose): f for f in fams} + for i, fu in enumerate(_fut.as_completed(futs), 1): + r = fu.result() + rows.append(r) + print(f"[{i:3d}/{len(fams)}] {r['family']:<22} {r['status']:<12} {r.get('note','')}") + + rows.sort(key=lambda r: r["family"]) + print("\n" + "=" * 78) + for r in rows: + print(f" {r['family']:<22} {r['status']:<12} {r.get('note','')}") + + # ⚠️ **PASS 수만 적지 마라.** 못 잰 것과 실패한 것을 한 칸에 넣으면 다음 사람이 + # "이 계열이 못 한다" 로 읽는다. 성격별로 따로 센다. + n = {} + for r in rows: + n[r["status"]] = n.get(r["status"], 0) + 1 + measured = sum(v for k, v in n.items() if k in ("PASS", "FAIL")) + print(f"\nPASS {n.get('PASS', 0)}/{measured} (잰 것 기준)") + unmeasured = {k: v for k, v in n.items() if k not in ("PASS", "FAIL")} + if unmeasured: + print(" 못 잼: " + " · ".join(f"{k} {v}" for k, v in sorted(unmeasured.items())) + + " ← 실패가 아니다. PASS 수와 나란히 놓지 마라") + + if a.results: + with open(a.results, "w", encoding="utf-8") as f: + json.dump(rows, f, ensure_ascii=False, indent=1) + print(f" → {a.results}") + return 0 if n.get("FAIL", 0) == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/verify/dense_head/fetch_checkpoints.py b/tools/verify/mmdet/dense_head/fetch_checkpoints.py similarity index 100% rename from tools/verify/dense_head/fetch_checkpoints.py rename to tools/verify/mmdet/dense_head/fetch_checkpoints.py diff --git a/tools/verify/mmdet/dense_head/glip_decode.py b/tools/verify/mmdet/dense_head/glip_decode.py new file mode 100644 index 0000000..67cd0cb --- /dev/null +++ b/tools/verify/mmdet/dense_head/glip_decode.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""GLIP 의 15텐서 → 박스·점수·라벨. **호스트(numpy) 참조 구현.** + +컴파일된 그래프가 내는 것은 `cls_logits · bbox_pred · centerness` 15개뿐이고, 거기서 +박스를 만드는 규칙은 `mmdet/models/dense_heads/atss_vlfusion_head.py` 의 +`_predict_by_feat_single` 이다. 그걸 **여기서 먼저 정확히 재현해** C++ 로 옮길 때 +비교 기준으로 쓴다 — C++ 를 바로 쓰면 「값이 틀렸는데 디코드 탓인지 그래프 탓인지」를 +못 가른다. + +일반 검출기와 두 곳이 다르다. + +1. **점수가 클래스가 아니라 토큰이다.** `cls_logits` 의 마지막 축 256 은 **텍스트 토큰 + 슬롯**이라 argmax 가 뜻이 없다. `positive_map`(문구 → 토큰 인덱스들)으로 **평균**내 + 문구 점수를 만든다. 그 map 은 토크나이저가 만들므로 **호스트가 넘긴다.** +2. **박스 코더가 `DeltaXYWHBBoxCoderForGLIP`** 이다. 표준 Delta 디코드에서 `-1`/`+1` 이 + 네 군데 붙는다(아래 `delta2bbox_glip` 주석). + +⚠️ **점수는 곱이 아니라 `sqrt(곱)`** 이다. 곱으로 두면 점수가 통째로 낮아져 score_thr 에서 + 개수가 줄고, 「박스는 맞는데 개수차」로 보인다. +""" +import numpy as np + + +def convert_grounding_to_cls_scores(logits, positive_map, num_phrases): + """토큰 로짓 → 문구 점수. `atss_vlfusion_head.py:32` 와 같은 식. + + logits: (N, n_tokens) — **이미 sigmoid 를 거친 값** + positive_map: {문구번호 j(1부터): [토큰 인덱스…]} + 반환: (N, num_phrases) — 열 j-1 이 문구 j 의 점수 + """ + out = np.zeros((logits.shape[0], num_phrases), dtype=np.float32) + for j, toks in positive_map.items(): + if not toks: + continue + out[:, int(j) - 1] = logits[:, np.asarray(toks, dtype=np.int64)].mean(-1) + return out + + +def anchors(feat_hw, stride, octave_base_scale=8.0, center_offset=0.5): + """`AnchorGenerator(ratios=[1.0], scales_per_octave=1)` 의 격자 앵커 (N, 4). + + ratio 1·octave 1 이라 앵커는 정사각 하나뿐이다 — 변 = stride × octave_base_scale. + `center_offset=0.5` 라 격자 중심이 칸 가운데다. + """ + h, w = feat_hw + size = stride * octave_base_scale + cx = (np.arange(w, dtype=np.float32) + center_offset) * stride + cy = (np.arange(h, dtype=np.float32) + center_offset) * stride + cx, cy = np.meshgrid(cx, cy) # (h, w) — 행 우선 + cx, cy = cx.reshape(-1), cy.reshape(-1) + half = size / 2.0 + return np.stack([cx - half, cy - half, cx + half, cy + half], axis=1) + + +def delta2bbox_glip(rois, deltas, means=(0., 0., 0., 0.), stds=(0.1, 0.1, 0.2, 0.2), + max_shape=None, wh_ratio_clip=16 / 1000): + """`DeltaXYWHBBoxCoderForGLIP.decode`. 표준 Delta 와 **네 군데** 다르다. + + | | 표준 | GLIP | + |---|---|---| + | 앵커 중심 | `(x1+x2)*0.5` | `(x1+x2-1)*0.5` | + | 박스 복원 | `gxy ∓ gwh*0.5` | `gxy ∓ (gwh-1)*0.5` | + | 클립 | `max_shape` | `max_shape - 1` | + | 마지막 | — | `x2,y2 += 1` (**후처리 뒤**, 여기가 아니다) | + + mmdet 주석에도 "very strange bbox decoder logic" 이라 적혀 있다 — 공식 구현의 mAP 에 + 맞추려고 남긴 것이라 **한 칸이라도 빼면 박스가 1px 씩 밀린다.** + """ + d = deltas.reshape(-1, 4) * np.asarray(stds, np.float32) + np.asarray(means, np.float32) + dxy, dwh = d[:, :2], d[:, 2:] + pxy = (rois[:, :2] + rois[:, 2:] - 1) * 0.5 # ⚠️ -1 + pwh = rois[:, 2:] - rois[:, :2] + + max_ratio = abs(np.log(wh_ratio_clip)) + dwh = np.clip(dwh, -max_ratio, max_ratio) + gxy = pxy + pwh * dxy + gwh = pwh * np.exp(dwh) + + x1y1 = gxy - (gwh - 1) * 0.5 # ⚠️ -1 + x2y2 = gxy + (gwh - 1) * 0.5 # ⚠️ -1 + boxes = np.concatenate([x1y1, x2y2], axis=-1) + if max_shape is not None: + h, w = max_shape + boxes[:, 0::2] = np.clip(boxes[:, 0::2], 0, w - 1) # ⚠️ -1 + boxes[:, 1::2] = np.clip(boxes[:, 1::2], 0, h - 1) + return boxes + + +def _nms(boxes, scores, iou_thr): + """클래스 무관 NMS. 라벨별로 부르는 쪽에서 갈라 준다.""" + order = scores.argsort()[::-1] + x1, y1, x2, y2 = boxes[:, 0], boxes[:, 1], boxes[:, 2], boxes[:, 3] + area = (x2 - x1).clip(0) * (y2 - y1).clip(0) + keep = [] + while order.size: + i = order[0] + keep.append(i) + if order.size == 1: + break + rest = order[1:] + xx1 = np.maximum(x1[i], x1[rest]) + yy1 = np.maximum(y1[i], y1[rest]) + xx2 = np.minimum(x2[i], x2[rest]) + yy2 = np.minimum(y2[i], y2[rest]) + inter = (xx2 - xx1).clip(0) * (yy2 - yy1).clip(0) + iou = inter / np.maximum(area[i] + area[rest] - inter, 1e-9) + order = rest[iou <= iou_thr] + return np.asarray(keep, dtype=np.int64) + + +def decode(cls_logits, bbox_preds, centernesses, positive_map, num_phrases, + img_shape, strides=(8, 16, 32, 64, 128), + score_thr=0.05, nms_pre=1000, nms_thr=0.6, max_per_img=100): + """15텐서 → (boxes (M,4), scores (M,), labels (M,)). + + cls_logits[i] : (HW, n_tokens) — 레벨 i + bbox_preds[i] : (4, H, W) + centernesses[i]: (1, H, W) + """ + all_boxes, all_scores, all_labels = [], [], [] + for lvl, stride in enumerate(strides): + bp = bbox_preds[lvl] + _, h, w = bp.shape + # (4,H,W) → (HW,4). torch 의 `permute(1,2,0).reshape(-1,4)` 와 같은 순서다. + deltas = bp.transpose(1, 2, 0).reshape(-1, 4) + ctr = 1.0 / (1.0 + np.exp(-centernesses[lvl].reshape(-1))) # sigmoid + logit = cls_logits[lvl] + scores = convert_grounding_to_cls_scores( + 1.0 / (1.0 + np.exp(-logit)), positive_map, num_phrases) # (HW, P) + + flat = scores.reshape(-1) + idx = np.nonzero(flat > score_thr)[0] + if idx.size == 0: + continue + if idx.size > nms_pre: # 상위 nms_pre 개만 + idx = idx[np.argsort(flat[idx])[::-1][:nms_pre]] + keep_loc, labels = idx // num_phrases, idx % num_phrases + # ⚠️ 곱이 아니라 **sqrt(곱)** 이다. + sc = np.sqrt(flat[idx] * ctr[keep_loc]) + pri = anchors((h, w), float(stride))[keep_loc] + all_boxes.append(delta2bbox_glip(pri, deltas[keep_loc], max_shape=img_shape)) + all_scores.append(sc) + all_labels.append(labels) + + if not all_boxes: + return (np.zeros((0, 4), np.float32), np.zeros((0,), np.float32), + np.zeros((0,), np.int64)) + boxes = np.concatenate(all_boxes).astype(np.float32) + scores = np.concatenate(all_scores).astype(np.float32) + labels = np.concatenate(all_labels).astype(np.int64) + + # 라벨별 NMS (mmdet `batched_nms` 와 같은 뜻). + keep = [] + for lab in np.unique(labels): + sel = np.nonzero(labels == lab)[0] + keep.append(sel[_nms(boxes[sel], scores[sel], nms_thr)]) + keep = np.concatenate(keep) if keep else np.zeros((0,), np.int64) + keep = keep[np.argsort(scores[keep])[::-1][:max_per_img]] + + boxes, scores, labels = boxes[keep], scores[keep], labels[keep] + # ⚠️ **후처리 뒤에 +1.** mmdet 이 "1 을 안 더하면 공식 mAP 와 안 맞는다" 고 적어 둔 곳이다. + boxes[:, 2:] = boxes[:, 2:] + 1 + return boxes, scores, labels diff --git a/tools/verify/dense_head/head_support_map.py b/tools/verify/mmdet/dense_head/head_support_map.py similarity index 99% rename from tools/verify/dense_head/head_support_map.py rename to tools/verify/mmdet/dense_head/head_support_map.py index a2c9976..8f4cfb4 100644 --- a/tools/verify/dense_head/head_support_map.py +++ b/tools/verify/mmdet/dense_head/head_support_map.py @@ -32,7 +32,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import vconfig # noqa: E402 -V = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +V = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) FE = V + "/tools/frontend/mmdet" CFG, ARGS = vconfig.load() MM, PY = CFG.configs, sys.executable diff --git a/tools/verify/dense_head/mmdet_families.py b/tools/verify/mmdet/dense_head/mmdet_families.py similarity index 90% rename from tools/verify/dense_head/mmdet_families.py rename to tools/verify/mmdet/dense_head/mmdet_families.py index 2999dd4..a517514 100644 --- a/tools/verify/dense_head/mmdet_families.py +++ b/tools/verify/mmdet/dense_head/mmdet_families.py @@ -34,8 +34,31 @@ "masktrack_rcnn")} # 이미지는 **저장소 안**에 있다. 사용자가 준 `--image` 의 디렉토리에서 찾으면 # 엉뚱한 곳(`~/pics/bench-image.jpg`)을 가리키고, 상대경로면 하위 프로세스의 cwd 로 풀린다. +# ⚠️ `..` 은 **넷**이다. dense_head → mmdet → verify → tools → vision.cpp. +# 셋이면 `tools/tests/input` 이라 없는 경로다. 그러면 `test_image` 가 default 로 +# 조용히 떨어져 **위 매핑이 통째로 무효가 된다** — 트래커를 사람 없는 이미지로 재고 +# 양쪽 0건이 나와 `EMPTY` 로 찍힌다(2026-09-01 실측). _IMG_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), - "..", "..", "..", "tests", "input")) + "..", "..", "..", "..", "tests", "input")) + + +# 계열별 **입력 크기**. 지정이 없으면 호출자가 준 기본값을 쓴다. +# +# ⚠️ **이 표가 없으면 전수를 돌릴 때마다 같은 계열이 다시 실패한다.** `fpg`(Feature +# Pyramid Grids)는 P5 아래로 레벨을 더 쌓는데 800·512 가 거기서 안 나눠떨어져 +# `size of tensor a (25) must match b (26) at dim 3` 로 죽는다. 2026-08-18 에 +# 「1024 로 재측정」으로 해결됐고 0.28px 로 통과하는데, **그 해결이 코드에 안 들어가 +# 있어서** 09-01 전수에서 또 EXPORT_FAIL 이 났다. +# → wiki `pitfall/하네스가-못-잰-것을-대상이-못-하는-것으로-적지-마라` +# +# ⚠️ **여기 걸린 계열은 분모가 다르다.** 다른 크기에서 잰 값을 같은 전수 PASS 수에 +# 합치지 마라 — 측정 조건이 섞인다. 보고할 때 크기를 같이 적는다. +_SIZE = {"fpg": 1024} + + +def test_size(fam, default): + """이 계열로 잴 때 쓸 입력 한 변. 지정이 없으면 `default`.""" + return _SIZE.get(fam, default) def test_image(fam, default): diff --git a/tools/verify/dense_head/run_vfnet_head.cpp b/tools/verify/mmdet/dense_head/run_vfnet_head.cpp similarity index 100% rename from tools/verify/dense_head/run_vfnet_head.cpp rename to tools/verify/mmdet/dense_head/run_vfnet_head.cpp diff --git a/tools/verify/dense_head/vconfig.py b/tools/verify/mmdet/dense_head/vconfig.py similarity index 100% rename from tools/verify/dense_head/vconfig.py rename to tools/verify/mmdet/dense_head/vconfig.py diff --git a/tools/verify/dense_head/verify.toml b/tools/verify/mmdet/dense_head/verify.toml similarity index 85% rename from tools/verify/dense_head/verify.toml rename to tools/verify/mmdet/dense_head/verify.toml index 675cb9f..0dd4fa9 100644 --- a/tools/verify/dense_head/verify.toml +++ b/tools/verify/mmdet/dense_head/verify.toml @@ -10,14 +10,16 @@ # mmdet 저장소. 아래에 `configs/` 와 `checkpoints/` 가 있어야 한다. mmdet = "~/mmbuild/mmdetection" # g2c(컴파일러) 루트. 기본값은 vision.cpp 의 부모 = 이 브랜치의 g2c. -g2c = "../../../.." +g2c = "../../../../.." # 트래커·반지도 래퍼 config 를 안쪽 검출기로 푸는 전처리기. # 없으면 그 계열만 INIT_FAIL 로 남는다 — 없다고 전체가 멈추지는 않는다. # ⚠️ 예전에는 **형제 저장소**(`../../../../../GTX_Compiler/...`)를 가리켰는데, 이 트리에서는 # `/tmp/GTX_Compiler` 로 풀려 **조용히 꺼진 채** 돌았다(존재 검사만 하고 없으면 넘어간다). # 도구를 상위 저장소로 옮겼으니 그쪽을 가리킨다 — vision.cpp 는 서브모듈이라 -# `../../../../` 가 g2c 루트다(`g2c` 항목과 같은 기준). -unwrap = "../../../../test_script/mmdet/mmdet_unwrap_config.py" +# `../../../../../` 가 g2c 루트다(`g2c` 항목과 같은 기준). +# ⚠️ 2026-08-27 verify/ 를 생태계별로 재편하며 이 파일이 한 단계 깊어졌다 — +# `..` 하나를 안 늘렸더니 `g2c` 가 vision.cpp 를 가리켰다(존재는 하니 조용히 통과). +unwrap = "../../../../../test_script/mmdet/mmdet_unwrap_config.py" # 중간 산출물(.pt · 생성 .cpp · gguf · 덤프). 지워도 된다. workdir = "/tmp/visp-verify-heads" # head 지원 여부 조사(head_support_map.py) 전용 작업 폴더. diff --git a/tools/verify/mmdet/dense_head/verify_glip_box.py b/tools/verify/mmdet/dense_head/verify_glip_box.py new file mode 100644 index 0000000..8baeddf --- /dev/null +++ b/tools/verify/mmdet/dense_head/verify_glip_box.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""GLIP 의 **박스**를 검증한다 — 텐서가 아니라 최종 검출 결과로. + +두 단계로 나눠 잰다. **한 번에 재면 틀렸을 때 디코드 탓인지 그래프 탓인지 못 가른다.** + + ① 디코드만 : torch 15텐서 → 우리 디코드 vs torch 15텐서 → mmdet 디코드 + ② 전체 : C++ 15텐서 → 우리 디코드 vs torch 15텐서 → mmdet 디코드 + +①이 0 이면 디코드 규칙(`glip_decode.py`)이 mmdet 과 같다는 뜻이고, 그때 ②의 오차는 +전부 그래프(fp16·커널) 몫이다. + + python verify_glip_box.py [--caption "person. bicycle. car. dog."] [--size 512] + +⚠️ **positive_map 은 호스트가 만든다.** 토크나이저가 문구를 토큰 인덱스로 나눈 결과라 + C++ 토크나이저가 없어도 여기까지 온다 — 그건 CLI 를 만들 때 할 일이다. +""" +import argparse +import os +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +V = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..")) +P = os.path.abspath(os.path.join(V, "..")) +FE = V + "/tools/frontend/mmdet" +sys.path.insert(0, HERE) +sys.path.insert(0, FE) + +MM = os.path.expanduser("~/mmbuild/mmdetection") +CFG = MM + "/configs/glip/glip_atss_swin-t_a_fpn_dyhead_pretrain_obj365.py" +CK = MM + "/checkpoints/glip_tiny_a_mmdet-b3654169.pth" + +# 판정 기준. 박스는 픽셀, 점수는 절대차. 저장소 관례(박스 하네스)를 따른다. +BOX_TOL_PX = 2.0 +SCORE_TOL = 0.05 + + +def _match(a_boxes, a_scores, a_labels, b_boxes, b_scores, b_labels): + """두 검출 집합을 점수 내림차순으로 짝지어 최대 오차를 낸다. + + NMS 를 거친 뒤라 **순서가 곧 짝**이다(같은 입력·같은 임계값이면 같은 순서가 나온다). + 개수가 다르면 그 자체가 결함이므로 따로 보고한다. + """ + n = min(len(a_boxes), len(b_boxes)) + if n == 0: + return dict(count_a=len(a_boxes), count_b=len(b_boxes), + box_px=float("nan"), score=float("nan"), label_bad=0) + box_px = float(np.abs(a_boxes[:n] - b_boxes[:n]).max()) + score = float(np.abs(a_scores[:n] - b_scores[:n]).max()) + label_bad = int((a_labels[:n] != b_labels[:n]).sum()) + return dict(count_a=len(a_boxes), count_b=len(b_boxes), + box_px=box_px, score=score, label_bad=label_bad) + + +def main(): + ap = argparse.ArgumentParser(prog="verify_glip_box") + ap.add_argument("--caption", default="person. bicycle. car. dog.") + ap.add_argument("--size", type=int, default=512) + ap.add_argument("--workdir", default="/tmp/visp-glip-box") + ap.add_argument("--skip-cpp", action="store_true", help="①(디코드만) 만 잰다") + a = ap.parse_args() + + import torch + import mmdet_wrap + import glip_decode + + mmdet_wrap.trace_friendly_ops() + mmdet_wrap.allow_mmengine_checkpoint_globals() + from mmdet.apis import init_detector + det = init_detector(CFG, CK, device="cpu") + det.eval() + + tok = det.language_model.tokenizer + if not hasattr(tok, "batch_encode_plus"): + tok.batch_encode_plus = tok.__call__ # transformers 5.x 에서 삭제된 4.x API + + # 캡션 → 토큰 + positive_map. **mmdet 자신의 함수로 만든다** — 규칙을 다시 짜면 틀린다. + tokenized, caption_string, tokens_positive, entities = \ + det.get_tokens_and_prompts(a.caption, True) + # ⚠️ `get_positive_map` 은 **튜플**을 돌려준다 — + # `(label→token dict, positive_map 텐서)`. 앞엣것이 이미 우리가 쓸 dict 다 + # (`plus=1` 이 안에서 걸려 라벨이 1부터다). 다시 변환하면 안 된다. + pos_map, _positive_map = det.get_positive_map(tokenized, tokens_positive) + n_phrase = len(entities) + print(f"[caption] {caption_string!r}") + print(f"[phrases] {entities} → positive_map {len(pos_map)}문구") + + with torch.no_grad(): + text_dict = det.language_model([caption_string]) + emb = text_dict["embedded"].contiguous() + + SZ = a.size + g = torch.Generator().manual_seed(0) + x = torch.randn(1, 3, SZ, SZ, generator=g) + with torch.no_grad(): + feats = det.extract_feat(x) + cls_logits, bbox_preds, centerness = det.bbox_head(feats, {"embedded": emb}) + + # ── ① 디코드만 ──────────────────────────────────────────────────────────── + ours = glip_decode.decode( + [t[0].numpy() for t in cls_logits], + [t[0].numpy() for t in bbox_preds], + [t[0].numpy() for t in centerness], + {int(k): list(v) for k, v in pos_map.items()}, n_phrase, (SZ, SZ)) + + from mmengine.structures import InstanceData # noqa: F401 (mmdet 내부에서 쓴다) + ref = det.bbox_head.predict_by_feat( + cls_logits, bbox_preds, centerness, + batch_img_metas=[{"img_shape": (SZ, SZ), "scale_factor": (1.0, 1.0), + "ori_shape": (SZ, SZ)}], + batch_token_positive_maps=[pos_map], rescale=False)[0] + ref_b = ref.bboxes.numpy() + ref_s = ref.scores.numpy() + ref_l = ref.labels.numpy() + + m1 = _match(ours[0], ours[1], ours[2], ref_b, ref_s, ref_l) + print(f"\n① 디코드만 개수 {m1['count_a']} vs {m1['count_b']} · " + f"박스 {m1['box_px']:.4f}px · 점수 {m1['score']:.5f} · 라벨불일치 {m1['label_bad']}") + + if a.skip_cpp: + return + + # ── ② 전체 (C++ 텐서 + 우리 디코드) ─────────────────────────────────────── + d = a.workdir + gen = os.path.join(d, "out") + run_dump = os.path.join(gen, "run_dump") + gguf = os.path.join(gen, "Glip.gguf") + if not (os.path.exists(run_dump) and os.path.exists(gguf)): + print(f"\n② 건너뜀 — 컴파일 산출물이 없다: {gen}\n" + f" (`verify_heads.py glip` 로 먼저 굽거나 --skip-cpp 를 준다)") + return + np.ascontiguousarray(x[0].numpy().transpose(1, 2, 0)).tofile(os.path.join(d, "in.bin")) + subprocess.run([run_dump, gguf, os.path.join(d, "in.bin"), os.path.join(d, "cpp"), + str(SZ)], cwd=d, capture_output=True, text=True) + + # 러너 덤프 순서 = 그래프 출력 순서 = cls×5, bbox×5, centerness×5 + cpp_cls, cpp_box, cpp_ctr = [], [], [] + for i in range(5): + s = cls_logits[i][0].shape # (HW, n_tok) + cpp_cls.append(np.fromfile(os.path.join(d, f"cpp.out.{i}.bin"), + dtype="float32").reshape(s)) + for i in range(5): + c, h, w = bbox_preds[i][0].shape + # 러너는 cwhn(HWC)로 쓴다 — torch CHW 로 되돌린다. + cpp_box.append(np.fromfile(os.path.join(d, f"cpp.out.{5 + i}.bin"), + dtype="float32").reshape(h, w, c).transpose(2, 0, 1)) + for i in range(5): + c, h, w = centerness[i][0].shape + cpp_ctr.append(np.fromfile(os.path.join(d, f"cpp.out.{10 + i}.bin"), + dtype="float32").reshape(h, w, c).transpose(2, 0, 1)) + + full = glip_decode.decode(cpp_cls, cpp_box, cpp_ctr, + {int(k): list(v) for k, v in pos_map.items()}, + n_phrase, (SZ, SZ)) + m2 = _match(full[0], full[1], full[2], ref_b, ref_s, ref_l) + print(f"② 전체 개수 {m2['count_a']} vs {m2['count_b']} · " + f"박스 {m2['box_px']:.4f}px · 점수 {m2['score']:.5f} · 라벨불일치 {m2['label_bad']}") + + ok = (m2["count_a"] == m2["count_b"] and m2["label_bad"] == 0 + and m2["box_px"] < BOX_TOL_PX and m2["score"] < SCORE_TOL) + print(f"\n{'PASS' if ok else 'FAIL'} (박스 {BOX_TOL_PX}px · 점수 {SCORE_TOL} · 라벨 0 · 개수차 0)") + + +if __name__ == "__main__": + main() diff --git a/tools/verify/dense_head/verify_heads.py b/tools/verify/mmdet/dense_head/verify_heads.py similarity index 95% rename from tools/verify/dense_head/verify_heads.py rename to tools/verify/mmdet/dense_head/verify_heads.py index 96836da..e27b2de 100644 --- a/tools/verify/dense_head/verify_heads.py +++ b/tools/verify/mmdet/dense_head/verify_heads.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""tools/detect/head.cpp 가 mmdet 의 bbox_head 를 재현하는지 계열별로 잰다. +"""src/visp/mmdet_head.cpp 가 mmdet 의 bbox_head 를 재현하는지 계열별로 잰다. 백본만 컴파일하고 head 는 C++ 부품이 조립한다. 계열마다: mmdet → backbone .pt + 파라미터 헤더 → 컴파일 → run_mmdet 빌드 → @@ -61,8 +61,6 @@ def _all_families(): ROOT = CFG.workdir -# head.cpp 를 미리 컴파일해 둘 자리(계열 무관). 지우면 다시 만든다. -HEAD_OBJ = os.path.join(ROOT, "head.o") # 검증 빌드의 최적화 수준. 수치는 libggml 이 내므로 -O1 로 충분하다(컴파일이 빠르다). OPT = CFG.opt # 판정 기준: 출력 텐서의 **상대 L1/L2 거리**. 둘 다 임계 이하여야 PASS. @@ -72,8 +70,8 @@ def _all_families(): # 저장소 관례(verify_pt.py)의 REL_L2_TOL=0.05 를 따른다. L1_TOL = CFG.l1 L2_TOL = CFG.l2 -# g2c(컴파일러)와 vision.cpp 경로. 이 파일은 vision.cpp/tools/verify/dense_head/ 에 있다. -V = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +# g2c(컴파일러)와 vision.cpp 경로. 이 파일은 vision.cpp/tools/verify/mmdet/dense_head/ 에 있다. +V = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) P = CFG.g2c PY = sys.executable FE = V + "/tools/frontend/mmdet" @@ -195,7 +193,7 @@ def save(tag, lst): # 실측(3계열 96초): g2c 46% · export 21% · ref 21% · g++ 9% · 실행 3%. PHASE = {} _PHASE_LOCK = __import__("threading").Lock() -# ⚠️ head.o 빌드용 락은 **따로** 둔다. 계측 락을 재사용하면 `run()` 안에서 같은 락을 +# ⚠️ 공용 빌드 락은 **따로** 둔다. 계측 락을 재사용하면 `run()` 안에서 같은 락을 # 다시 잡아 자기 자신을 기다린다(비재진입 Lock → 교착). 실제로 한 번 걸렸다. _HEAD_LOCK = __import__("threading").Lock() @@ -353,7 +351,7 @@ def run(cmd, cwd, env_extra=None, timeout=2400, phase=None): raise SystemExit( "SubB 의 bbox_pred 가 None 이다 — with_reg=False 인 head 다(좌표를 다른 head 가 낸다). " "이 하네스는 (cls, box) 쌍을 전제하므로 이 계열은 two-stage 하네스로 재라: " - "tools/verify/roi/verify_postproc_roi.py") + "tools/verify/mmdet/roi/verify_postproc_roi.py") for _k in range(len(_o) // 2): _c, _b = _o[2 * _k], _o[2 * _k + 1] np.ascontiguousarray(_c.numpy()).tofile("ref.cls.%%d.bin" %% _k) @@ -399,7 +397,7 @@ def _no_box(fam, cfg_path, cw, d): "-DVISP_ARCH_HEADER=\"visp/arch/Fam.h\"", "-I" + os.path.join(gen, "inc"), "-I" + V + "/src", "-I" + V + "/include", "-I" + V + "/depend/llama/ggml/include", - V + "/tools/verify/backbone/run_dump.cpp", os.path.join(gen, "Fam.cpp"), + V + "/tools/verify/common/run_dump.cpp", os.path.join(gen, "Fam.cpp"), "-L" + V + "/build/lib", "-lvisioncpp", "-lggml", "-lggml-base", "-lggml-cpu", "-Wl,-rpath," + V + "/build/lib", "-o", os.path.join(gen, "run_dump")], d, phase="3_build_nb") @@ -534,7 +532,7 @@ def _two_stage(fam, cfg_path, cw, d): "-IincA", "-IincB", "-I" + V + "/include", "-I" + V + "/src", "-I" + V + "/depend/llama/ggml/include", "-I" + V + "/depend/llama/vendor", - V + "/tools/verify/backbone/run_frcnn.cpp", + V + "/tools/verify/mmdet/backbone/run_frcnn.cpp", "out_FRCNN_SubA/FRCNN_SubA.cpp", "out_" + subs[0] + "/" + subs[0] + ".cpp", "-L" + V + "/build/lib", "-lvisioncpp", "-lggml", "-lggml-base", "-lggml-cpu", "-Wl,-rpath," + V + "/build/lib", "-o", "run_frcnn"], fr, phase="3_build2") @@ -587,7 +585,7 @@ def _two_stage(fam, cfg_path, cw, d): def one(fam, rel, ckpt): d = os.path.join(ROOT, fam) - os.makedirs(d, exist_ok=True) # ROOT 도 같이 생긴다(head.o 자리) + os.makedirs(d, exist_ok=True) # ROOT 도 같이 생긴다 open(os.path.join(d, "_stub.py"), "w").write(STUB) cfg_path = os.path.join(MM, rel) if not os.path.exists(cfg_path): @@ -672,9 +670,15 @@ def _unwrap(out_name, *extra): kind = next((l.split(":")[-1].strip() for l in open(ph) if l.startswith("// head_type")), "?") # ⚠️ **텍스트 조건부 계열은 `bbox_head(feats)` 로 못 부른다.** GLIP·GroundingDINO 는 # 이미지 feature 말고 **텍스트 임베딩**을 같이 받는다(`ATSSVLFusionHead.forward() - # missing 1 required positional argument`). 박스를 판정 기준으로 삼을 수가 없으므로 - # 박스 없는 계열과 같은 자로 잰다 — **컴파일된 그래프**를 torch 와 댄다. - # ⚠️ 그래서 이 숫자는 "이 계열이 검증됐다" 가 아니라 "컴파일 범위가 맞다" 는 뜻이다. + # missing 1 required positional argument`). 박스 하네스로는 못 재므로 **컴파일된 + # 그래프의 출력**을 torch 와 대는 `_no_box` 로 보낸다. + # + # ⚠️ **컴파일 범위가 계열마다 다르다 — 숫자를 같은 뜻으로 읽지 마라.** + # · `glip`: 2026-08-27 부터 **융합헤드까지** 한 그래프다(`MMDetTextCond` 가 텍스트 + # 임베딩을 상수로 굽는다) → 출력 15텐서. 이 숫자는 **헤드까지 검증**한 값이다. + # 아직 없는 것은 **디코드**(박스 delta·토큰→문구 점수 집계)뿐이다. + # · `grounding_dino`·`mm_grounding_dino`: head 가 DETR 꼴이라 아직 백본+넥뿐이다. + # 그 숫자는 "이 계열이 검증됐다" 가 아니라 "컴파일 범위가 맞다" 는 뜻이다. try: from mmengine.config import Config as _C if (_C.fromfile(cfg_path).get("model") or {}).get("language_model"): @@ -715,29 +719,13 @@ def _unwrap(out_name, *extra): if r.returncode != 0: return fam, "WEIGHTS_FAIL", _last_error(r.stderr)[:70] - # 3) run_mmdet 빌드 (백본 .cpp + head.cpp 를 함께 컴파일) + # 3) run_mmdet 빌드 (백본 .cpp 만 컴파일한다 — head 조립과 디코드는 libvisioncpp 안에 + # 있다). 라이브러리를 고쳤으면 먼저 다시 빌드해야 한다. gen = os.path.join(d, "out") inc = os.path.join(gen, "inc", "visp", "arch") os.makedirs(inc, exist_ok=True) import shutil shutil.copy(os.path.join(gen, "Fam.h"), inc) - # head.cpp 는 `ARCH`·파라미터 헤더를 안 쓴다 → **계열마다 다시 컴파일할 이유가 없다.** - # 한 번 .o 로 만들어 두고 링크만 한다(38계열이면 37번을 아낀다). - # 여러 계열이 동시에 들어와도 한 번만 만든다. - with _HEAD_LOCK: - # ⚠️ **head.cpp 가 바뀌면 캐시를 버린다.** mtime 비교가 없으면 조립기를 고쳐도 - # 옛 오브젝트로 링크돼 **값이 그대로**다 — 고친 줄 알고 한참 헤맨다(실제로 겪었다). - _src = V + "/tools/detect/head.cpp" - if (os.path.exists(HEAD_OBJ) - and os.path.getmtime(HEAD_OBJ) < max(os.path.getmtime(_src), - os.path.getmtime(V + "/tools/detect/head.h"))): - os.remove(HEAD_OBJ) - b = run(["g++", "-std=c++20", OPT, "-c", V + "/tools/detect/head.cpp", - "-I" + V + "/include", "-I" + V + "/src", "-I" + V + "/tools/detect", - "-I" + V + "/depend/llama/ggml/include", "-I" + V + "/depend/llama/vendor", - "-o", HEAD_OBJ], d, phase="3a_head") if not os.path.exists(HEAD_OBJ) else None - if not os.path.exists(HEAD_OBJ): - return fam, "BUILD_FAIL", "head.o: " + (_last_error(b.stderr)[:70]) # 최적화 수준은 **수치와 무관**하다 — 실제 계산은 libggml(사전 빌드)이 한다. # 이 코드는 그래프를 짜기만 하므로 -O1 이면 충분하고, 컴파일이 훨씬 빠르다. @@ -754,7 +742,7 @@ def _unwrap(out_name, *extra): "-I" + gen + "/inc", "-I" + V + "/include", "-I" + V + "/src", "-I" + V + "/tools/detect", "-I" + V + "/depend/llama/ggml/include", "-I" + V + "/depend/llama/vendor", - V + "/tools/verify/backbone/run_mmdet.cpp", HEAD_OBJ, + V + "/tools/verify/mmdet/backbone/run_mmdet.cpp", gen + "/Fam.cpp", "-L" + V + "/build/lib", "-lvisioncpp", "-lggml", "-lggml-base", "-lggml-cpu", "-Wl,-rpath," + V + "/build/lib", "-o", gen + "/run_mmdet"], d, phase="3b_build") diff --git a/tools/verify/dense_head/verify_postproc.py b/tools/verify/mmdet/dense_head/verify_postproc.py similarity index 87% rename from tools/verify/dense_head/verify_postproc.py rename to tools/verify/mmdet/dense_head/verify_postproc.py index 857ebed..fe6b4bb 100644 --- a/tools/verify/dense_head/verify_postproc.py +++ b/tools/verify/mmdet/dense_head/verify_postproc.py @@ -40,7 +40,16 @@ def cpp_boxes(gen_dir, image, size, thr): - """run_mmdet 을 돌려 디코드된 박스를 받는다(.bin = 박스당 6값).""" + """run_mmdet 을 돌려 디코드된 박스를 받는다(.bin = 박스당 6값). + + ⚠️ **`image` 는 기준값이 쓴 것과 같은 픽셀이어야 한다.** 여기 오는 것이 + `.bin`(전처리된 CWHN f32)이면 그대로 넘긴다 — 러너가 전처리를 건너뛴다. + jpg 를 넘기면 **러너가 자기 방식으로 디코드·리사이즈**하는데, 기준값 쪽은 + PIL `BILINEAR`(축소 때 안티에일리어싱이 들어간다)로 줄인다. 같은 사진이라도 + 픽셀이 달라져 점수가 계통적으로 어긋난다(실측 중앙값 5e-04~9e-04). + 그 크기면 컷(0.30) 언저리의 검출 하나가 갈려 **개수차 1** 이 난다. + two-stage 하네스는 처음부터 `in.bin` 을 한 번 만들어 양쪽에 준다. + """ out = os.path.join(gen_dir, "_postproc_check.bin") exe = os.path.join(gen_dir, "run_mmdet") if not os.path.exists(exe): @@ -60,6 +69,11 @@ def _one_gguf(gen_dir): return os.path.join(gen_dir, g[0]) +# 기준값이 만든 픽셀을 러너에게도 주기 위한 슬롯. `main()` 이 채운다. +# 리스트인 이유는 함수 서명을 안 바꾸려는 것뿐이다(부르는 곳이 여럿이다). +_SHARED_INPUT = [None] + + def mmdet_boxes(cfg, ckpt, image, size, thr, to_rgb): """mmdet 자신의 predict_by_feat — 앵커·디코드·NMS 의 정본.""" from mmdet.apis import init_detector @@ -67,8 +81,11 @@ def mmdet_boxes(cfg, ckpt, image, size, thr, to_rgb): # mmdet v3 체크포인트는 학습 메타(HistoryBuffer)를 함께 담고 있어 torch 2.6 의 # weights_only=True 기본값에서 로드가 거부된다(rtmdet 이 그랬다). 프론트엔드가 # 이미 쓰는 우회를 그대로 쓴다 — 필요한 클래스만 이름으로 허용한다. + # ⚠️ `..` 은 **셋**이다. dense_head → mmdet → verify → tools 로 세 칸 올라가야 + # `tools/frontend/mmdet` 이다. 둘이면 없는 경로라 import 가 조용히 실패하고, + # 허용 목록이 안 걸려 DETR 계열 8개가 통째로 RUN_FAIL 났다(2026-09-01 실측). sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), - "..", "..", "frontend", "mmdet")) + "..", "..", "..", "frontend", "mmdet")) try: import mmdet_wrap mmdet_wrap.allow_mmengine_checkpoint_globals() @@ -97,6 +114,11 @@ def mmdet_boxes(cfg, ckpt, image, size, thr, to_rgb): # `expected scalar type Double but found Float` 로 죽는다. 계열 탓처럼 보이지만 # 전처리 dtype 문제다. x = ((np.ascontiguousarray(x) - mean) / std).astype(np.float32) + # ⚠️ **러너에게도 이 픽셀을 그대로 준다.** 각자 jpg 를 열면 리사이즈 구현이 + # 달라 픽셀이 갈리고, 그러면 백엔드가 아니라 **리사이즈를 재게 된다.** + # `x` 는 이미 HWC(=러너의 cwhn) 라 그대로 쓴다. + if _SHARED_INPUT[0]: + np.ascontiguousarray(x).tofile(_SHARED_INPUT[0]) t = torch.from_numpy(x).permute(2, 0, 1).unsqueeze(0) meta = {"img_shape": (size, size), "ori_shape": (size, size), @@ -226,8 +248,12 @@ def main(): except Exception: # 계열을 못 알아내도 계속 간다 pass - got = cpp_boxes(gen, image, size, thr) + # ⚠️ **순서를 바꾸지 마라.** 기준값 쪽이 픽셀을 만들면서 `in.bin` 을 떨구고, + # 러너는 그 파일을 받는다. 러너를 먼저 돌리면 파일이 없어 jpg 로 떨어지고 + # **각자 리사이즈**하게 된다(그게 원래 상태였다). + _SHARED_INPUT[0] = os.path.join(gen, "_postproc_in.bin") ref = mmdet_boxes(cfg, ckpt, image, size, thr, to_rgb) + got = cpp_boxes(gen, _SHARED_INPUT[0], size, thr) print(f"\nmmdet {len(ref)}건 · run_mmdet {len(got)}건") rows = match(ref, got) diff --git a/tools/verify/seg/run_maskrcnn.cpp b/tools/verify/mmdet/inst_seg/run_maskrcnn.cpp similarity index 100% rename from tools/verify/seg/run_maskrcnn.cpp rename to tools/verify/mmdet/inst_seg/run_maskrcnn.cpp diff --git a/tools/verify/roi/run_frcnn.cpp b/tools/verify/mmdet/roi/run_frcnn.cpp similarity index 100% rename from tools/verify/roi/run_frcnn.cpp rename to tools/verify/mmdet/roi/run_frcnn.cpp diff --git a/tools/verify/roi/run_roi_verify.cpp b/tools/verify/mmdet/roi/run_roi_verify.cpp similarity index 100% rename from tools/verify/roi/run_roi_verify.cpp rename to tools/verify/mmdet/roi/run_roi_verify.cpp diff --git a/tools/verify/roi/run_rpn_verify.cpp b/tools/verify/mmdet/roi/run_rpn_verify.cpp similarity index 100% rename from tools/verify/roi/run_rpn_verify.cpp rename to tools/verify/mmdet/roi/run_rpn_verify.cpp diff --git a/tools/verify/roi/verify_postproc_roi.py b/tools/verify/mmdet/roi/verify_postproc_roi.py similarity index 70% rename from tools/verify/roi/verify_postproc_roi.py rename to tools/verify/mmdet/roi/verify_postproc_roi.py index 47d8b6b..1ef302d 100644 --- a/tools/verify/roi/verify_postproc_roi.py +++ b/tools/verify/mmdet/roi/verify_postproc_roi.py @@ -33,9 +33,13 @@ import time HERE = os.path.dirname(os.path.abspath(__file__)) -V = os.path.abspath(os.path.join(HERE, "..", "..", "..")) # vision.cpp (tools/verify/roi 에서 3단계 위) +V = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..")) # vision.cpp (tools/verify/mmdet/roi 에서 4단계 위) FE = os.path.join(V, "tools", "frontend", "mmdet") -DH = os.path.join(V, "tools", "verify", "dense_head") +# ⚠️ `dense_head` 는 `verify/` 바로 밑이 아니라 `verify/mmdet/` 밑이다. 빠뜨리면 +# `os.path.join` 이 없는 경로를 만들고 `sys.path.insert` 도 성공한다 — +# `import mmdet_families` 만 죽는다. → wiki `하네스-상대경로는-틀려도-안-죽는다` +DH = os.path.join(V, "tools", "verify", "mmdet", "dense_head") +assert os.path.isdir(DH), f"dense_head 를 못 찾는다: {DH}" GGUF_PY = os.path.join(V, "depend", "llama", "gguf-py") G2C = os.path.abspath(os.path.join(V, "..")) # vision.cpp 를 담은 컴파일러 루트 MM = os.path.expanduser(os.environ.get("MMDET", "~/mmbuild/mmdetection")) @@ -68,6 +72,15 @@ # 판정 기준 — dense head 와 **같게 둔다**. 다르면 18계열 숫자와 나란히 못 놓는다. BOX_TOL, SCORE_TOL, THR = 2.0, 0.05, 0.30 +# 마스크 축 — `paste_mask` **전** 로짓의 상대 L1. 텐서 축이 이미 쓰는 수라 새 기준을 +# 만들지 않는다. ⚠️ **이진 마스크 IoU 는 쓰지 않는다** — 0.5 이진화는 절벽이라 fp16 +# 타이 플립이 그대로 점수에 실린다. +MASK_TOL = 5e-2 +# ⚠️ **게이트는 이 집합뿐이다.** 다른 마스크 계열은 수치만 찍는다 — 모수가 1인 축을 +# 전 계열 문턱으로 올리면, 재본 적 없는 계열을 측정 없이 재분류하게 된다. +# 수치가 모이면 **별도 이슈로** 전체 게이트를 켤지 판단한다. +MASK_GATE = {"groie"} + def run(cmd, cwd, env_extra=None, timeout=3600): env = dict(os.environ, OMP_NUM_THREADS="1") @@ -143,6 +156,37 @@ def last_error(text): print("REF_OK", len(res.scores)) ''' +# 마스크 축 기준값 — **러너가 쓴 그 박스**로 mmdet 의 마스크 갈래를 돌린다. +# +# ⚠️ **mmdet 이 스스로 고른 검출로 재면 안 된다.** 박스가 0.05px 라도 다르면 RoIAlign +# 격자가 달라져 로짓이 통째로 움직인다 — 그러면 마스크 head 가 아니라 **박스 차이**를 +# 재게 된다. 러너가 낸 좌표를 그대로 먹여야 마스크 갈래만 남는다. +# ⚠️ `_mask_forward` 를 부른다. RoI 추출기 선택·shared_head 까지 mmdet 자신의 순서다 — +# 우리가 다시 조립하면 그 조립을 재게 된다(`groie` 는 추출기가 `SumGenericRoiExtractor`). +MREF = r''' +import _stub, sys, numpy as np, torch +cfg, ckpt, size, npy, bx, out = (sys.argv[1], sys.argv[2], int(sys.argv[3]), + sys.argv[4], sys.argv[5], sys.argv[6]) +sys.path.insert(0, "%(FE)s") +try: + import mmdet_wrap; mmdet_wrap.allow_mmengine_checkpoint_globals() +except Exception: + pass +from mmdet.apis import init_detector +from mmdet.structures.bbox import bbox2roi +from frcnn_to_pt import _desync_norm +det = init_detector(_desync_norm(cfg), ckpt, device="cpu"); det.eval() +x = np.load(npy) +t = torch.from_numpy(x).permute(2, 0, 1).unsqueeze(0).contiguous() +boxes = np.load(bx) +with torch.no_grad(): + feats = det.extract_feat(t) + rois = bbox2roi([torch.from_numpy(boxes.astype("float32"))]) + mp = det.roi_head._mask_forward(feats, rois)["mask_preds"] +np.save(out, mp.numpy()) +print("MREF_OK", tuple(mp.shape)) +''' + # 전처리 상수는 detector 의 `data_preprocessor` 가 정본이다. 러너와 torch 에 **같은 배열**을 # 주기 위해, 여기서 한 번만 만들어 `.npy`(torch용) 와 `.bin`(러너용, cwhn) 로 내보낸다. PREP = r''' @@ -390,11 +434,21 @@ def _one(fam, size, image, workdir, keep, verbose): # 크래시가 없어 조용히 틀린다. mask head 의 14→28 deconv 가 그 경로다. has_miou = bool(J.get("has_mask_iou")) MO = int(J.get("mask_roi_out", 14)) + # 마스크 축을 재려면 SubC(mask head)를 구워야 한다. head 가 **하나일 때만** 굽는다 — + # HTC·SCNet 은 `mask_head` 가 `ModuleList` 라 통째로 태울 수 없다(`mask_head_single`). + # ⚠️ **여기서 실패해도 박스 판정을 바꾸지 않는다.** 아래 컴파일 루프가 SubC 만 + # 따로 처리하는 이유다 — 마스크는 더 재는 축이지 문턱이 아니다. + want_mask = bool(J.get("has_mask")) and bool(J.get("mask_head_single")) + mask_jobs = [] + if want_mask: + # ⚠️ 배치가 **1 이 아닐 수 있다.** 마스크 추출기가 `GenericRoIExtractor` 면 러너가 + # 레벨 L 개를 배치로 쌓아 넣고 SubC 가 그 안에서 합산한다(`mask_groie_levels`). + MLV = int(J.get("mask_groie_levels") or 0) or 1 + mask_jobs += [("MaskRCNN_SubC", "MaskRCNN_SubC", "out_MaskRCNN_SubC", + f"{MLV},{RC},{MO},{MO}")] if has_miou: - jobs += [("MaskRCNN_SubC", "MaskRCNN_SubC", "out_MaskRCNN_SubC", - f"1,{RC},{MO},{MO}"), - ("MSRCNN_SubD", "MSRCNN_SubD", "out_MSRCNN_SubD", - f"1,{RC + 1},{MO},{MO}")] + mask_jobs += [("MSRCNN_SubD", "MSRCNN_SubD", "out_MSRCNN_SubD", + f"1,{RC + 1},{MO},{MO}")] # Grid R-CNN: bbox head 에 회귀 분기가 없고 격자점 히트맵이 박스를 낸다. # 여기도 배치 1 이다 — grid head 가 deconv 를 두 번 탄다. has_grid = bool(J.get("has_grid")) @@ -402,7 +456,8 @@ def _one(fam, size, image, workdir, keep, verbose): if has_grid: jobs += [("GridRCNN_SubE", "GridRCNN_SubE", "out_GridRCNN_SubE", f"1,{RC},{GO},{GO}")] - for src, name, outdir, shape in jobs: + def compile_sub(src, name, outdir, shape): + """서브그래프 하나를 굽는다. 성공하면 None, 실패하면 사유 문자열.""" try: os.remove(os.path.join(fr, outdir, f"{name}.gguf")) # 위와 같은 이유 except OSError: @@ -413,13 +468,38 @@ def _one(fam, size, image, workdir, keep, verbose): from shared.compile.pipeline import main; main() '''], fr, {"PYTHONPATH": f"{d}:{fr}:{G2C}:{FE}:{GGUF_PY}"}) if not os.path.exists(os.path.join(fr, outdir, f"{name}.gguf")): - return fam, "COMPILE_FAIL", f"{src}: " + last_error(r.stderr)[:100], None + return f"{src}: " + last_error(r.stderr)[:100] + return None + + for src, name, outdir, shape in jobs: + why = compile_sub(src, name, outdir, shape) + if why: + return fam, "COMPILE_FAIL", why, None + + # 마스크 갈래는 **박스 판정과 분리한다.** 마스크 축만 끄고 박스는 그대로 잰다 — + # 마스크가 없다고 이미 맞은 박스를 못 쓰게 만들 이유가 없다. + # ⚠️ 단, 끈 사실을 **말한다.** 조용히 끄면 다음 사람이 "쟀는데 통과했다" 로 읽는다. + # ⚠️ **Mask Scoring R-CNN 은 예외다.** 거기서는 마스크가 더 재는 축이 아니라 + # **점수를 정하는 경로**다(`score *= mask_iou`) — SubC·SubD 중 하나만 없어도 + # 박스 점수가 틀리므로 예전처럼 COMPILE_FAIL 이어야 한다. + mask_off = None + for src, name, outdir, shape in mask_jobs: + why = compile_sub(src, name, outdir, shape) + if not why: + continue + if has_miou: + return fam, "COMPILE_FAIL", why, None + mask_off = why + want_mask = False + break # ③ 러너 빌드 — 빌드 라인은 build_frcnn_cpp.sh / verify_heads.py 와 같아야 한다. import shutil incs = [("FRCNN_SubA", "incA"), (subs[0], "incB")] + if want_mask: + incs += [("MaskRCNN_SubC", "incC")] if has_miou: - incs += [("MaskRCNN_SubC", "incC"), ("MSRCNN_SubD", "incD")] + incs += [("MSRCNN_SubD", "incD")] if has_grid: incs += [("GridRCNN_SubE", "incE")] for name, inc in incs: @@ -427,11 +507,12 @@ def _one(fam, size, image, workdir, keep, verbose): shutil.copy(os.path.join(fr, "out_" + name, name + ".h"), os.path.join(fr, inc, "visp", "arch")) extra = [] + if want_mask: + extra += ["-DARCH_C=MaskRCNN_SubC", + '-DVISP_ARCH_HEADER_C="visp/arch/MaskRCNN_SubC.h"', "-IincC"] if has_miou: - extra += ["-DARCH_C=MaskRCNN_SubC", "-DARCH_D=MSRCNN_SubD", - '-DVISP_ARCH_HEADER_C="visp/arch/MaskRCNN_SubC.h"', - '-DVISP_ARCH_HEADER_D="visp/arch/MSRCNN_SubD.h"', - "-IincC", "-IincD"] + extra += ["-DARCH_D=MSRCNN_SubD", + '-DVISP_ARCH_HEADER_D="visp/arch/MSRCNN_SubD.h"', "-IincD"] if has_grid: extra += ["-DARCH_E=GridRCNN_SubE", '-DVISP_ARCH_HEADER_E="visp/arch/GridRCNN_SubE.h"', "-IincE"] @@ -440,10 +521,10 @@ def _one(fam, size, image, workdir, keep, verbose): f'-DVISP_ARCH_HEADER_B="visp/arch/{subs[0]}.h"'] + extra + [ "-IincA", "-IincB", "-I" + V + "/include", "-I" + V + "/src", "-I" + V + "/depend/llama/ggml/include", "-I" + V + "/depend/llama/vendor", - V + "/tools/verify/backbone/run_frcnn.cpp", + V + "/tools/verify/mmdet/backbone/run_frcnn.cpp", "out_FRCNN_SubA/FRCNN_SubA.cpp", f"out_{subs[0]}/{subs[0]}.cpp"] + ( - ["out_MaskRCNN_SubC/MaskRCNN_SubC.cpp", "out_MSRCNN_SubD/MSRCNN_SubD.cpp"] - if has_miou else []) + ( + ["out_MaskRCNN_SubC/MaskRCNN_SubC.cpp"] if want_mask else []) + ( + ["out_MSRCNN_SubD/MSRCNN_SubD.cpp"] if has_miou else []) + ( ["out_GridRCNN_SubE/GridRCNN_SubE.cpp"] if has_grid else []) + [ "-L" + BUILD + "/lib", "-lvisioncpp", "-lggml", "-lggml-base", "-lggml-cpu", "-Wl,-rpath," + BUILD + "/lib", "-o", "run_frcnn"], fr) @@ -464,18 +545,20 @@ def _one(fam, size, image, workdir, keep, verbose): argv = [os.path.join(fr, "run_frcnn"), "out_FRCNN_SubA/FRCNN_SubA.gguf", ",".join(f"out_{s}/{subs[0]}.gguf" for s in subs), "frcnn.json", binp, pref, str(size)] - if has_miou: - argv += ["out_MaskRCNN_SubC/MaskRCNN_SubC.gguf", "out_MSRCNN_SubD/MSRCNN_SubD.gguf"] + # 자리 인자다 — 7=SubC · 8=SubD · 9=SubE. 없는 자리는 빈 문자열로 채운다. + if want_mask or has_miou or has_grid: + argv.append("out_MaskRCNN_SubC/MaskRCNN_SubC.gguf" if want_mask else "") + if has_miou or has_grid: + argv.append("out_MSRCNN_SubD/MSRCNN_SubD.gguf" if has_miou else "") if has_grid: - # 러너 인자는 자리로 읽는다 — SubE 는 9번째다. 마스크가 없으면 자리를 채운다. - while len(argv) < 9: - argv.append("") argv.append("out_GridRCNN_SubE/GridRCNN_SubE.gguf") rr = run(argv, fr, {"VISP_BACKEND": "cpu", **prop_env}) if not os.path.exists(pref + ".boxes.bin"): return fam, "RUN_FAIL", last_error(rr.stderr)[:110], None - got = np.fromfile(pref + ".boxes.bin", dtype="float32").reshape(-1, 6) - got = got[got[:, 4] >= THR] + got_all = np.fromfile(pref + ".boxes.bin", dtype="float32").reshape(-1, 6) + # ⚠️ 마스크 로짓은 **거르기 전** 검출 전부에 대해 나온다 — 러너가 그 순서로 돌렸다. + # 거른 뒤 배열로 짝지으면 행이 밀린다. + got = got_all[got_all[:, 4] >= THR] # ⑥ mmdet 기준값 open(os.path.join(d, "ref.py"), "w").write(REF % {"FE": FE}) @@ -502,6 +585,17 @@ def _one(fam, size, image, workdir, keep, verbose): # 안 적으면 나중에 0건이 "대상이 못 한다" 인지 "안 맞는 사진을 넣었다" 인지 못 가른다. if os.path.basename(image) != os.path.basename(default_image): note += f" · {os.path.basename(image)}" + # ── ⑦ 마스크 축 — `paste_mask` 전 로짓의 상대 L1 ───────────────────────── + # ⚠️ **박스 판정과 섞지 않는다.** 게이트는 `MASK_GATE` 뿐이고, 나머지 계열은 + # 수치만 남긴다. 못 잰 계열에 「통과」도 「실패」도 적지 않는다. + if mask_off: + note += f" · 마스크 못 잼({mask_off[:40]})" + elif want_mask: + m_note, m_ok = _mask_axis(fam, d, fr, pref, cfg, ckpt, size, npy, got_all, prop_env) + note += " · " + m_note + if fam in MASK_GATE and not m_ok: + ok = False + if verbose and rows: for r, g, db in rows: print(f" {int(r[5]):4d} [{r[0]:6.1f},{r[1]:6.1f},{r[2]:6.1f},{r[3]:6.1f}] {r[4]:.3f}" @@ -511,6 +605,99 @@ def _one(fam, size, image, workdir, keep, verbose): return fam, ("PASS" if ok else "FAIL"), note, dt +def _mask_axis(fam, d, fr, pref, cfg, ckpt, size, npy, got_all, prop_env): + """마스크 로짓을 mmdet 과 대조한다. `(비고 문자열, 통과 여부)`. + + ⚠️ **못 잰 것을 실패로 적지 않는다.** 로짓 파일이 없거나 기준값이 안 나오면 + 「못 잼」이라고 쓰고 `True` 를 돌려준다 — 게이트 계열이라도 측정 실패를 + 대상의 실패로 바꾸지 않는다. 게이트는 **수치가 나왔는데 넘었을 때** 걸린다. + """ + import numpy as np + lg, dims_p = pref + ".mlogit.bin", pref + ".mlogit.dims.bin" + if not (os.path.exists(lg) and os.path.exists(dims_p)): + return "마스크 못 잼(로짓 없음)", True + dims = np.fromfile(dims_p, dtype="float32") + if dims.size < 4: + return "마스크 못 잼(축 미상)", True + MD, NCLS, MH, MW = (int(v) for v in dims[:4]) + if MD <= 0: + return "마스크 0건", True + + got = np.fromfile(lg, dtype="float32") + want_n = MD * NCLS * MH * MW + if got.size != want_n: + return f"마스크 못 잼(원소 {got.size} != {want_n})", True + # 러너는 행마다 **cwhn** 으로 낸다: ((y*MW + x) * NCLS + k). torch 는 NCHW 다. + got = got.reshape(MD, MH, MW, NCLS).transpose(0, 3, 1, 2) + + bx = os.path.join(d, "mboxes.npy") + np.save(bx, got_all[:MD, :4].astype("float32")) + open(os.path.join(d, "mref.py"), "w").write(MREF % {"FE": FE}) + refp = os.path.join(d, "mref.npy") + q = run([PY, "mref.py", cfg, ckpt, str(size), npy, bx, refp], d, + {"PYTHONPATH": f"{d}:{FE}", **prop_env}) + if "MREF_OK" not in (q.stdout or ""): + return "마스크 못 잼(기준값: " + last_error(q.stderr)[:40] + ")", True + ref = np.load(refp) + if ref.shape != got.shape: + return f"마스크 못 잼(모양 {tuple(ref.shape)} vs {tuple(got.shape)})", True + + den = float(np.abs(ref).sum()) + if den <= 0: + return "마스크 못 잼(기준값이 0)", True + rel = float(np.abs(ref - got).sum()) / den + # 이진 IoU 는 **적되 판정에 쓰지 않는다** — 0.5 이진화는 절벽이라 fp16 타이 플립이 + # 그대로 점수에 실린다. 눈으로 볼 때만 쓴다. + a, b = ref > 0, got > 0 + inter, uni = float((a & b).sum()), float((a | b).sum()) + iou = inter / uni if uni else 1.0 + tag = "PASS" if rel < MASK_TOL else "FAIL" + if fam not in MASK_GATE: + tag = "수치만" # 게이트가 아닌 계열은 판정하지 않는다 + return f"마스크 rel L1 {rel:.2e} (IoU {iou:.3f}, {MD}건) {tag}", rel < MASK_TOL + + +# 계열 하나의 피크 RSS 는 **3.26GB** 다(실측 2026-08-25, 60초 샘플링 — 최대 기여는 +# `ref.py` 의 `init_detector` + 800px 추론). WSL 은 `.wslconfig` 로 13GB 상한이다. +# 그래서 워커 2가 안전선이고 3이 상한이다 — 그 이상은 swap 으로 밀려 **더 느려진다.** +# ⚠️ **OOM 을 내면 WSL 이 통째로 죽는다.** 이 저장소는 이미 겪었다(2026-08-10: +# OOM 0건인데 크래시 덤프 626회로 C: 가 찼다). `.wslconfig` 의 `maxCrashDumpCount=0` +# 이 그 대응이니 되돌리지 마라. +WORKER_CAP = 3 +PER_FAM_MB = 3400 # 계열당 피크 RSS(실측 3.26GB)에 여유를 붙인 값 + + +def _avail_mb(): + """가용 메모리(MB). 못 읽으면 가드를 끈다 — 측정 실패로 막지 않는다.""" + try: + for line in open("/proc/meminfo"): + if line.startswith("MemAvailable:"): + return int(line.split()[1]) // 1024 + except Exception: + pass + return 1 << 30 + + +def _safe_workers(want): + """요청한 워커 수를 **가용 메모리로 다시 깎는다.** 플래그를 그대로 믿지 않는다. + + ⚠️ 왜 플래그를 안 믿나 — 이 기계는 세션이 여럿 떠 있고 그때그때 여유가 다르다. + `--workers 3` 이 어제 됐다고 오늘 되는 게 아니다. **재서 정한다.** + """ + want = max(1, int(want)) + if want > WORKER_CAP: + print(f" ⚠️ --workers {want} 는 상한 {WORKER_CAP} 을 넘는다 — {WORKER_CAP} 로 낮춘다." + f" 계열당 피크 RSS 가 {PER_FAM_MB}MB 다(실측).", flush=True) + want = WORKER_CAP + avail = _avail_mb() + fit = max(1, avail // PER_FAM_MB) + if fit < want: + print(f" ⚠️ 가용 메모리 {avail}MB → 워커 {want} → **{fit}** 로 낮춘다" + f" (계열당 {PER_FAM_MB}MB).", flush=True) + return fit + return want + + def two_stage_families(): """config 로 판정한다 — 이름으로 짐작하지 않는다. roi_head 가 있으면 two-stage. @@ -556,6 +743,9 @@ def main(): # 수정이 특정 조건에서만 도는 코드면(예: `NS>1`·`RSF>0`) 영향 계열만 골라 재라. ap.add_argument("--skip-pass", metavar="results.json", help="이전 결과에서 PASS 였던 계열은 건너뛴다") + ap.add_argument("--workers", type=int, default=1, + help=f"동시 실행 계열 수(기본 1, 상한 {WORKER_CAP}). " + "가용 메모리를 읽어 더 낮출 수 있다") ap.add_argument("-v", "--verbose", action="store_true") a = ap.parse_args() @@ -587,21 +777,60 @@ def main(): # 배너가 거짓말하지 않게 — 계열별 지정이 끼어들 수 있으면 그렇다고 적는다. img_note = os.path.basename(a.image) if a.image else \ f"{os.path.basename(DEFAULT_IMAGE)} (계열별 지정 적용)" - print(f"size={a.size} · image={img_note} · thr={THR}" + # ⚠️ **머리글이 실제 측정 조건을 말해야 한다.** `a.size` 만 찍으면 계열별 크기 + # 표(`MF._SIZE`)로 올라간 계열이 로그에 거짓으로 적힌다 — `fpg` 를 1024 로 + # 재고도 `size=800` 이라고 찍혔다(2026-09-01). 합계는 입력 크기와 같이 묶는다. + over = {f: MF.test_size(f, a.size) for f in fams if MF.test_size(f, a.size) != a.size} + size_note = f"{a.size}" + ( + " (계열별: " + ", ".join(f"{f}={v}" for f, v in sorted(over.items())) + ")" + if over else "") + print(f"size={size_note} · image={img_note} · thr={THR}" f" · 판정: 박스<{BOX_TOL}px 점수<{SCORE_TOL} 라벨0 개수차0") print(f"{len(fams)}계열: {' '.join(fams)}\n") - res = [] - for i, fam in enumerate(fams, 1): - print(f"[{i}/{len(fams)}] {fam} …", flush=True) + nw = _safe_workers(a.workers) + if nw > 1: + print(f"워커 {nw}개로 돈다 (가용 {_avail_mb()}MB · 계열당 {PER_FAM_MB}MB 가정)\n", + flush=True) + + def _one_guarded(fam): + """한 계열. **판정 로직은 순차와 완전히 같다** — 감싸기만 한다.""" try: - row = one(fam, a.size, a.image, a.workdir, a.keep, a.verbose) + # 계열별 크기 표를 따른다 — `fpg` 는 800 에서 안 나눠떨어진다 + return one(fam, MF.test_size(fam, a.size), a.image, + a.workdir, a.keep, a.verbose) except subprocess.TimeoutExpired: - row = (fam, "TIMEOUT", "-", None) + return (fam, "TIMEOUT", "-", None) except Exception as e: # 한 계열이 죽어도 스윕은 계속한다 - row = (fam, "HARNESS_FAIL", f"{type(e).__name__}: {e}"[:110], None) - res.append(row) - print(f" {row[1]:14s} {row[2]}" + (f" ({row[3]:.0f}s)" if row[3] else ""), flush=True) + return (fam, "HARNESS_FAIL", f"{type(e).__name__}: {e}"[:110], None) + + res = [] + # ⚠️ **계열마다 작업 폴더가 따로다**(`workdir/`) — 겹치는 파일이 없어야 + # 병렬이 안전하다. `one()` 이 이미 그렇게 짜여 있다(정리도 그 폴더 단위). + # 여기서 `workdir` 하나를 공유하는 파일은 `results.json` 뿐이고, 그건 부모만 쓴다. + if nw <= 1: + for i, fam in enumerate(fams, 1): + print(f"[{i}/{len(fams)}] {fam} …", flush=True) + row = _one_guarded(fam) + res.append(row) + print(f" {row[1]:14s} {row[2]}" + (f" ({row[3]:.0f}s)" if row[3] else ""), + flush=True) + with open(os.path.join(a.workdir, "results.json"), "w") as f: + json.dump(res, f, indent=1, ensure_ascii=False) + else: + import concurrent.futures as _fut + with _fut.ThreadPoolExecutor(max_workers=nw) as ex: + futs = {ex.submit(_one_guarded, f): f for f in fams} + for i, fu in enumerate(_fut.as_completed(futs), 1): + row = fu.result() + res.append(row) + print(f"[{i}/{len(fams)}] {row[0]:22s} {row[1]:14s} {row[2]}" + + (f" ({row[3]:.0f}s)" if row[3] else ""), flush=True) + with open(os.path.join(a.workdir, "results.json"), "w") as f: + json.dump(res, f, indent=1, ensure_ascii=False) + # ⚠️ **완료 순서로 오므로 다시 정렬한다.** 안 하면 실행마다 줄 순서가 달라져 + # 두 실행을 `diff` 로 못 댄다 — 회귀를 눈으로 보는 길이 막힌다. + res.sort(key=lambda r: fams.index(r[0])) with open(os.path.join(a.workdir, "results.json"), "w") as f: json.dump(res, f, indent=1, ensure_ascii=False) diff --git a/tools/verify/tracking/run_bytetrack_verify.cpp b/tools/verify/mmdet/tracking/run_bytetrack_verify.cpp similarity index 100% rename from tools/verify/tracking/run_bytetrack_verify.cpp rename to tools/verify/mmdet/tracking/run_bytetrack_verify.cpp diff --git a/tools/verify/mmpose/verify_pose.py b/tools/verify/mmpose/verify_pose.py new file mode 100644 index 0000000..7d065fe --- /dev/null +++ b/tools/verify/mmpose/verify_pose.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +"""mmpose 계열을 g2c 로 컴파일해 C++(ggml) 출력이 PyTorch 와 같은지 계열별로 잰다. + +`tools/verify/mmseg/verify_seg.py` 와 같은 자로 잰다 — 앵커도 NMS 도 없으니 +`backbone → (neck) → head.forward` 를 한 그래프로 굽고 **출력 텐서를 그대로 대조**한다. +디코드(히트맵 argmax·소수점 보정, SimCC 의 1D argmax)는 후처리라 그래프 밖이다. + + python verify_pose.py # metafile 에 있는 계열 전부 + python verify_pose.py hrnet # 골라서 + python verify_pose.py --list # 목록만 + +⚠️ **입력이 정방이 아니다.** topdown 은 사람 박스를 `codec.input_size`(대개 192x256, WxH)로 + 잘라 넣는다. 정방으로 재면 히트맵 크기가 config 와 달라져 **다른 것을 재게 된다.** +""" +import argparse +import glob +import os +import subprocess +import sys + +sys.stdout.reconfigure(line_buffering=True) + +HERE = os.path.dirname(os.path.abspath(__file__)) +V = os.path.abspath(os.path.join(HERE, "..", "..", "..")) +P = os.path.abspath(os.path.join(V, "..")) +FE = V + "/tools/frontend/mmpose" +FE_DET = V + "/tools/frontend/mmdet" +GGUF_PY = V + "/depend/llama/gguf-py" +PY = sys.executable + +MM = os.path.expanduser("~/mmbuild/mmpose") +CKPT = MM + "/checkpoints" +WORKDIR = "/tmp/visp-pose-verify" +OPT = "-O1" +L1_TOL = L2_TOL = 0.05 + + +def families(): + """`configs/**/metafile.yml` 에서 (계열, config, weights URL). **손으로 안 적는다.** + + mmpose 의 metafile 은 `configs//<...>//*.yml` 로 깊이가 들쭉날쭉하다 → + 계열 이름은 **metafile 이 있는 디렉터리 이름**으로 잡고, 같은 이름이 여럿이면 첫 것만. + """ + import yaml + out, seen = [], set() + for p in sorted(glob.glob(MM + "/configs/**/*.yml", recursive=True)): + try: + d = yaml.safe_load(open(p)) or {} + except Exception: + continue + models = d.get("Models") or [] + if not models: + continue + fam = os.path.basename(os.path.dirname(p)) + if fam in seen: + continue + m = models[0] + cfg = m.get("Config") or "" + if not cfg: + continue + cfg = cfg if cfg.startswith("/") else MM + "/" + cfg + # ⚠️ **metafile 의 Config 경로가 낡아 있을 수 있다.** mmpose 1.x 로 옮기면서 + # 파일은 metafile 옆으로 왔는데 `Config:` 는 0.x 경로(`configs/body/ + # 2d_kpt_sview_rgb_img/...`)를 그대로 들고 있는 것이 있다(`cpm_jhmdb.yml`). + # 그러면 `CONFIG_NONE` 이 되어 **컴파일러가 못 하는 것처럼 보인다** — 아니다. + # 파일명이 같은 것이 metafile 디렉터리에 있으면 그것을 쓴다. + if not os.path.exists(cfg): + alt = os.path.join(os.path.dirname(p), os.path.basename(cfg)) + if os.path.exists(alt): + cfg = alt + seen.add(fam) + out.append((fam, cfg, m.get("Weights", ""))) + return out + + +# ⚠️ **메모리 가드.** 위키 `wsl-계속-터짐` 5번째 원인 — 폭주한 프로세스가 없어도 **합**이 +# 넘치면 WSL 이 통째로 죽는다(2026-08-27 실측: 내 스윕 3.6GB + 다른 python 2.25GB + +# VS Code 3.4GB + claude 세션 5개 1.5GB = 13GB 상한 초과 → OOM 8건). +# 가용 메모리가 이 밑이면 **새 계열을 안 띄우고 기다린다.** 느려질지언정 안 죽는다. +MIN_FREE_MB = int(os.environ.get("VISP_MIN_FREE_MB", "2500")) +# 한 서브프로세스가 이 이상 잡으면 **그놈만** 죽는다(VM 이 아니라). 위키의 인덱서 대책과 같다. +MAX_SUBPROC_GB = int(os.environ.get("VISP_MAX_SUBPROC_GB", "8")) + + +def _avail_mb(): + try: + for line in open("/proc/meminfo"): + if line.startswith("MemAvailable:"): + return int(line.split()[1]) // 1024 + except Exception: + pass + return 1 << 30 # 못 읽으면 가드를 끈다(측정 실패로 막지 않는다) + + +_MEM_GATE = __import__("threading").Lock() + + +def _wait_for_memory(fam): + """가용 메모리가 회복될 때까지 기다린다. + + ⚠️ **락으로 감싼다.** 워커가 여럿이면 둘이 동시에 검사를 통과한 뒤 **둘 다** 할당해 + 가드가 무의미해진다(`verify_seg.py` 와 같은 이유·같은 수법). 한 번에 하나만 + 문을 지나게 하고, 지난 뒤엔 바로 놓는다 — 할당은 서브프로세스가 하므로 + 그때까지 붙들 필요가 없다. + + ⚠️ 이 게이트는 `/proc/meminfo` 의 **시스템 전역** 값을 본다. 그래서 mmseg 스윕과 + mmpose 스윕을 **동시에 돌려도 서로를 자동으로 throttle 한다** — 두 프로세스가 + 서로를 몰라도 된다. 동시 운용할 때는 `VISP_MIN_FREE_MB` 를 올려 잡는다. + """ + import time + waited = 0 + with _MEM_GATE: + while _avail_mb() < MIN_FREE_MB: + if waited == 0: + print(f" … 메모리 대기 ({fam}): 가용 {_avail_mb()}MB < {MIN_FREE_MB}MB", + flush=True) + time.sleep(5) + waited += 5 + if waited > 600: # 10분을 기다려도 안 풀리면 그냥 간다(교착 방지) + print(f" … 10분 대기 후에도 부족 — 그대로 진행한다 ({fam})", flush=True) + break + + +def run(cmd, cwd=None, env=None, timeout=1800): + e = dict(os.environ) + e.setdefault("OMP_NUM_THREADS", "1") + # glibc 이 스레드마다 arena 를 잡아 RSS 가 부푼다. torch 서브프로세스에서 크게 온다. + e.setdefault("MALLOC_ARENA_MAX", "2") + if env: + e.update(env) + + def _limit(): + # ⚠️ **주소공간 상한.** 폭주하면 이 프로세스만 MemoryError 로 죽고 VM 은 산다. + # 위키 `wsl-계속-터짐` 의 인덱서 대책(`ulimit -v`)과 같은 수법이다. + import resource + n = MAX_SUBPROC_GB * (1 << 30) + try: + resource.setrlimit(resource.RLIMIT_AS, (n, n)) + except (ValueError, OSError): + pass + + try: + return subprocess.run(cmd, cwd=cwd, env=e, capture_output=True, + text=True, timeout=timeout, preexec_fn=_limit) + except subprocess.TimeoutExpired: + return subprocess.CompletedProcess(cmd, 124, "", "timeout") + + +def _last_error(text): + lines = [l.strip() for l in (text or "").splitlines() if l.strip()] + for l in reversed(lines): + if "Error" in l or "error" in l or "assert" in l.lower(): + return l + return lines[-1] if lines else "" + + +REF = r''' +import os, sys, numpy as np, torch +sys.path.insert(0, "%(FE)s"); sys.path.insert(0, "%(FE_DET)s") +import mmpose_wrap +IN = mmpose_wrap.input_shape("%(CFG)s") # 이미지면 (1,3,H,W), pose-lift 면 (1,C,1) +m, shapes = mmpose_wrap.build("%(CFG)s", "%(CK)s", IN) +torch.save(m, "pose.pt") +open("shape.txt", "w").write(",".join(str(v) for v in IN)) +H, W = (IN[2], IN[3]) if len(IN) == 4 else (1, 1) +open("size.txt", "w").write(f"{W} {H}") +# 고정 시드 — 시드 없이 뽑으면 실행마다 숫자가 흔들려 회귀 대조가 흐려진다. +g = torch.Generator().manual_seed(0) +x = torch.randn(*IN, generator=g) +with torch.no_grad(): + outs = m(x) +if isinstance(outs, torch.Tensor): + outs = (outs,) +sh = [] +for i, t in enumerate(outs): + a = t[0].detach().numpy().astype("float32") + np.ascontiguousarray(a).tofile("ref.out.%%d.bin" %% i) + sh.append(list(a.shape)) +open("ref.shapes.txt", "w").write("\n".join(" ".join(map(str, s)) for s in sh)) +if len(IN) == 4: + np.ascontiguousarray(x[0].numpy().transpose(1, 2, 0)).tofile("in.bin") # 러너 입력(cwhn) +else: + # 이미지가 아닌 입력(pose-lift)은 그대로 낸다. ggml ne=[C,1,1,1] 로 먹이고 + # 생성 코드의 cwhn→whcn 변환이 [1,1,C,1] 로 돌려놓는다. + np.ascontiguousarray(x[0].numpy().astype("float32")).tofile("in.bin") +print("REF_OK", len(outs), W, H) +''' + + +def verify(fam, cfg, weights): + import shutil + d = os.path.join(WORKDIR, fam) + os.makedirs(d, exist_ok=True) + ck = os.path.join(CKPT, os.path.basename(weights)) if weights else "" + if not ck or not os.path.exists(ck): + return fam, "CKPT_NONE", "체크포인트 미다운로드 (--fetch 로 받는다)" + if not os.path.exists(cfg): + return fam, "CONFIG_NONE", os.path.basename(cfg) + + _wait_for_memory(fam) + for stale in ("pose.pt", "ref.shapes.txt", "size.txt", "shape.txt"): + try: + os.remove(os.path.join(d, stale)) + except FileNotFoundError: + pass + + open(os.path.join(d, "ref.py"), "w").write( + REF % {"FE": FE, "FE_DET": FE_DET, "CFG": cfg, "CK": ck}) + r = run([PY, "ref.py"], d, {"PYTHONPATH": f"{FE}:{FE_DET}"}) + if not os.path.exists(os.path.join(d, "ref.shapes.txt")): + return fam, "REF_FAIL", _last_error(r.stderr)[:70] + W, H = [int(v) for v in open(os.path.join(d, "size.txt")).read().split()] + in_shape = [int(v) for v in + open(os.path.join(d, "shape.txt")).read().split(",")] + # 이미지가 아닌 입력이면 러너에 ne 를 직접 준다(`VISP_IN_NE`). cwhn = [C,W,H,N]. + run_env = ({} if len(in_shape) == 4 + else {"VISP_IN_NE": f"{in_shape[1]},1,1,1"}) + + gen = os.path.join(d, "out") + r = run([PY, "-c", ''' +import sys +sys.argv = ["g2c","--model","pose.pt","--name","Pose","--output","out","--input-shape","%s"] +from shared.compile.pipeline import main; main() +''' % ",".join(str(v) for v in in_shape)], d, {"PYTHONPATH": f"{FE}:{FE_DET}:{P}:{GGUF_PY}"}) + if not os.path.exists(os.path.join(gen, "Pose.gguf")): + return fam, "COMPILE_FAIL", _last_error(r.stderr)[:70] + n_unhandled = open(os.path.join(gen, "Pose.cpp")).read().count("unhandled op") + + inc = os.path.join(gen, "inc", "visp", "arch") + os.makedirs(inc, exist_ok=True) + shutil.copy(os.path.join(gen, "Pose.h"), inc) + b = run(["g++", "-std=c++20", OPT, "-DARCH=Pose", + '-DVISP_ARCH_HEADER="visp/arch/Pose.h"', + "-I" + os.path.join(gen, "inc"), "-I" + V + "/src", "-I" + V + "/include", + "-I" + V + "/depend/llama/ggml/include", + V + "/tools/verify/common/run_dump.cpp", os.path.join(gen, "Pose.cpp"), + "-L" + V + "/build/lib", "-lvisioncpp", "-lggml", "-lggml-base", "-lggml-cpu", + "-Wl,-rpath," + V + "/build/lib", "-o", os.path.join(gen, "run_dump")], d) + if not os.path.exists(os.path.join(gen, "run_dump")): + return fam, "BUILD_FAIL", _last_error(b.stderr)[:70] + + for f in os.listdir(d): + if f.startswith("cpp.out.") and f.endswith(".bin"): + os.remove(os.path.join(d, f)) + x = run([os.path.join(gen, "run_dump"), os.path.join(gen, "Pose.gguf"), + os.path.join(d, "in.bin"), os.path.join(d, "cpp"), str(W), str(H)], + d, run_env) + + import numpy as np + shapes = [[int(v) for v in ln.split()] + for ln in open(os.path.join(d, "ref.shapes.txt")).read().splitlines() + if ln.strip()] + worst_l1 = worst_l2 = 0.0 + for i, sh in enumerate(shapes): + pr, pc = os.path.join(d, f"ref.out.{i}.bin"), os.path.join(d, f"cpp.out.{i}.bin") + if not os.path.exists(pc): + return fam, "RUN_FAIL", (_last_error(x.stderr) or f"러너가 out_{i} 를 안 냈다")[:70] + a = np.fromfile(pr, dtype="float32") + c = np.fromfile(pc, dtype="float32") + if a.size != c.size: + return fam, "SHAPE_MISMATCH", f"out_{i}: torch {a.size} vs 러너 {c.size}" + # torch CHW ↔ 러너 HWC. 안 맞추면 값이 아니라 배치가 어긋난다. + # 러너는 언제나 **cwhn**(메모리 순서 n,h,w,c)으로 낸다. torch 쪽(배치를 벗긴) + # 축을 앞에서부터 (n,c,h,w) 로 읽어 같은 자로 맞춘다. + if len(sh) == 3: + ch, h, w = sh + c = c.reshape(h, w, ch).transpose(2, 0, 1).reshape(-1) + elif len(sh) == 4: + # ⚠️ **3D 히트맵 head 가 여기 온다.** InterNet 은 torch [1,42,64,64,48] + # (= N,C,D,H,W)를 내고 배치를 벗기면 4D 다. 러너 쪽 ne 는 + # [64,48,64,42] = (c=D, w=W, h=H, n=C) 라 **D 가 채널 자리**다. + # 안 맞추면 첫 원소만 맞고 rel L1 이 1.76 으로 뜬다 — 값이 아니라 배치다. + n, ch, h, w = sh + c = c.reshape(n, h, w, ch).transpose(0, 3, 1, 2).reshape(-1) + den = max(float(np.abs(a).sum()), 1e-9) + worst_l1 = max(worst_l1, float(np.abs(a - c).sum()) / den) + worst_l2 = max(worst_l2, float(np.linalg.norm(a - c)) + / max(float(np.linalg.norm(a)), 1e-9)) + ok = worst_l1 < L1_TOL and worst_l2 < L2_TOL + dims = f"{W}x{H}" if len(in_shape) == 4 else "x".join(str(v) for v in in_shape) + note = f"L1 {worst_l1:.2e} · L2 {worst_l2:.2e} · {dims} · 출력 {len(shapes)}" + if n_unhandled: + note += f" · ⚠ unhandled op {n_unhandled}" + return fam, "PASS" if ok else "FAIL", note + + +def fetch(sel): + import urllib.request + os.makedirs(CKPT, exist_ok=True) + for fam, _cfg, w in sel: + if not w: + continue + dst = os.path.join(CKPT, os.path.basename(w)) + if os.path.exists(dst): + continue + print(f" 받는 중 {fam}: {os.path.basename(w)}", flush=True) + try: + urllib.request.urlretrieve(w, dst + ".part") + os.replace(dst + ".part", dst) + except Exception as e: + print(f" 실패 {fam}: {type(e).__name__}: {e}", flush=True) + + +def main(): + ap = argparse.ArgumentParser(prog="verify_pose") + ap.add_argument("families", nargs="*") + ap.add_argument("--list", action="store_true") + ap.add_argument("--fetch", action="store_true") + ap.add_argument("--workdir", default=None) + a = ap.parse_args() + global WORKDIR + if a.workdir: + WORKDIR = a.workdir + + all_fams = families() + sel = [f for f in all_fams if not a.families or f[0] in a.families] + if a.fetch: + fetch(sel) + if a.list: + for fam, cfg, w in all_fams: + has = os.path.exists(os.path.join(CKPT, os.path.basename(w))) if w else False + print(f"{'O' if has else '-'} {fam:24s} {os.path.basename(cfg)}") + print(f"\n계열 {len(all_fams)}") + return + + print(f"mmpose={MM} g2c={P} workdir={WORKDIR} tol=L1{L1_TOL}/L2{L2_TOL}") + print(f"{'계열':<26}{'판정':<14}비고") + print("-" * 92) + rows = [] + for i, (fam, cfg, w) in enumerate(sel, 1): + fam, verdict, note = verify(fam, cfg, w) + mark = "O" if verdict == "PASS" else ("X" if verdict == "FAIL" else "-") + print(f"[{i:3d}/{len(sel)}] {fam:<22} {mark} {verdict:<14} {note}") + rows.append((fam, verdict)) + print(f"\nPASS {sum(1 for _f, v in rows if v == 'PASS')}/{len(rows)}") + + +if __name__ == "__main__": + main() diff --git a/tools/verify/mmseg/mmseg_families.py b/tools/verify/mmseg/mmseg_families.py new file mode 100644 index 0000000..600e38b --- /dev/null +++ b/tools/verify/mmseg/mmseg_families.py @@ -0,0 +1,38 @@ +"""mmseg_families.py — 계열 → (대표 config, 체크포인트 URL) 의 **예외 목록**. + +기본은 `configs/<계열>/metafile.yaml` 의 첫 모델이다(`verify_seg.families()`). +손목록을 정본으로 쓰면 거기 없는 계열이 존재하지 않는 것처럼 보이므로 **열거는 그대로 +metafile 이 하고, 여기서는 틀린 줄만 덮어쓴다.** mmdet 쪽 +`tools/verify/mmdet/dense_head/mmdet_families.py` 의 `_OVERRIDE_LIST` 와 같은 구조다. + +⚠️ **저장소(`~/mmbuild/mmsegmentation`)의 metafile 을 직접 고치지 마라.** vendor 트리라 + 재클론하면 날아가고 왜 고쳤는지도 안 남는다. 예외는 전부 이 파일에 적는다. +""" + +# (계열, config 상대경로, 체크포인트 URL 또는 "" = metafile 것을 그대로 쓴다) +_OVERRIDE_LIST = [ + # ⚠️ metafile 오타. `configs/emanet/metafile.yaml:22` 가 + # `eemanet_r50-d8_4xb2-80k_cityscapes-512x1024.py`(맨 앞 `e` 가 둘)를 가리킨다. + # 실제 파일은 `emanet_r50-...` 로 **존재한다** — 「범위 밖」이 아니라 오타 하나로 + # `CONFIG_NONE` 이 되던 것이다(2026-08-31 확인, mmsegmentation main). + # 체크포인트 URL 은 metafile 것이 맞으므로 덮어쓰지 않는다. + ("emanet", "emanet/emanet_r50-d8_4xb2-80k_cityscapes-512x1024.py", ""), +] + +OVERRIDE = {f: (c, w) for f, c, w in _OVERRIDE_LIST} + + +def apply(fam, cfg, weights, configs_root): + """metafile 이 준 (cfg, weights) 에 예외를 얹어 돌려준다. + + `configs_root` 는 `/configs`. 예외가 없으면 받은 것을 그대로 낸다. + """ + import os + if fam not in OVERRIDE: + return cfg, weights + c, w = OVERRIDE[fam] + if c: + cfg = os.path.join(configs_root, c) + if w: + weights = w + return cfg, weights diff --git a/tools/verify/mmseg/probe_seg.py b/tools/verify/mmseg/probe_seg.py new file mode 100644 index 0000000..16c8ddc --- /dev/null +++ b/tools/verify/mmseg/probe_seg.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""mmseg 한 계열을 **층별로 갈라** 컴파일한다 — 값이 어디서 벌어지는지 찾는 도구. + +`verify_seg.py` 는 최종 출력 하나만 대조해 PASS/FAIL 을 낸다. FAIL 이 났을 때 +**어느 층에서 벌어졌는지**는 안 알려준다. 이 도구는 백본 4단계와 head 내부 단계를 +**한 그래프의 출력으로 같이 내보내** 단계마다 torch 와 댄다. + + python probe_seg.py fcn --stage backbone # 백본 4단계 + python probe_seg.py fcn --stage head # head 입력 + 내부 + cls_seg + python probe_seg.py fcn --stage final # 최종 seg_logits (verify_seg 와 같다) + +계열·체크포인트·입력 크기는 `verify_seg.py` 와 **같은 곳에서** 온다(metafile · crop_size). +따로 적어 두면 갈린다. + +⚠️ **head 를 두 번 부르지 마라.** 중간값을 꺼내려고 `h.convs(x)` 로 펼쳐 부른 뒤 + `h(f)` 도 부르면 같은 서브모듈이 그래프에 **두 번** 들어가 이름이 갈리고 + (`Sequential[decode_head]` vs `FCNHead[decode_head]`) 가중치 바인딩이 어긋난다. + 2026-08-27 에 그것 때문에 rel L1 1.94 라는 **가짜 실패**를 만들었다. + 아래 `_head_stages` 는 head.forward 를 그대로 펼친 것이고, 끝에서 다시 부르지 않는다. + +⚠️ **랜덤 초기화로 재지 마라** — `verify_seg.py` 와 같은 이유다. 항등 초기값이 + 빠진 연산을 덮는다. 체크포인트가 없으면 그 계열은 돌리지 않는다. + +status: 2026-08-27 `~/work/seg/seg_probe.py`(추적 안 되던 중복본)에서 저장소로 옮기며 + 512² 고정 → 계열별 crop, 손목록 3계열 → metafile 열거로 바꿨다. + **그 두 변경 뒤 아직 안 돌려봤다 — 첫 사용자는 결과를 의심하고 보라.** +""" +import argparse +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +V = os.path.abspath(os.path.join(HERE, "..", "..", "..")) # vision.cpp +P = os.path.abspath(os.path.join(V, "..")) # g2c 루트 +sys.path.insert(0, HERE) + +import verify_seg as VS # noqa: E402 + + +def _head_stages(h, feats): + """decode_head 를 펼쳐 단계별 텐서를 모은다. head 는 **한 번만** 탄다.""" + import torch + xin = feats[h.in_index] if isinstance(h.in_index, int) else \ + torch.cat([feats[i] for i in h.in_index], dim=1) + outs = [xin] # head 입력 (백본 마지막) + if hasattr(h, "psp_modules"): # PSPHead / PSPNet 계열 + # ⚠️ `PPM.forward` 가 pool → conv → **resize(입력 크기로 업샘플)** 까지 한다. + # 가지를 따로 부르면 (1,1)·(2,2)… 인 채로 나와 concat 이 깨진다. + ppm = list(h.psp_modules(xin)) + outs.extend(ppm) + y = h.bottleneck(torch.cat([xin] + ppm, dim=1)) + outs.append(y) + elif hasattr(h, "convs"): # FCNHead 계열 + y = h.convs(xin) + outs.append(y) + if getattr(h, "concat_input", False): + y = h.conv_cat(torch.cat([xin, y], dim=1)) + outs.append(y) + else: + return None # 펼치는 법을 모르는 head + outs.append(h.conv_seg(y)) # dropout 은 eval 에서 항등 + return outs + + +REF = r''' +import os, sys, numpy as np, torch +sys.path.insert(0, "%(FE)s"); sys.path.insert(0, "%(FE_DET)s"); sys.path.insert(0, "%(HERE)s") +import mmseg_wrap +from probe_seg import _head_stages +from mmseg.apis import init_model +import torch.nn as nn + +H, W = mmseg_wrap.crop_size("%(CFG)s") +mmseg_wrap.trace_friendly_ops() +import mmdet_wrap; mmdet_wrap.allow_mmengine_checkpoint_globals() +seg = init_model("%(CFG)s", "%(CK)s", device="cpu"); seg.eval() + + +class Probe(nn.Module): + def __init__(s, seg, stage): + super().__init__() + s.backbone = seg.backbone + s.neck = seg.neck if getattr(seg, "with_neck", False) else None + s.decode_head = seg.decode_head + s.stage = stage + + def forward(s, x): + f = s.backbone(x) + if s.neck is not None: + f = s.neck(f) + if s.stage == "backbone": + return tuple(f) + if s.stage == "final": + o = s.decode_head(f) + return o if isinstance(o, tuple) else (o,) + st = _head_stages(s.decode_head, f) + if st is None: + raise SystemExit("HEAD_UNKNOWN: 이 head 는 펼치는 법을 모른다 — --stage final 로 재라") + return tuple(st) + + +m = Probe(seg, "%(STAGE)s").eval() +torch.save(m, "seg.pt") +open("size.txt", "w").write(f"{W} {H}") +g = torch.Generator().manual_seed(0) # verify_seg 와 같은 고정 입력 +x = torch.randn(1, 3, H, W, generator=g) +with torch.no_grad(): + outs = m(x) +sh = [] +for i, t in enumerate(outs): + a = t[0].detach().numpy() + np.ascontiguousarray(a).tofile(f"ref.out.{i}.bin") + sh.append(list(a.shape)) +np.ascontiguousarray(x[0].numpy().transpose(1, 2, 0)).tofile("in.bin") +open("ref.shapes.txt", "w").write("\n".join(" ".join(map(str, s)) for s in sh)) +print("[ref]", sh, flush=True) +''' + + +def main(): + ap = argparse.ArgumentParser(prog="probe_seg") + ap.add_argument("family") + ap.add_argument("--stage", default="backbone", + choices=["backbone", "head", "final"]) + ap.add_argument("--workdir", default="/tmp/visp-seg-probe") + a = ap.parse_args() + + fams = {f[0]: f for f in VS.families()} + if a.family not in fams: + sys.exit(f"모르는 계열: {a.family} (verify_seg.py --list 로 확인)") + fam, cfg, w = fams[a.family] + ck = os.path.join(VS.CKPT, os.path.basename(w)) if w else "" + if not ck or not os.path.exists(ck): + sys.exit(f"{fam}: 체크포인트가 없다 — 랜덤으로 재면 안 된다 " + f"(verify_seg.py --fetch {fam})") + + d = os.path.join(a.workdir, f"{fam}-{a.stage}") + os.makedirs(d, exist_ok=True) + src = os.path.join(d, "_ref.py") + open(src, "w").write(REF % {"FE": VS.FE, "FE_DET": VS.FE_DET, "HERE": HERE, + "CFG": cfg, "CK": ck, "STAGE": a.stage}) + print(f"{fam} · stage={a.stage} · workdir={d}") + import subprocess + r = subprocess.run([VS.PY, src], cwd=d) + if r.returncode: + sys.exit(r.returncode) + print("\n기준값을 냈다. 컴파일·대조는 verify_seg.py 의 경로를 그대로 쓴다 —\n" + f" {d}/seg.pt · ref.out.*.bin · size.txt") + + +if __name__ == "__main__": + main() diff --git a/tools/verify/mmseg/verify_seg.py b/tools/verify/mmseg/verify_seg.py new file mode 100644 index 0000000..3db955c --- /dev/null +++ b/tools/verify/mmseg/verify_seg.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python3 +"""mmseg 계열을 g2c 로 컴파일해 C++(ggml) 출력이 PyTorch 와 같은지 계열별로 잰다. + +mmdet 하네스(`tools/verify/mmdet/dense_head/verify_heads.py`)와 **재는 방식이 같고 훨씬 짧다** — +세그멘테이션은 앵커도 NMS 도 박스 디코드도 없어서 `backbone → (neck) → decode_head` 를 +한 그래프로 컴파일하고 **출력 텐서를 그대로 대조**하면 끝난다. + +점수는 **상대 L1/L2** 다. cosine 은 쓰지 않는다 — 스케일 불변이라 크기가 통째로 틀려도 +1.0 이 나온다. + + python verify_seg.py # metafile 에 있는 계열 전부 + python verify_seg.py fcn pspnet # 골라서 + python verify_seg.py --list # 목록만 + +⚠️ **랜덤 초기화로 재지 마라.** 항등 초기값(BN γ=1·β=0)이 빠진 연산을 덮는다. + 체크포인트가 없으면 그 계열은 CKPT_NONE 으로 남긴다 — **못 잰 것이지 통과가 아니다.** +""" +import argparse +import os +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import mmseg_families + +sys.stdout.reconfigure(line_buffering=True) + +HERE = os.path.dirname(os.path.abspath(__file__)) +V = os.path.abspath(os.path.join(HERE, "..", "..", "..")) # vision.cpp +P = os.path.abspath(os.path.join(V, "..")) # g2c 루트 +FE = V + "/tools/frontend/mmseg" +FE_DET = V + "/tools/frontend/mmdet" +GGUF_PY = V + "/depend/llama/gguf-py" +PY = sys.executable + +MM = os.path.expanduser("~/mmbuild/mmsegmentation") +CKPT = MM + "/checkpoints" +WORKDIR = "/tmp/visp-seg-verify" +SZ = 0 # 0 = config 의 crop 을 쓴다. `--size` 로만 덮어쓴다 +OPT = "-O1" +# 판정 기준: 출력 텐서의 상대 L1/L2. 저장소 관례(mmdet 하네스)와 같은 값을 쓴다. +L1_TOL = L2_TOL = 0.05 + + +def families(): + """`configs/*/metafile.yaml` 에서 (계열, config, weights URL) 을 읽는다. + + **손으로 적지 않는다** — 목록을 손으로 적으면 거기 없는 계열이 존재하지 않는 것처럼 + 보인다(mmdet 에서 `pisa`·`rpn` 이 실제로 그렇게 몇 주 동안 안 보였다). + 계열마다 metafile 의 **첫 모델**을 대표로 쓴다. + """ + import glob + import yaml + out = [] + for p in sorted(glob.glob(MM + "/configs/*/metafile.y*ml")): + fam = os.path.basename(os.path.dirname(p)) + try: + models = yaml.safe_load(open(p)).get("Models") or [] + except Exception: + continue + if not models: + continue + m = models[0] + cfg = MM + "/" + m["Config"] if not m["Config"].startswith("/") else m["Config"] + w = m.get("Weights", "") + # ⚠️ metafile 이 틀린 줄이 있다(emanet 의 config 이름 오타). vendor 트리를 고치면 + # 재클론에 날아가므로 **예외는 저장소 안 `mmseg_families.py` 에 적고** 여기서 + # 얹는다. 열거 자체는 그대로 metafile 이 한다 — 손목록은 정본이 아니다. + cfg, w = mmseg_families.apply(fam, cfg, w, MM + "/configs") + out.append((fam, cfg, w)) + return out + + +# ⚠️ **메모리 가드.** 위키 `wsl-계속-터짐` 5번째 원인 — 폭주한 프로세스가 없어도 **합**이 +# 넘치면 WSL 이 통째로 죽는다(2026-08-27 실측: 내 스윕 3.6GB + 다른 python 2.25GB + +# VS Code 3.4GB + claude 세션 5개 1.5GB = 13GB 상한 초과 → OOM 8건). +# 가용 메모리가 이 밑이면 **새 계열을 안 띄우고 기다린다.** 느려질지언정 안 죽는다. +MIN_FREE_MB = int(os.environ.get("VISP_MIN_FREE_MB", "2500")) +# 한 서브프로세스가 이 이상 잡으면 **그놈만** 죽는다(VM 이 아니라). 위키의 인덱서 대책과 같다. +MAX_SUBPROC_GB = int(os.environ.get("VISP_MAX_SUBPROC_GB", "8")) + + +def _avail_mb(): + try: + for line in open("/proc/meminfo"): + if line.startswith("MemAvailable:"): + return int(line.split()[1]) // 1024 + except Exception: + pass + return 1 << 30 # 못 읽으면 가드를 끈다(측정 실패로 막지 않는다) + + +_MEM_GATE = __import__("threading").Lock() + + +def _wait_for_memory(fam): + """가용 메모리가 회복될 때까지 기다린다. + + ⚠️ **락으로 감싼다.** 워커가 여럿이면 둘이 동시에 검사를 통과한 뒤 **둘 다** 할당해 + 가드가 무의미해진다. 한 번에 하나만 문을 지나게 하고, 지난 뒤엔 바로 놓는다 + (할당은 서브프로세스가 하므로 그때까지 붙들 필요가 없다). + """ + import time + waited = 0 + with _MEM_GATE: + while _avail_mb() < MIN_FREE_MB: + if waited == 0: + print(f" … 메모리 대기 ({fam}): 가용 {_avail_mb()}MB < {MIN_FREE_MB}MB", + flush=True) + time.sleep(5) + waited += 5 + if waited > 600: # 10분을 기다려도 안 풀리면 그냥 간다(교착 방지) + print(f" … 10분 대기 후에도 부족 — 그대로 진행한다 ({fam})", flush=True) + break + + +def run(cmd, cwd=None, env=None, timeout=1800): + e = dict(os.environ) + e.setdefault("OMP_NUM_THREADS", "1") + # glibc 이 스레드마다 arena 를 잡아 RSS 가 부푼다. torch 서브프로세스에서 크게 온다. + e.setdefault("MALLOC_ARENA_MAX", "2") + if env: + e.update(env) + + def _limit(): + # ⚠️ **주소공간 상한.** 폭주하면 이 프로세스만 MemoryError 로 죽고 VM 은 산다. + # 위키 `wsl-계속-터짐` 의 인덱서 대책(`ulimit -v`)과 같은 수법이다. + import resource + n = MAX_SUBPROC_GB * (1 << 30) + try: + resource.setrlimit(resource.RLIMIT_AS, (n, n)) + except (ValueError, OSError): + pass + + try: + return subprocess.run(cmd, cwd=cwd, env=e, capture_output=True, + text=True, timeout=timeout, preexec_fn=_limit) + except subprocess.TimeoutExpired: + return subprocess.CompletedProcess(cmd, 124, "", "timeout") + + +def _last_error(text): + lines = [l.strip() for l in (text or "").splitlines() if l.strip()] + for l in reversed(lines): + if "Error" in l or "error" in l or "assert" in l.lower(): + return l + return lines[-1] if lines else "" + + +REF = r''' +import os, sys, numpy as np, torch +sys.path.insert(0, "%(FE)s"); sys.path.insert(0, "%(FE_DET)s") +import mmseg_wrap +# ⚠️ **입력 크기를 하나로 고정하지 않는다.** config 의 crop 이 곧 그 모델이 보는 크기다 +# (512x1024 · 640x640 · 680x680 · 1024x1024 …). 고정하면 잰 것이 그 모델이 아니고, +# ViT 계열은 `pos_embed` 토큰 수가 안 맞아 아예 죽는다. +# ⚠️ 기본은 **config 의 crop** 이다. `--size` 는 진단용 덮어쓰기다(크기 탓인지 가를 때). +# 2026-08-27 까지 `--size` 는 설정만 되고 **아무 데도 안 쓰였다** — 헤더에는 512 라 +# 찍히는데 실제로는 crop 으로 쟀다. 거짓말하는 플래그였다. +_ov = "%(SIZE)s" +H, W = ([int(_ov), int(_ov)] if _ov else mmseg_wrap.crop_size("%(CFG)s")) +m, shapes = mmseg_wrap.build("%(CFG)s", "%(CK)s", (H, W)) +torch.save(m, "seg.pt") +open("size.txt", "w").write(f"{W} {H}") +# ⚠️ **고정 입력**을 쓴다. 시드 없이 뽑으면 실행마다 숫자가 1~5%% 흔들려 회귀 대조가 +# 흐려진다(mmdet 의 no-box 계열에서 실제로 겪었다). +g = torch.Generator().manual_seed(0) +x = torch.randn(1, 3, H, W, generator=g) +with torch.no_grad(): + outs = m(x) +if isinstance(outs, torch.Tensor): + outs = (outs,) +sh = [] +for i, t in enumerate(outs): + a = t[0].detach().numpy().astype("float32") + np.ascontiguousarray(a).tofile("ref.out.%%d.bin" %% i) + sh.append(list(a.shape)) +open("ref.shapes.txt", "w").write("\n".join(" ".join(map(str, s)) for s in sh)) +np.ascontiguousarray(x[0].numpy().transpose(1, 2, 0)).tofile("in.bin") # 러너 입력(cwhn) +print("REF_OK", len(outs)) +''' + + + +def _phase(fam, name): + """계열 안의 **단계**를 찍는다. + + 하네스는 원래 계열이 끝나야 한 줄을 냈다. 계열 하나가 10~20분이라 + 그동안 `tail -f` 가 조용해서 「멈춘 것」과 구분이 안 됐다. + `VISP_SEG_PHASE=0` 으로 끌 수 있다(회귀 로그를 옛 판과 줄 단위로 대조할 때). + """ + if os.environ.get("VISP_SEG_PHASE", "1") != "0": + print(f" · {fam}: {name}", flush=True) + + +def verify(fam, cfg, weights): + import shutil + d = os.path.join(WORKDIR, fam) + os.makedirs(d, exist_ok=True) + ck = os.path.join(CKPT, os.path.basename(weights)) if weights else "" + if not ck or not os.path.exists(ck): + return fam, "CKPT_NONE", "체크포인트 미다운로드 (--fetch 로 받는다)" + if not os.path.exists(cfg): + return fam, "CONFIG_NONE", os.path.basename(cfg) + + # ⚠️ **먼저 지운다.** 지난 실행의 산출물이 남아 있으면 export 가 실패해도 아래 존재 + # 검사를 통과해 **낡은 것으로 계속 간다**(mmdet 하네스에서 실제로 겪었다). + _wait_for_memory(fam) + for stale in ("seg.pt", "ref.shapes.txt", "size.txt"): + try: + os.remove(os.path.join(d, stale)) + except FileNotFoundError: + pass + + open(os.path.join(d, "ref.py"), "w").write( + REF % {"SIZE": (str(SZ) if SZ else ""), "FE": FE, "FE_DET": FE_DET, "CFG": cfg, "CK": ck}) + _phase(fam, "ref 생성 (torch 기준값)") + r = run([PY, "ref.py"], d, {"PYTHONPATH": f"{FE}:{FE_DET}"}) + if not os.path.exists(os.path.join(d, "ref.shapes.txt")): + return fam, "REF_FAIL", _last_error(r.stderr)[:70] + W, H = [int(v) for v in open(os.path.join(d, "size.txt")).read().split()] + + _phase(fam, "컴파일 (g2c → .cpp/.gguf)") + gen = os.path.join(d, "out") + r = run([PY, "-c", ''' +import sys +sys.argv = ["g2c","--model","seg.pt","--name","Seg","--output","out","--input-shape","1,3,%d,%d"] +from shared.compile.pipeline import main; main() +''' % (H, W)], d, {"PYTHONPATH": f"{FE}:{FE_DET}:{P}:{GGUF_PY}"}) + if not os.path.exists(os.path.join(gen, "Seg.gguf")): + return fam, "COMPILE_FAIL", _last_error(r.stderr)[:70] + src = open(os.path.join(gen, "Seg.cpp")).read() + n_unhandled = src.count("unhandled op") + + inc = os.path.join(gen, "inc", "visp", "arch") + os.makedirs(inc, exist_ok=True) + shutil.copy(os.path.join(gen, "Seg.h"), inc) + _phase(fam, "빌드 (g++)") + b = run(["g++", "-std=c++20", OPT, "-DARCH=Seg", + '-DVISP_ARCH_HEADER="visp/arch/Seg.h"', + "-I" + os.path.join(gen, "inc"), "-I" + V + "/src", "-I" + V + "/include", + "-I" + V + "/depend/llama/ggml/include", + V + "/tools/verify/common/run_dump.cpp", os.path.join(gen, "Seg.cpp"), + "-L" + V + "/build/lib", "-lvisioncpp", "-lggml", "-lggml-base", "-lggml-cpu", + "-Wl,-rpath," + V + "/build/lib", "-o", os.path.join(gen, "run_dump")], d) + if not os.path.exists(os.path.join(gen, "run_dump")): + return fam, "BUILD_FAIL", _last_error(b.stderr)[:70] + + for f in os.listdir(d): + if f.startswith("cpp.out.") and f.endswith(".bin"): + os.remove(os.path.join(d, f)) + _phase(fam, "실행 (ggml)") + x = run([os.path.join(gen, "run_dump"), os.path.join(gen, "Seg.gguf"), + os.path.join(d, "in.bin"), os.path.join(d, "cpp"), str(W), str(H)], d) + + import numpy as np + shapes = [[int(v) for v in ln.split()] + for ln in open(os.path.join(d, "ref.shapes.txt")).read().splitlines() + if ln.strip()] + worst_l1 = worst_l2 = 0.0 + for i, sh in enumerate(shapes): + pr, pc = os.path.join(d, f"ref.out.{i}.bin"), os.path.join(d, f"cpp.out.{i}.bin") + if not os.path.exists(pc): + return fam, "RUN_FAIL", (_last_error(x.stderr) or f"러너가 out_{i} 를 안 냈다")[:70] + a = np.fromfile(pr, dtype="float32") + c = np.fromfile(pc, dtype="float32") + if a.size != c.size: + return fam, "SHAPE_MISMATCH", f"out_{i}: torch {a.size} vs 러너 {c.size}" + # ⚠️ **축 순서가 다르다.** torch 는 CHW, 러너(ggml)는 HWC 로 쓴다. 안 맞추면 값이 + # 아니라 배치가 어긋나 rel L1 이 1.4~2.0(= 무관한 두 텐서)으로 나온다. + if len(sh) == 3: + ch, h, w = sh + c = c.reshape(h, w, ch).transpose(2, 0, 1).reshape(-1) + den = max(float(np.abs(a).sum()), 1e-9) + worst_l1 = max(worst_l1, float(np.abs(a - c).sum()) / den) + worst_l2 = max(worst_l2, float(np.linalg.norm(a - c)) + / max(float(np.linalg.norm(a)), 1e-9)) + ok = worst_l1 < L1_TOL and worst_l2 < L2_TOL + note = f"L1 {worst_l1:.2e} · L2 {worst_l2:.2e} · {W}x{H} · 출력 {len(shapes)}" + # ⚠️ unhandled op 이 있으면 **통과해도 통과가 아니다** — 그 연산이 사라진 채 우연히 + # 수치가 맞은 것일 수 있다. 판정 옆에 반드시 남긴다. + if n_unhandled: + note += f" · ⚠ unhandled op {n_unhandled}" + return fam, "PASS" if ok else "FAIL", note + + +def fetch(sel): + import urllib.request + os.makedirs(CKPT, exist_ok=True) + for fam, _cfg, w in sel: + if not w: + continue + dst = os.path.join(CKPT, os.path.basename(w)) + if os.path.exists(dst): + continue + print(f" 받는 중 {fam}: {os.path.basename(w)}", flush=True) + try: + urllib.request.urlretrieve(w, dst + ".part") + os.replace(dst + ".part", dst) + except Exception as e: + print(f" 실패 {fam}: {type(e).__name__}: {e}", flush=True) + + +def main(): + ap = argparse.ArgumentParser(prog="verify_seg") + ap.add_argument("families", nargs="*") + ap.add_argument("--list", action="store_true", help="계열 목록만 찍는다") + ap.add_argument("--fetch", action="store_true", help="없는 체크포인트를 받는다") + ap.add_argument("--size", type=int, default=None) + ap.add_argument("--workdir", default=None) + ap.add_argument("--workers", type=int, default=2, + help="동시에 돌릴 계열 수(기본 2). ⚠️ 계열마다 torch 를 띄운다 — " + "12GB 에서 3을 넘기면 위험하다. 각 서브프로세스는 " + f"`ulimit -v {MAX_SUBPROC_GB}GB` 로 묶여 있어 폭주해도 " + "그 계열만 죽는다.") + a = ap.parse_args() + + global SZ, WORKDIR + SZ = a.size or 0 # 0 = 계열별 crop (기본) + if a.workdir: + WORKDIR = a.workdir + + all_fams = families() + sel = [f for f in all_fams if not a.families or f[0] in a.families] + if a.fetch: + fetch(sel) + if a.list: + for fam, cfg, w in all_fams: + has = os.path.exists(os.path.join(CKPT, os.path.basename(w))) if w else False + print(f"{'O' if has else '-'} {fam:22s} {os.path.basename(cfg)}") + print(f"\n계열 {len(all_fams)} · 체크포인트 보유 " + f"{sum(1 for _f, _c, w in all_fams if w and os.path.exists(os.path.join(CKPT, os.path.basename(w))))}") + return + + print(f"mmseg={MM} g2c={P} " + f"size={'계열별 crop' if not SZ else f'{SZ}² (--size 덮어쓰기)'} " + f"workdir={WORKDIR} tol=L1{L1_TOL}/L2{L2_TOL}") + print(f"{'계열':<24}{'판정':<14}비고") + print("-" * 86) + rows = [] + nw = max(1, int(a.workers)) + if nw == 1 or len(sel) == 1: + for i, (fam, cfg, w) in enumerate(sel, 1): + fam, verdict, note = verify(fam, cfg, w) + mark = "O" if verdict == "PASS" else ("X" if verdict == "FAIL" else "-") + print(f"[{i:3d}/{len(sel)}] {fam:<20} {mark} {verdict:<14} {note}") + rows.append((fam, verdict)) + else: + # 계열끼리 독립이라 병렬로 돌린다. 일은 전부 **서브프로세스**가 하므로 스레드로 + # 충분하다(GIL 은 subprocess 대기 중 풀린다). 계열당 52초 중 추론은 6초뿐이고 + # 나머지는 torch import·컴파일이라, 병렬화가 유일하게 큰 레버다. + # ⚠️ 완료 순서대로 찍는다 — 줄마다 계열 이름이 있어 순서가 섞여도 읽힌다. + from concurrent.futures import ThreadPoolExecutor, as_completed + idx = {f[0]: i for i, f in enumerate(sel, 1)} + done = 0 + with ThreadPoolExecutor(max_workers=nw) as ex: + futs = [ex.submit(verify, f, c, w) for f, c, w in sel] + for fu in as_completed(futs): + fam, verdict, note = fu.result() + mark = "O" if verdict == "PASS" else ("X" if verdict == "FAIL" else "-") + done += 1 + print(f"[{done:3d}/{len(sel)}] {fam:<20} {mark} {verdict:<14} {note}", + flush=True) + rows.append((fam, verdict)) + rows.sort(key=lambda r: idx.get(r[0], 0)) + n_pass = sum(1 for _f, v in rows if v == "PASS") + print(f"\nPASS {n_pass}/{len(rows)}") + + +if __name__ == "__main__": + main() diff --git a/tools/vision_install.py b/tools/vision_install.py new file mode 100755 index 0000000..03abd59 --- /dev/null +++ b/tools/vision_install.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""vision_install.py — g2c 생성물을 등록하고 vision.cpp 를 다시 빌드한다. + + vision-install ResNet + vision-install Seg --mean 123.675,116.28,103.53 --std 58.395,57.12,57.375 + +install_arch.py 로 등록한 뒤 CMake configure·build 를 수행하고, cmake --install 로 +/bin/vision-cli 를 갱신한다. 기본 prefix 는 ~/.local 이다. + +빌드가 필요한지는 여기서 판단하지 않는다. arch/*.cpp glob 에 CONFIGURE_DEPENDS 가 붙어 +있어 CMake 가 스스로 판단하고, 바뀐 것이 없으면 빌드 시스템이 넘어간다. 직접 판정하면 +바이너리는 있는데 새 모델이 들어가지 않은 상태를 놓친다. + +등록 옵션은 모델 폴더의 install.json 에 남는다. 값이 틀려도 오류 없이 결과만 달라지므로 +기록이 없으면 같은 값을 재현할 수 없다. 옵션을 생략하면 그 파일의 값을 사용한다. +""" +import argparse +import json +import os +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +VCPP = os.path.dirname(HERE) +INSTALL_ARCH = os.path.join(HERE, "install_arch.py") + + +def find_source(name, explicit): + """생성물 폴더를 찾는다. 기본 규칙은 output// 이다.""" + cands = [explicit] if explicit else [os.path.join("output", name), "output", "out"] + for d in cands: + if os.path.isfile(os.path.join(d, name + ".cpp")): + return d + sys.exit( + "error: {}.cpp 를 찾지 못했다. 찾아본 곳: {}\n" + " g2c 의 --output 경로를 --source 로 지정한다.".format(name, ", ".join(cands)) + ) + + +def load_saved_args(source): + path = os.path.join(source, "install.json") + if not os.path.isfile(path): + return [] + with open(path, encoding="utf-8") as f: + return json.load(f).get("install_args", []) + + +def save_args(source, extra): + with open(os.path.join(source, "install.json"), "w", encoding="utf-8") as f: + json.dump({"install_args": extra}, f, ensure_ascii=False, indent=2) + f.write("\n") + + +def cached_archs(build_dir): + """빌드 디렉터리에 선택 목록이 걸려 있으면 그 값을 돌려준다.""" + cache = os.path.join(build_dir, "CMakeCache.txt") + if not os.path.isfile(cache): + return None + for line in open(cache, encoding="utf-8", errors="replace"): + if line.startswith("VISP_ARCHS:"): + value = line.split("=", 1)[1].strip() + return [v for v in value.split(";") if v] or None + return None + + +def run(cmd): + print("+ " + " ".join(cmd), flush=True) + rc = subprocess.run(cmd).returncode + if rc != 0: + sys.exit(rc) + + +def main(argv=None): + ap = argparse.ArgumentParser( + prog="vision-install", + description="g2c 생성물을 vision.cpp 에 등록하고 다시 빌드한다.", + epilog="여기서 인식하지 않는 옵션은 install_arch.py 로 그대로 넘어간다 " + "(--mmdet, --mean, --std, --size 등).", + ) + ap.add_argument("name", help="아키텍처 이름. g2c 의 --name 값이다") + ap.add_argument("--source", default=None, help="생성물 폴더 (기본: output/)") + ap.add_argument("--vision-dir", default=VCPP, help="vision.cpp 체크아웃") + ap.add_argument("--build-dir", default=None, help="빌드 디렉터리 (기본: /build)") + ap.add_argument("--jobs", "-j", type=int, default=4, help="빌드 병렬 수") + ap.add_argument("--prefix", default=os.path.expanduser("~/.local"), + help="vision-cli 설치 위치 (기본: ~/.local). /bin/vision-cli 가 된다") + ap.add_argument("--no-build", action="store_true", help="등록만 하고 빌드하지 않는다") + ap.add_argument("--no-install", action="store_true", + help="빌드만 하고 설치하지 않는다. 빌드 트리의 실행 파일을 직접 사용한다") + args, extra = ap.parse_known_args(argv) + + build_dir = args.build_dir or os.path.join(args.vision_dir, "build") + source = find_source(args.name, args.source) + if not extra: + extra = load_saved_args(source) + + run([sys.executable, INSTALL_ARCH, source, "--name", args.name] + extra) + save_args(source, extra) + + if args.no_build: + print("\n등록만 수행했다. 빌드: cmake --build {} -j{}".format(build_dir, args.jobs)) + return + + selected = cached_archs(build_dir) + run(["cmake", "-S", args.vision_dir, "-B", build_dir]) + run(["cmake", "--build", build_dir, "-j{}".format(args.jobs)]) + + built = os.path.join(build_dir, "bin", "vision-cli") + print() + print("등록: " + os.path.join(args.vision_dir, "src/visp/arch", args.name + ".cpp")) + print("빌드: " + built) + + if not args.no_install: + # 실행 파일만 복사하면 libvisioncpp.so 를 빌드 트리 경로로 계속 찾는다. cmake --install + # 은 라이브러리를 함께 두고 RPATH 를 설치 위치 기준으로 다시 쓴다. + run(["cmake", "--install", build_dir, "--prefix", args.prefix]) + print("설치: " + os.path.join(args.prefix, "bin", "vision-cli")) + + if selected and args.name not in selected: + joined = ";".join(selected) + print() + print('주의: 이 빌드 디렉터리는 VISP_ARCHS="{}" 로 설정되어 있어'.format(joined)) + print(" {} 은 실행 파일에 들어가지 않았다. 포함하려면 다시 설정한다:".format(args.name)) + print(' cmake -S {} -B {} -DVISP_ARCHS="{}"'.format( + args.vision_dir, build_dir, ";".join(selected + [args.name]))) + + +if __name__ == "__main__": + main()