Skip to content

ggml-openvino : make op support a property of the registry #307

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
cavusmustafa wants to merge 1 commit into
base: dev_backend_openvino
Choose a base branch
from
+745 −499
Draft

ggml-openvino : make op support a property of the registry #307

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
473 changes: 36 additions & 437 deletions ggml/src/ggml-openvino/ggml-openvino.cpp
View file
Open in desktop

Large diffs are not rendered by default.

557 changes: 557 additions & 0 deletions ggml/src/ggml-openvino/openvino/op_support.cpp
View file
Open in desktop

Large diffs are not rendered by default.

71 changes: 71 additions & 0 deletions ggml/src/ggml-openvino/openvino/op_support.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
#pragma once

// Per-op support rules for the OpenVINO backend.
//
// Every entry in the translator table names one of these. The registry requires it, so
// a translator cannot be added without also stating when it may be used - which is what
// keeps the gate and the translators from drifting apart. Previously the conditions
// lived in a switch in ggml-openvino.cpp that 19 of the 54 registered ops never
// reached, so those ops were accepted unchecked and failed later during translation.

#include "ggml.h"

#include <string>

// Why the gate turned a node away. Default-constructed means supported.
struct ggml_openvino_op_support {
bool is_supported = true;
std::string reason;

operator bool() const { return is_supported; }
};

namespace ov {
namespace frontend {
namespace ggml {

// A rule is a pure function of one node. It must give the same answer every time it is
// asked: the scheduler consults the gate again on every graph rebuild, and a rule that
// changed its mind would move a node between backends mid-run.
//
// NOT EVERY NODE CAN BE DECLINED. If the destination tensor already has a buffer when the
// gate runs, no other backend can take the node and the scheduler aborts instead of falling
// back (ggml-backend.cpp, "pre-allocated tensor ... that cannot run the operation"). The CPU
// backend can only accept an OpenVINO buffer when the buffer type reports is_host, and it
// does not. In practice this means the nodes that write the KV cache: SET_ROWS into
// cache_k_l* / cache_v_l*, and views over them. Ordinary compute nodes have no buffer yet,
// so declining them is always safe, and so is declining a KV read.
//
// Before adding a rule that can fire on a cache write, check that it cannot fire in a real
// model. supports_get_rows_set_rows() is the one to watch: it serves GET_ROWS, which is never
// pre-allocated, and SET_ROWS, which is the cache write.
using SupportsFunction = ggml_openvino_op_support (*)(const ggml_tensor * op);

ggml_openvino_op_support supports_add_id(const ggml_tensor * op);
ggml_openvino_op_support supports_add_mul_sub(const ggml_tensor * op);
ggml_openvino_op_support supports_argsort(const ggml_tensor * op);
ggml_openvino_op_support supports_concat(const ggml_tensor * op);
ggml_openvino_op_support supports_cpy(const ggml_tensor * op);
ggml_openvino_op_support supports_div(const ggml_tensor * op);
ggml_openvino_op_support supports_flash_attn_ext(const ggml_tensor * op);
ggml_openvino_op_support supports_gated_delta_net(const ggml_tensor * op);
ggml_openvino_op_support supports_get_rows_set_rows(const ggml_tensor * op);
ggml_openvino_op_support supports_mul_mat(const ggml_tensor * op);
ggml_openvino_op_support supports_mul_mat_id(const ggml_tensor * op);
ggml_openvino_op_support supports_pad(const ggml_tensor * op);
ggml_openvino_op_support supports_permute(const ggml_tensor * op);
ggml_openvino_op_support supports_pool_2d(const ggml_tensor * op);
ggml_openvino_op_support supports_repeat(const ggml_tensor * op);
ggml_openvino_op_support supports_reshape(const ggml_tensor * op);
ggml_openvino_op_support supports_rope(const ggml_tensor * op);
ggml_openvino_op_support supports_set(const ggml_tensor * op);
ggml_openvino_op_support supports_ssm_conv(const ggml_tensor * op);
ggml_openvino_op_support supports_sum_rows(const ggml_tensor * op);
ggml_openvino_op_support supports_transpose(const ggml_tensor * op);
ggml_openvino_op_support supports_tri(const ggml_tensor * op);
ggml_openvino_op_support supports_unconstrained(const ggml_tensor * op);
ggml_openvino_op_support supports_view(const ggml_tensor * op);

} // namespace ggml
} // namespace frontend
} // namespace ov
114 changes: 57 additions & 57 deletions ggml/src/ggml-openvino/openvino/op_table.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,65 +19,65 @@ namespace ov {
namespace frontend {
namespace ggml {

std::unordered_map<std::string, CreatorFunction> get_supported_ops() {
std::unordered_map<std::string, OpEntry> get_supported_ops() {
using namespace ov::op;
return {
{"GGML_OP_ADD", op::translate_add },
{"GGML_OP_ADD1", op::translate_1to1_match_2_inputs<v1::Add> },
{"GGML_OP_ADD_ID", op::translate_add_id },
{"GGML_OP_CONCAT", op::translate_concat },
{"GGML_OP_CONT", op::translate_cont },
{"GGML_OP_DIV", op::translate_div },
{"GGML_OP_FILL", op::translate_fill },
{"GGML_OP_GET_ROWS", op::translate_get_rows },
{"GGML_OP_IM2COL", op::translate_im2col },
{"GGML_OP_MUL", op::translate_1to1_match_2_inputs<v1::Multiply>},
{"GGML_OP_MUL_MAT", op::translate_mulmat },
{"GGML_OP_MUL_MAT_ID", op::translate_mul_mat_id },
{"GGML_OP_PERMUTE", op::translate_permute },
{"GGML_OP_RESHAPE", op::translate_reshape },
{"GGML_OP_RMS_NORM", op::translate_rms_norm },
{"GGML_OP_NORM", op::translate_norm },
{"GGML_OP_L2_NORM", op::translate_l2_norm },
{"GGML_OP_SUM_ROWS", op::translate_sum_rows },
{"GGML_OP_ROPE", op::translate_rope },
{"GGML_OP_SCALE", op::translate_scale },
{"GGML_OP_SQR", op::translate_sqr },
{"GGML_OP_SQRT", op::translate_sqrt },
{"GGML_OP_SOFT_MAX", op::translate_soft_max },
{"GGML_OP_ARGSORT", op::translate_argsort },
{"GGML_OP_SUB", op::translate_1to1_match_2_inputs<v1::Subtract>},
{"GGML_OP_TRANSPOSE", op::translate_transpose },
{"GGML_UNARY_OP_GELU", op::translate_1to1_match_1_input<v7::Gelu> },
{"GGML_UNARY_OP_SIGMOID", op::translate_1to1_match_1_input<v0::Sigmoid> },
{"GGML_UNARY_OP_SILU", op::translate_unary_silu },
{"GGML_UNARY_OP_SOFTPLUS", op::translate_unary_softplus },
{"GGML_UNARY_OP_TANH", op::translate_1to1_match_1_input<v0::Tanh> },
{"GGML_UNARY_OP_SIGMOID", op::translate_1to1_match_1_input<v0::Sigmoid> },
{"GGML_UNARY_OP_EXP", op::translate_1to1_match_1_input<v0::Exp> },
{"GGML_UNARY_OP_NEG", op::translate_1to1_match_1_input<v0::Negative> },
{"GGML_UNARY_OP_RELU", op::translate_1to1_match_1_input<v0::Relu> },
{"GGML_OP_VIEW", op::translate_view },
{"GGML_GLU_OP_SWIGLU", op::translate_glu_swiglu },
{"GGML_GLU_OP_SWIGLU_OAI", op::translate_glu_swiglu_oai },
{"GGML_GLU_OP_SWIGLU_CLAMP", op::translate_glu_swiglu_clamp },
{"GGML_GLU_OP_GEGLU", op::translate_glu_geglu },
{"GGML_GLU_OP_GEGLU_QUICK", op::translate_glu_geglu_quick },
{"GGML_OP_SET_ROWS", op::translate_set_rows },
{"GGML_OP_CPY", op::translate_cpy },
{"GGML_OP_FLASH_ATTN_EXT", op::translate_flash_attn_ext },
{"GGML_OP_CLAMP", op::translate_clamp },
{"GGML_OP_PAD", op::translate_pad },
{"GGML_OP_SSM_CONV", op::translate_ssm_conv },
{"GGML_OP_GATED_DELTA_NET", op::translate_gated_delta_net },
{"GGML_OP_REPEAT", op::translate_repeat },
{"GGML_OP_CUMSUM", op::translate_cumsum },
{"GGML_OP_FILL", op::translate_fill },
{"GGML_OP_DIAG", op::translate_diag },
{"GGML_OP_TRI", op::translate_tri },
{"GGML_OP_SET", op::translate_set },
{"GGML_OP_POOL_2D", op::translate_pool_2d },
{"GGML_OP_ROLL", op::translate_roll },
{"GGML_OP_ADD", {op::translate_add, supports_add_mul_sub}},
{"GGML_OP_ADD1", {op::translate_1to1_match_2_inputs<v1::Add>, supports_unconstrained}},
{"GGML_OP_ADD_ID", {op::translate_add_id, supports_add_id}},
{"GGML_OP_CONCAT", {op::translate_concat, supports_concat}},
{"GGML_OP_CONT", {op::translate_cont, supports_unconstrained}},
{"GGML_OP_DIV", {op::translate_div, supports_div}},
{"GGML_OP_FILL", {op::translate_fill, supports_unconstrained}},
{"GGML_OP_GET_ROWS", {op::translate_get_rows, supports_get_rows_set_rows}},
{"GGML_OP_IM2COL", {op::translate_im2col, supports_unconstrained}},
{"GGML_OP_MUL", {op::translate_1to1_match_2_inputs<v1::Multiply>, supports_add_mul_sub}},
{"GGML_OP_MUL_MAT", {op::translate_mulmat, supports_mul_mat}},
{"GGML_OP_MUL_MAT_ID", {op::translate_mul_mat_id, supports_mul_mat_id}},
{"GGML_OP_PERMUTE", {op::translate_permute, supports_permute}},
{"GGML_OP_RESHAPE", {op::translate_reshape, supports_reshape}},
{"GGML_OP_RMS_NORM", {op::translate_rms_norm, supports_unconstrained}},
{"GGML_OP_NORM", {op::translate_norm, supports_unconstrained}},
{"GGML_OP_L2_NORM", {op::translate_l2_norm, supports_unconstrained}},
{"GGML_OP_SUM_ROWS", {op::translate_sum_rows, supports_sum_rows}},
{"GGML_OP_ROPE", {op::translate_rope, supports_rope}},
{"GGML_OP_SCALE", {op::translate_scale, supports_unconstrained}},
{"GGML_OP_SQR", {op::translate_sqr, supports_unconstrained}},
{"GGML_OP_SQRT", {op::translate_sqrt, supports_unconstrained}},
{"GGML_OP_SOFT_MAX", {op::translate_soft_max, supports_unconstrained}},
{"GGML_OP_ARGSORT", {op::translate_argsort, supports_argsort}},
{"GGML_OP_SUB", {op::translate_1to1_match_2_inputs<v1::Subtract>, supports_add_mul_sub}},
{"GGML_OP_TRANSPOSE", {op::translate_transpose, supports_transpose}},
{"GGML_UNARY_OP_GELU", {op::translate_1to1_match_1_input<v7::Gelu>, supports_unconstrained}},
{"GGML_UNARY_OP_SIGMOID", {op::translate_1to1_match_1_input<v0::Sigmoid>, supports_unconstrained}},
{"GGML_UNARY_OP_SILU", {op::translate_unary_silu, supports_unconstrained}},
{"GGML_UNARY_OP_SOFTPLUS", {op::translate_unary_softplus, supports_unconstrained}},
{"GGML_UNARY_OP_TANH", {op::translate_1to1_match_1_input<v0::Tanh>, supports_unconstrained}},
{"GGML_UNARY_OP_SIGMOID", {op::translate_1to1_match_1_input<v0::Sigmoid>, supports_unconstrained}},
{"GGML_UNARY_OP_EXP", {op::translate_1to1_match_1_input<v0::Exp>, supports_unconstrained}},
{"GGML_UNARY_OP_NEG", {op::translate_1to1_match_1_input<v0::Negative>, supports_unconstrained}},
{"GGML_UNARY_OP_RELU", {op::translate_1to1_match_1_input<v0::Relu>, supports_unconstrained}},
{"GGML_OP_VIEW", {op::translate_view, supports_view}},
{"GGML_GLU_OP_SWIGLU", {op::translate_glu_swiglu, supports_unconstrained}},
{"GGML_GLU_OP_SWIGLU_OAI", {op::translate_glu_swiglu_oai, supports_unconstrained}},
{"GGML_GLU_OP_SWIGLU_CLAMP",{op::translate_glu_swiglu_clamp, supports_unconstrained}},
{"GGML_GLU_OP_GEGLU", {op::translate_glu_geglu, supports_unconstrained}},
{"GGML_GLU_OP_GEGLU_QUICK", {op::translate_glu_geglu_quick, supports_unconstrained}},
{"GGML_OP_SET_ROWS", {op::translate_set_rows, supports_get_rows_set_rows}},
{"GGML_OP_CPY", {op::translate_cpy, supports_cpy}},
{"GGML_OP_FLASH_ATTN_EXT", {op::translate_flash_attn_ext, supports_flash_attn_ext}},
{"GGML_OP_CLAMP", {op::translate_clamp, supports_unconstrained}},
{"GGML_OP_PAD", {op::translate_pad, supports_pad}},
{"GGML_OP_SSM_CONV", {op::translate_ssm_conv, supports_ssm_conv}},
{"GGML_OP_GATED_DELTA_NET", {op::translate_gated_delta_net, supports_gated_delta_net}},
{"GGML_OP_REPEAT", {op::translate_repeat, supports_repeat}},
{"GGML_OP_CUMSUM", {op::translate_cumsum, supports_unconstrained}},
{"GGML_OP_FILL", {op::translate_fill, supports_unconstrained}},
{"GGML_OP_DIAG", {op::translate_diag, supports_unconstrained}},
{"GGML_OP_TRI", {op::translate_tri, supports_tri}},
{"GGML_OP_SET", {op::translate_set, supports_set}},
{"GGML_OP_POOL_2D", {op::translate_pool_2d, supports_pool_2d}},
{"GGML_OP_ROLL", {op::translate_roll, supports_unconstrained}},
// solve_tri has accuracy issues on GPU
// {"GGML_OP_SOLVE_TRI", op::translate_solve_tri },
};
Expand Down
20 changes: 19 additions & 1 deletion ggml/src/ggml-openvino/openvino/op_table.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
#pragma once

#include "node_context.h"
#include "op_support.h"

#include <utility>

namespace ov {
namespace frontend {
Expand Down Expand Up @@ -60,7 +63,22 @@ GGML_OP_CONVERTER(translate_roll);

} // namespace op

std::unordered_map<std::string, CreatorFunction> get_supported_ops();
// One entry per op: how to translate it, and when it may be used. Both members are
// required, so a translator cannot be registered without a support rule - that is what
// keeps the gate from drifting away from what the translators actually accept.
struct OpEntry {
CreatorFunction translate;
SupportsFunction supports;

// Both arguments are required on purpose. Without this constructor OpEntry would be
// an aggregate, and {translate_foo} would compile with supports silently null - so
// the one guarantee this type exists to provide would not hold.
OpEntry(CreatorFunction translate, SupportsFunction supports) :
translate(std::move(translate)),
supports(supports) {}
};

std::unordered_map<std::string, OpEntry> get_supported_ops();

} // namespace ggml
} // namespace frontend
Expand Down
4 changes: 2 additions & 2 deletions ggml/src/ggml-openvino/openvino/translate_session.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ void preprocess(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) {
} // namespace

TranslateSession::TranslateSession(const frontend::InputModel::Ptr & input_model,
const std::unordered_map<std::string, CreatorFunction> & translator_map,
const std::unordered_map<std::string, OpEntry> & translator_map,
bool naive) :
m_input_model(input_model),
m_translator_map(translator_map),
Expand Down Expand Up @@ -303,7 +303,7 @@ std::shared_ptr<Model> TranslateSession::translate_graph(const frontend::InputMo
FRONT_END_OP_CONVERSION_CHECK(it != m_translator_map.end(), "Translation for operation type ", operation_type,
" is not implemented.");
NodeContext node_context(decoder, tensor_map, node_idx, this);
ov::OutputVector converted_outputs = it->second(node_context);
ov::OutputVector converted_outputs = it->second.translate(node_context);

const auto & node_output_names = decoder->get_output_names(node_idx);
FRONT_END_OP_CONVERSION_CHECK(node_output_names.size() == converted_outputs.size(), "Number of ",
Expand Down
5 changes: 3 additions & 2 deletions ggml/src/ggml-openvino/openvino/translate_session.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

#include "input_model.h"
#include "node_context.h"
#include "op_table.h"

namespace ov {
namespace frontend {
Expand All @@ -10,7 +11,7 @@ namespace ggml {
class TranslateSession {
public:
TranslateSession(const frontend::InputModel::Ptr & input_model,
const std::unordered_map<std::string, CreatorFunction> & translator_map,
const std::unordered_map<std::string, OpEntry> & translator_map,
bool naive = false);

std::shared_ptr<Model> get_converted_model();
Expand All @@ -19,7 +20,7 @@ class TranslateSession {
private:
std::shared_ptr<Model> apply_transformations(std::shared_ptr<Model> model);
const frontend::InputModel::Ptr m_input_model;
const std::unordered_map<std::string, CreatorFunction> & m_translator_map;
const std::unordered_map<std::string, OpEntry> & m_translator_map;
std::shared_ptr<Model> m_ov_model;
bool m_naive;
};
Expand Down
Loading