-
Notifications
You must be signed in to change notification settings - Fork 454
/
compute.h
118 lines (99 loc) · 4.2 KB
/
compute.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
// Copyright 2015 The Gemmlowp Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// compute.h: the central stage of the Gemm computation, operates
// on already-packed LHS and RHS blocks and calls the Gemm kernel
// to compute a block of the product.
#ifndef GEMMLOWP_INTERNAL_COMPUTE_H_
#define GEMMLOWP_INTERNAL_COMPUTE_H_
#include "block_params.h"
#include "kernel.h"
#include "pack.h"
namespace gemmlowp {
template <typename PackedLhs, typename PackedRhs, typename PackedResult>
class ComputeImpl {
typedef typename PackedLhs::KernelSideFormat KernelLhsFormat;
typedef typename PackedRhs::KernelSideFormat KernelRhsFormat;
typedef KernelFormat<KernelLhsFormat, KernelRhsFormat> Format;
const KernelBase& kernel_;
const BlockParams& block_params_;
PackedResult* const packed_result_;
const PackedLhs& packed_lhs_;
const PackedRhs& packed_rhs_;
public:
ComputeImpl(const KernelBase& _kernel, const BlockParams& _block_params,
PackedResult* _packed_result, const PackedLhs& _packed_lhs,
const PackedRhs& _packed_rhs)
: kernel_(_kernel),
block_params_(_block_params),
packed_result_(_packed_result),
packed_lhs_(_packed_lhs),
packed_rhs_(_packed_rhs) {}
void Compute(int depth) {
depth = RoundUp<Format::kDepth>(depth);
assert(depth <= block_params_.l2_depth);
for (int d = 0; d < depth; d += block_params_.l1_depth) {
int ds = std::min(block_params_.l1_depth, depth - d);
for (int r = 0; r < block_params_.l2_rows; r += block_params_.l1_rows) {
int rs = std::min(block_params_.l1_rows, block_params_.l2_rows - r);
ComputeL1(r, rs, 0, block_params_.l2_cols, d, ds);
}
}
}
private:
static void MarkPackedResultBlockAsInitialized(
const MatrixMap<std::int32_t, MapOrder::ColMajor>& packed_result_block) {
#ifdef GEMMLOWP_MARK_MEMORY_AS_INITIALIZED
for (int col = 0; col < packed_result_block.cols(); col++) {
MarkMemoryAsInitialized(
packed_result_block.data() + col * packed_result_block.cols_stride(),
packed_result_block.rows());
}
#else
(void)packed_result_block;
#endif
}
void ComputeRun(int start_row, int start_col, int start_depth,
int depth) GEMMLOWP_NOINLINE {
packed_lhs_.seek_run(start_row, start_depth);
packed_rhs_.seek_run(start_col, start_depth);
auto packed_result_block = packed_result_->Map().block(
start_row, start_col, Format::kRows, Format::kCols);
kernel_.Run(packed_result_block.data(), packed_result_block.rows_stride(),
packed_result_block.cols_stride(), packed_lhs_.current_data(),
packed_rhs_.current_data(), start_depth, depth);
MarkPackedResultBlockAsInitialized(packed_result_block);
}
void ComputeL1(int start_row, int rows, int start_col, int cols,
int start_depth, int depth) {
assert(rows % Format::kRows == 0);
assert(cols % Format::kCols == 0);
assert(depth % Format::kDepth == 0);
for (int c = 0; c < cols; c += Format::kCols) {
for (int r = 0; r < rows; r += Format::kRows) {
ComputeRun(start_row + r, start_col + c, start_depth, depth);
}
}
}
};
template <typename PackedLhs, typename PackedRhs, typename PackedResult>
void Compute(const KernelBase& kernel, const BlockParams& block_params,
PackedResult* packed_result, const PackedLhs& packed_lhs,
const PackedRhs& packed_rhs, int depth) {
ScopedProfilingLabel label("compute");
ComputeImpl<PackedLhs, PackedRhs, PackedResult> impl(
kernel, block_params, packed_result, packed_lhs, packed_rhs);
impl.Compute(depth);
}
} // namespace gemmlowp
#endif // GEMMLOWP_INTERNAL_COMPUTE_H_