-
Notifications
You must be signed in to change notification settings - Fork 5
/
HelloSP.cu
403 lines (339 loc) · 13.4 KB
/
HelloSP.cu
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
#include <iostream>
#include <stdio.h>
#include <algorithm>
#include <ctime>
#include <cstdlib>
#include <cmath>
#include <random>
#include <assert.h>
#include <cuda.h>
#include <curand_kernel.h>
#include <thrust/device_vector.h>
#include <thrust/sequence.h>
#include <thrust/fill.h>
#include <thrust/device_ptr.h>
#include <thrust/device_malloc.h>
#include <thrust/device_free.h>
#include <thrust/device_vector.h>
#include <thrust/random.h>
#include <thrust/transform.h>
#include <thrust/iterator/counting_iterator.h>
#include <thrust/generate.h>
#include "SpatialPooler.cu"
#define checkError(ans) { gpuAssert((ans), __FILE__, __LINE__); }
inline void gpuAssert(cudaError_t code, const char *file, int line, bool abort=true)
{
if (code != cudaSuccess)
{
fprintf(stderr,"CUDA error: %s %s %d\n", cudaGetErrorString(code), file, line);
if (abort) exit(code);
}
}
using namespace std;
typedef unsigned int UInt;
typedef float Real;
UInt* generatePotentialPoolsWithVariableLength(UInt* potentialPools, int cols, const UInt IN_BLOCK_SIZE, Real potentialPct, const UInt MAX_CONNECTED)
{
int connected = 0;
for(int i=0; i < cols; i++)
{
connected = 0;
for(int j=0; j < IN_BLOCK_SIZE; j++)
{
if((Real)(rand()%100)/100 <= potentialPct && connected < MAX_CONNECTED)
{
potentialPools[i*MAX_CONNECTED + connected++] = j;
}
}
// numPotential[i] = connected;
}
return potentialPools;
}
// This approach will work only when IN_BLOCK_SIZE !>> num_desired_connected (Knuth's algorithm)
// In our case, usually IN_BLOCK_SIZE !>> num_desired_connected
UInt* generatePotentialPoolsEqualLengts(UInt* potentialPools, const UInt SP_SIZE, const UInt IN_BLOCK_SIZE, const UInt num_desired_connected)
{
for(int i=0; i < SP_SIZE; i++)
{
int connected = 0;
for(int j=0; j < IN_BLOCK_SIZE; j++)
{
if((Real)(rand()%100)/100 <= (num_desired_connected-connected)/(IN_BLOCK_SIZE-j) && connected < num_desired_connected)
{
potentialPools[i*num_desired_connected + connected++] = j;
}
}
}
return potentialPools;
}
UInt* generatePotentialPoolsUsingShuffle(UInt* potentialPools, const UInt SP_SIZE, const UInt IN_BLOCK_SIZE, const UInt MAX_CONNECTED)
{
vector<UInt> indeces(IN_BLOCK_SIZE);
iota(indeces.begin(), indeces.end(), 0);
// We could also do this on the device
// thrust::host_vector<UInt> indeces(IN_BLOCK_SIZE);
// thrust::sequence(input_indeces.begin(), input_indeces.end(), 0, 1);
for(int i=0; i < SP_SIZE; i++) {
random_shuffle(indeces.begin(), indeces.end());
copy(indeces.begin(), indeces.begin()+MAX_CONNECTED, &potentialPools[i*MAX_CONNECTED]);
// This may slightly improve performance, but slows down initialization
sort(&potentialPools[i*MAX_CONNECTED], &potentialPools[(i+1)*MAX_CONNECTED]);
}
return potentialPools;
}
Real initPermanencesConnected(Real synPermConnected_, Real synPermMax_)
{
Real p = synPermConnected_ +
(synPermMax_ - synPermConnected_)*((Real)((rand()%100))/100);
return p;
}
Real initPermanencesNotConnected(Real synPermConnected_)
{
Real p = synPermConnected_ * (Real)((rand()%100))/100;
return p;
}
Real* generatePermanences(Real* permanences, int cols, int inputSize, UInt* potential, Real connectedPct,
Real synPermConnected_, Real synPermMax_, const UInt MAX_CONNECTED, UInt* numPotential,
const UInt BLOCK_SIZE, const UInt IN_BLOCK_SIZE)
{
int connected = 0;
int curr_block = 0;
bool found = false;
for(int i=0; i < cols; i++)
{
connected = 0;
// We need to only go through the input block corresponding to the current column
// This means we need to convert current column to the input block number
curr_block = floor(i / BLOCK_SIZE);
// j is the global index of connection in the input matrix
for(int j=curr_block*IN_BLOCK_SIZE; j < curr_block*IN_BLOCK_SIZE + IN_BLOCK_SIZE; j++)
{
// Find if this input is potentially connected with this column
found=false;
for(int k=0; k < numPotential[i]; k++)
{
if(potential[i*MAX_CONNECTED+k] == j % IN_BLOCK_SIZE) {
found = true;
break;
}
}
// If there is, decide if it will be. The structure of the data is as follows:
// potential[col][index of the synapse on the segment] = index of input in the block
// permanences[col][index of the synapse on the segment] = permanence of the synapse
if(found)
{
if((Real)(rand()%100)/100 <= connectedPct)
{
permanences[i*MAX_CONNECTED+connected++] = initPermanencesConnected(synPermConnected_, synPermMax_);
}
else
{
permanences[i*MAX_CONNECTED+connected++] = initPermanencesNotConnected(synPermConnected_);
}
}
}
}
return permanences;
}
// There should also be a parameter to raise permanences so that minimum number of synapses is connected.
UInt** computeConnected(Real** permanences, UInt** potential, UInt cols, UInt inputSize,
Real synPermConnected_, const UInt MAX_CONNECTED, UInt* numPotential)
{
UInt** connected_arr = new UInt*[cols];
int connected = 0;
for(int i=0; i < inputSize; i++)
{
connected = 0;
connected_arr[i] = new UInt[MAX_CONNECTED];
for(int j=0; j < numPotential[i]; j++)
{
if(permanences[i][j] < synPermConnected_)
{
connected_arr[i][connected++] = j;
}
}
}
return connected_arr;
}
bool* generate01(bool* ar, size_t size, Real inDensity)
{
for(int i=0; i < size; i++)
{
ar[i] = (Real)(rand()%100)/100 <= inDensity ? 1 : 0;
}
return ar;
}
struct prg : public thrust::unary_function<unsigned int,bool>
{
Real IN_DENSITY;
__host__ __device__
prg(Real ind) : IN_DENSITY(ind) {}
__host__ __device__
bool operator()(const unsigned int thread_id) const
{
thrust::default_random_engine rng;
thrust::uniform_real_distribution<float> dist(0, 1);
rng.discard(thread_id);
return dist(rng) <= IN_DENSITY ? true : false;
}
};
void visualize_input(bool* in_host, UInt* potentialPools, Real* permanences, UInt* numPotential, const UInt IN_SIZE, const UInt SP_SIZE, const UInt IN_BLOCK_SIZE, const UInt MAX_CONNECTED)
{
printf("POTENTIAL CONNECTIONS WITH PERMANENCES\n");
for(int i=0; i<SP_SIZE; i++)
{
for(int j=0; j<MAX_CONNECTED; j++)
printf("%d \t", potentialPools[i*MAX_CONNECTED+j]);
printf("\n");
for(int j=0; j<numPotential[i]; j++)
printf("%.2f\t", permanences[i*MAX_CONNECTED+j]);
printf("\n");
printf("%d \n", numPotential[i]);
}
printf("INPUT SDR\n");
for(int i=0; i<IN_SIZE; i++)
{
printf("%d ", in_host[i]);
if(i % IN_BLOCK_SIZE == 0 && i > 0)
printf("\n");
}
printf("\n");
}
void visualize_input_generated_on_device(thrust::device_vector<bool>& in_vector, UInt* pot_pools_host, const UInt MAX_CONNECTED, const UInt SP_SIZE)
{
printf("INPUT\n");
thrust::copy(in_vector.begin(), in_vector.end(), std::ostream_iterator<bool>(std::cout, " "));
printf("\n");
// This overlows stdout buffer (better write to a file if necessary)
// printf("POTENTIAL POOLS");
// for(int i=0; i<SP_SIZE; i++)
// {
// for(int j=0; j<MAX_CONNECTED; j++)
// printf("%d \t", pot_pools_host[i*MAX_CONNECTED+j]);
// printf("\n");
// }
}
void visualize_output(bool* cols_host, const UInt SP_SIZE, UInt BLOCK_SIZE)
{
printf("OUTPUT\n");
for(int i=0; i<SP_SIZE; i++)
{
printf("%d ", cols_host[i]);
if(i % BLOCK_SIZE == 0 && i > 0)
printf("\n");
}
printf("\n");
// The final sparsity will approach target with increasing block size
int ones = 0;
for(int i=0; i < SP_SIZE; i++)
if(cols_host[i] > 0) ones++;
printf("Sparsity: %f \n", (Real)ones/SP_SIZE);
}
void printErrorMessage(cudaError_t error, int memorySize){
printf("==================================================\n");
printf("MEMORY ERROR : %s\n", cudaGetErrorString(error));
printf("==================================================\n");
}
int main(int argc, const char * argv[])
{
srand(time(NULL));
// construct input args
args ar;
ar.iteration_num=0;
ar.learn=true;
ar.localAreaDensity=0.02; // SP density after inhibition
ar.potentialPct=0.1; //
ar.connectedPct=0.5;
ar.stimulusThreshold=0;
ar.synPermTrimThreshold=0.025;
ar.synPermMax=1.0;
ar.synPermConnected=0.1;
ar.synPermActiveInc=0.05;
ar.synPermInactiveDec=0.008;
ar.synPermBelowStimulusInc=ar.synPermConnected / 10.0;
ar.dutyCyclePeriod=1000;
ar.boostStrength=0.05; // 0 means no boosting
ar.minPctOdc=0.001;
ar.update_period=50;
ar.SP_SIZE = 32768;
ar.IN_SIZE = 131072;
ar.BLOCK_SIZE = 1024;
ar.NUM_BLOCKS = ar.SP_SIZE/ar.BLOCK_SIZE;
ar.IN_BLOCK_SIZE = ar.IN_SIZE/ar.NUM_BLOCKS; // Size of chunk of input processed by a single cuda block
ar.MAX_CONNECTED = 1024;
ar.IN_DENSITY = 0.5; // Density of input connections
ar.num_connected = std::floor(ar.MAX_CONNECTED*ar.connectedPct);
// Host memory allocation
bool* cols_host = (bool*) malloc(ar.SP_SIZE*sizeof(bool));
// bool* in_host = (bool*) &cols_host[SP_SIZE];
UInt* pot_pools_host = (UInt*) malloc(ar.SP_SIZE*ar.num_connected*sizeof(UInt));
pot_pools_host = generatePotentialPoolsUsingShuffle(pot_pools_host, ar.SP_SIZE, ar.IN_BLOCK_SIZE, ar.num_connected);
// Host memory init
// in_host = generate01(in_host, IN_SIZE, IN_DENSITY);
// visualize_input(in_host, potentialPools, permanences, numPotential, IN_SIZE, SP_SIZE, ar.IN_BLOCK_SIZE, MAX_CONNECTED);
// Global memory pointers
args* ar_dev;
// Global memory allocation
checkError( cudaMalloc((void **) &ar_dev, sizeof(ar)) );
size_t pot_dev_pitch_in_bytes, per_dev_pitch_in_bytes;
checkError( cudaMallocPitch((void **) &ar.pot_dev, &pot_dev_pitch_in_bytes, ar.num_connected*sizeof(UInt), ar.SP_SIZE) );
checkError( cudaMallocPitch((void **) &ar.per_dev, &per_dev_pitch_in_bytes, ar.num_connected*sizeof(Real), ar.SP_SIZE) );
ar.pot_dev_pitch = pot_dev_pitch_in_bytes / sizeof(UInt);
ar.per_dev_pitch = per_dev_pitch_in_bytes / sizeof(Real);
checkError( cudaMalloc((void **) &ar.boosts_dev, ar.SP_SIZE*ar.num_connected*sizeof(Real)) );
// checkError( cudaMalloc((void **) &ar.in_dev, ar.IN_SIZE*sizeof(bool)) );
checkError( cudaMalloc((void **) &ar.olaps_dev, ar.SP_SIZE*sizeof(UInt)) );
checkError( cudaMalloc((void **) &ar.cols_dev, ar.SP_SIZE*sizeof(bool)) );
checkError( cudaMalloc((void **) &ar.numPot_dev, ar.SP_SIZE*sizeof(UInt)) );
checkError( cudaMalloc((void **) &ar.odc_dev, ar.MAX_CONNECTED*ar.SP_SIZE*sizeof(Real)) );
checkError( cudaMalloc((void **) &ar.adc_dev, ar.MAX_CONNECTED*ar.SP_SIZE*sizeof(Real)) );
checkError( cudaMalloc((void **) &ar.minOdc_dev, ar.NUM_BLOCKS*sizeof(Real)) );
checkError( cudaMalloc((void **) &ar.dev_states, ar.SP_SIZE*ar.BLOCK_SIZE*sizeof(curandState)) );
// Global memory initialization
setup_kernel<<<ar.NUM_BLOCKS, ar.BLOCK_SIZE>>>(ar.dev_states);
// Potential pools
// TODO: Unfortunately, when we generate potential pools on device, all activity seems to accumulate in the first and last block of SP for some reason
// thrust::device_vector<UInt> input_indeces(ar.IN_BLOCK_SIZE);
// UInt* indeces_ptr = thrust::raw_pointer_cast(&input_indeces[0]);
// thrust::sequence(input_indeces.begin(), input_indeces.end(), 0, 1);
// size_t sm = ar.BLOCK_SIZE*sizeof(UInt);
// generatePotentialPools<<<ar.SP_SIZE, ar.BLOCK_SIZE, sm>>>(ar.pot_dev, ar.pot_dev_pitch, ar.num_connected, indeces_ptr, ar.dev_states, ar.IN_BLOCK_SIZE);
// Permanences
generatePermanences<<<ar.SP_SIZE, ar.num_connected>>>(ar.per_dev, ar.per_dev_pitch, ar.connectedPct, ar.synPermConnected, ar.synPermMax, ar.dev_states);
// Boosts
thrust::device_ptr<float> boosts_ptr(ar.boosts_dev);
thrust::fill(boosts_ptr, boosts_ptr+ar.SP_SIZE*ar.num_connected*sizeof(Real), 1.0);
// Number of potentialy connected synapses - unnecessary if we want it variable
thrust::device_ptr<UInt> num_ptr(ar.numPot_dev);
thrust::fill(num_ptr, num_ptr+ar.SP_SIZE*sizeof(UInt), ar.num_connected);
// Input
thrust::device_vector<bool> in_vector(ar.IN_SIZE);
thrust::counting_iterator<unsigned int> index_sequence_begin(0);
thrust::transform(index_sequence_begin,
index_sequence_begin + ar.IN_SIZE,
in_vector.begin(),
prg(ar.IN_DENSITY));
ar.in_dev = thrust::raw_pointer_cast(&in_vector[0]);
visualize_input_generated_on_device(in_vector, pot_pools_host, ar.num_connected, ar.SP_SIZE);
// Memcpy to device
checkError( cudaMemcpy(ar_dev, (void**) &ar, sizeof(ar), cudaMemcpyHostToDevice) );
// checkError( cudaMemcpy(ar.in_dev, in_host, ar.IN_SIZE*sizeof(bool), cudaMemcpyHostToDevice) );
checkError( cudaMemcpy2D(ar.pot_dev, pot_dev_pitch_in_bytes, pot_pools_host, ar.num_connected*sizeof(UInt), ar.num_connected*sizeof(UInt), ar.SP_SIZE, cudaMemcpyHostToDevice) );
// Kernel call
// cudaThreadSynchronize();
size_t sm = ar.BLOCK_SIZE*(2*sizeof(Real) + sizeof(UInt)) + ar.IN_BLOCK_SIZE*sizeof(bool);
compute<<<ar.NUM_BLOCKS, ar.BLOCK_SIZE, sm>>>(ar_dev);
// cudaThreadSynchronize();
// Memcpy from device
checkError( cudaMemcpy(cols_host, ar.cols_dev, ar.SP_SIZE*sizeof(bool), cudaMemcpyDeviceToHost));
visualize_output(cols_host, ar.SP_SIZE, ar.BLOCK_SIZE);
// cudaFree(ar.in_dev);
cudaFree(ar.cols_dev);
cudaFree(ar.pot_dev);
cudaFree(ar.per_dev);
cudaFree(ar.boosts_dev);
cudaFree(ar.odc_dev);
cudaFree(ar.adc_dev);
cudaFree(ar.numPot_dev);
return 0;
}