Qore DataProvider Module Reference  1.2.1
DataProviderPipeline.qc.dox.h
1 // -*- mode: c++; indent-tabs-mode: nil -*-
3 
25 // assume local scope for variables, do not use "$" signs
26 // require type definitions everywhere
28 // enable all warnings
29 // allow weak references
30 
32 namespace DataProvider {
36 const PS_ABORTED = "ABORTED";
38 
40 const PS_RUNNING = "RUNNING";
41 
43 const PS_IDLE = "IDLE";
45 
47 public struct PipelineInfo {
49  string name;
50 
53 
56 
58 
60  string status;
61 
64 
67 
69 
71  bool bulk;
72 
75 
78 
80  float recs_per_sec;
81 };
82 
84 public struct PipelineOptionInfo {
86 
88  *code debug_log;
89 
91 
93  *code error_log;
94 
96 
98  *code info_log;
99 
101  /* it is called in the new pipeline queue thread with no arguments
102  */
104 
106  *string name;
107 };
108 
111 
112 public:
114  int id;
115 
117  Mutex lck;
118 
120  Counter cnt;
121 
123  Condition cond();
124 
126  Condition flush_cond();
127 
129  int queue_waiting = 0;
130 
132  int data_waiting = 0;
133 
135  int flush_waiting = 0;
136 
138  list<auto> queue;
139 
141  int size;
142 
144  int tid;
145 
147 
151  list<auto> elems();
152 
154  bool do_flush;
155 
158 
161 
163  constructor(DataProviderPipeline parent, Mutex lck, Counter cnt, int id, int size);
164 
165 
167  int getId();
168 
169 
171 
175  submit(auto qdata);
176 
177 
179  run(Counter run_cnt);
180 
181 
183 
185  waitDone();
186 
187 
190 protected:
191  runIntern();
192 public:
193 
194 
195 protected:
196  flushIntern();
197 public:
198 
199 };
200 
202 
208 
209 public:
210 
211 
212 protected:
214  string name;
215 
217  hash<string, PipelineQueue> pmap;
218 
220 
222  bool do_bulk = True;
223 
225  bool locked = False;
226 
228  Sequence seq(1);
229 
231  list<hash<ExceptionInfo>> error_list;
232 
235 
238 
240  int record_count = 0;
241 
243  *code info_log;
244 
246  *code error_log;
247 
249  *code debug_log;
250 
252  /* it is called in the new pipeline queue thread with no arguments
253  */
255 
257  Mutex lck();
258 
260  bool stop_flag;
261 
264 
266  Counter cnt();
267 
268 public:
269 
271 
275  constructor(*hash<PipelineOptionInfo> opts);
276 
277 
279 
281  copy();
282 
283 
285 
291  destructor();
292 
293 
295  string getName();
296 
297 
299  bool hasQueue(int id);
300 
301 
303  bool stopping();
304 
305 
307  bool aborting();
308 
309 
312 
313 
315 
320  append(AbstractDataProcessor processor);
321 
322 
324 
334  append(int id, AbstractDataProcessor processor);
335 
336 
338 
348  int appendQueue(int id);
349 
350 
352  bool isProcessing();
353 
354 
356  submit(auto _data);
357 
358 
360  submitData(AbstractIterator i);
361 
362 
365 
366 
369 
370 
372 
375  reset();
376 
377 
379 
382  waitDone();
383 
384 
386 
394  abort(*bool ignore_exceptions);
395 
396 
398 
402  hash<PipelineInfo> getInfo();
403 
404 
406  reportError(PipelineQueue queue, hash<ExceptionInfo> ex);
407 
408 
410  logInfo(string fmt);
411 
412 
414  logError(string fmt);
415 
416 
418  logDebug(string fmt);
419 
420 
422 private:
424 public:
425 
426 
428 
430 private:
431  submitIntern(auto _data);
432 public:
433 
434 
436 
438 private:
439  submitDataIntern(auto _data);
440 public:
441 
442 
444 
448 private:
450 public:
451 
452 
454 
456 private:
457  resetIntern();
458 public:
459 
460 
462 
464 private:
466 public:
467 
468 
470 
472 private:
474 public:
475 
476 
478 
480 private:
482 public:
483 
484 
486 private:
488 public:
489 
490 
492 private:
493  stopIntern();
494 public:
495 
496 
498 private:
500 public:
501 
502 };
503 };
504 
505 *list get_stack();
506 
507 
508 
logError(string fmt)
Logs to the error log, if set.
date date(date dt)
hash< string, PipelineQueue > pmap
Hash of queues keyed by queue ID.
Definition: DataProviderPipeline.qc.dox.h:217
*code info_log
a closure or call reference for info logging
Definition: DataProviderPipeline.qc.dox.h:98
int tid
TID of the background thread.
Definition: DataProviderPipeline.qc.dox.h:144
waitDone()
Wait for the queue to be empty, then wait for all terminating pipelines to be empty.
int getId()
Returns the pipeline ID.
PipelineQueue copyPipeline(PipelineQueue old_queue)
Called by the copy constructor to copy the queues.
bool do_flush
Flush pipeline flag.
Definition: DataProviderPipeline.qc.dox.h:154
date start_time
run start time
Definition: DataProviderPipeline.qc.dox.h:234
int id
Queue ID.
Definition: DataProviderPipeline.qc.dox.h:114
bool stop_flag
Stop flag.
Definition: DataProviderPipeline.qc.dox.h:260
submit(auto _data)
Submits data for processing.
const PS_ABORTED
Pipeline status: ABORTED.
Definition: DataProviderPipeline.qc.dox.h:37
Defines the abstract class for data provider iterators; the destructor releases the iterator...
Definition: AbstractDataProviderRecordIterator.qc.dox.h:33
const PS_IDLE
Pipeline status: IDLE.
Definition: DataProviderPipeline.qc.dox.h:43
date duration
Total time processing end to end.
Definition: DataProviderPipeline.qc.dox.h:74
int size
Maximum queue size.
Definition: DataProviderPipeline.qc.dox.h:141
bool locked
Locked flag.
Definition: DataProviderPipeline.qc.dox.h:225
string name
The name of the pipeline.
Definition: DataProviderPipeline.qc.dox.h:49
string name
A descriptive name for logging purposes.
Definition: DataProviderPipeline.qc.dox.h:214
int num_queues
Number of pipeline queues.
Definition: DataProviderPipeline.qc.dox.h:66
const True
Condition cond()
Queue condition variable.
*code error_log
Error log closure; takes a single format string and then arguments for format placeholders.
Definition: DataProviderPipeline.qc.dox.h:246
*date stop_time
Stop time for processing.
Definition: DataProviderPipeline.qc.dox.h:55
reset()
Resets the pipeline.
append(AbstractDataProcessor processor)
Appends a data processor to the default queue.
string getName()
Returns the pipeline name.
int appendQueue(int id)
Appends a new queue to an existing pipeline and returns the new queue ID.
float duration_secs
Total time processing end to end as a floating-point value in durationSecondsFloat.
Definition: DataProviderPipeline.qc.dox.h:77
bool do_bulk
Bulk flag.
Definition: DataProviderPipeline.qc.dox.h:222
bool bulk
Flag that indicates if the pipeline is capable of bulk record processing.
Definition: DataProviderPipeline.qc.dox.h:71
const False
string status
Pipeline status.
Definition: DataProviderPipeline.qc.dox.h:60
*code debug_log
Debug log closure; takes a single format string and then arguments for format placeholders.
Definition: DataProviderPipeline.qc.dox.h:249
const PS_RUNNING
Pipeline status: RUNNING.
Definition: DataProviderPipeline.qc.dox.h:40
Defines an abstract class for accepting data and outputting optionally transformed or filtered data...
Definition: AbstractDataProcessor.qc.dox.h:33
list< auto > elems()
Pipeline elements.
Condition flush_cond()
Flush condition variable.
Pipeline info.
Definition: DataProviderPipeline.qc.dox.h:47
int flush_waiting
Number of threads waiting on the flush cond.
Definition: DataProviderPipeline.qc.dox.h:135
date stop_time
run stop time (set in waitDone())
Definition: DataProviderPipeline.qc.dox.h:237
bool isProcessing()
Returns True if the pipeline is processing data.
bool aborting()
Returns True if the object is aborting.
abort(*bool ignore_exceptions)
Aborts execution of a pipeline in progress.
list< auto > list(...)
copy()
Copy constructor; creates an empty pipeline with the same configuration as the original.
list< hash< ExceptionInfo > > error_list
list of exceptions in pipelines
Definition: DataProviderPipeline.qc.dox.h:231
float recs_per_sec
Records processed per second end to end.
Definition: DataProviderPipeline.qc.dox.h:80
*code thread_callback
a closure or call reference for setting thread-local data in new pipeline queue threads ...
Definition: DataProviderPipeline.qc.dox.h:254
*string name
the name of the pipeline for logging purposes; if this key is not included, a generic name will be ge...
Definition: DataProviderPipeline.qc.dox.h:106
Defines the abstract class for data provider iterators; the destructor releases the iterator...
Definition: AbstractDataProviderBulkRecordInterface.qc.dox.h:33
Pipeline option info.
Definition: DataProviderPipeline.qc.dox.h:84
DataProviderPipeline parent
Parent object.
Definition: DataProviderPipeline.qc.dox.h:160
Defines a class for passing data through record processors.
Definition: DataProviderPipeline.qc.dox.h:207
bool abort_flag
Abort flag.
Definition: DataProviderPipeline.qc.dox.h:263
Pipeline element.
Definition: DataProviderPipeline.qc.dox.h:110
int data_waiting
Number of threads waiting on data.
Definition: DataProviderPipeline.qc.dox.h:132
reportError(PipelineQueue queue, hash< ExceptionInfo > ex)
Called from a pipeline queue object to report a fatal error durring processing.
destructor()
Destroys the object.
waitDone()
Waits for all queues to have processed remaining data.
submitIntern(auto _data)
Submits data for processing.
*code info_log
Info log closure; takes a single format string and then arguments for format placeholders.
Definition: DataProviderPipeline.qc.dox.h:243
submitData(AbstractIterator i)
Submits data for processing.
int queue_waiting
Number of threads waiting data to be removed from the queue.
Definition: DataProviderPipeline.qc.dox.h:129
checkUpdatePipelineIntern(int id)
Check if the given queue exists.
*date start_time
Start of processing.
Definition: DataProviderPipeline.qc.dox.h:52
logDebug(string fmt)
Logs to the debug log, if set.
list< auto > queue
Data queue.
Definition: DataProviderPipeline.qc.dox.h:138
checkSubmitIntern()
Throws an exception if the pipeline cannot be used; locks the pipeline for changes otherwise...
checkLockedIntern()
Throws an exception if the pipeline is locked.
*code debug_log
a closure or call reference for debug logging
Definition: DataProviderPipeline.qc.dox.h:88
bool hasQueue(int id)
Returns True if the given queue exists, False if not.
constructor(*hash< PipelineOptionInfo > opts)
Creates the object with the given options.
Sequence seq(1)
Pipeline ID sequence generator.
hash< PipelineInfo > getInfo()
Returns pipeline info.
int record_count
Number of input records submitted.
Definition: DataProviderPipeline.qc.dox.h:63
Counter cnt
Parent counter.
Definition: DataProviderPipeline.qc.dox.h:120
throwPipelineException()
Throws an exception if errors occured in background pipeline processing.
run(Counter run_cnt)
Processing thread.
stopInternUnlocked()
Stops all background pipeline queues; lock must be held.
Qore AbstractDataField class definition.
Definition: AbstractDataField.qc.dox.h:27
*code thread_callback
a closure or call reference for setting thread-local data in new pipeline queue threads ...
Definition: DataProviderPipeline.qc.dox.h:103
int record_count
Record count.
Definition: DataProviderPipeline.qc.dox.h:240
logInfo(string fmt)
Logs to the info log, if set.
stopIntern()
Stops all background pipeline queues.
bool stopping()
Returns True if the object is stopping.
Counter cnt()
Thread counter.
*code error_log
a closure or call reference for error logging
Definition: DataProviderPipeline.qc.dox.h:93
submit(auto qdata)
Submits data for processing.
bool data_flushed
Data flushed confirmation.
Definition: DataProviderPipeline.qc.dox.h:157
resetIntern()
Resets the pipeline.
submitDataIntern(auto _data)
Submits a single record for processing.
submitBulkIntern(AbstractDataProviderBulkRecordInterface i)
Submits bulk data for processing.
constructor(DataProviderPipeline parent, Mutex lck, Counter cnt, int id, int size)
Creates the object.
Mutex lck
Parent lock.
Definition: DataProviderPipeline.qc.dox.h:117
registerThread(PipelineQueue queue)
Registers a new thread.