1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
|
[global]
disabled = false
debug = false
verbosity = false
spoolDir = $SPLUNK_HOME/var/spool/splunk
spoolFile = <SAMPLE>
breaker = [^\r\n\s]+
mode = sample
sampletype = raw
interval = 60
delay = 0
timeMultiple = 1
count = 0
earliest = now
latest = now
randomizeCount = 0.2
randomizeEvents = false
outputMode = spool
fileMaxBytes = 10485760
fileBackupFiles = 5
splunkPort = 8089
splunkMethod = https
index = main
sourcetype = eventgen
host = 127.0.0.1
outputWorkers = 1
generator = default
rater = config
generatorWorkers = 1
timeField = _raw
threading = thread
profiler = false
maxIntervalsBeforeFlush = 3
maxQueueLength = 0
autotimestamps = [ <jsonlist> ]
autotimestamp = false
disableLoggingQueue = true
[<sample file name>]
* This stanza defines a given sample file contained within the samples directory.
* This stanza can be specified as a PCRE.
* Hardcoded to $SPLUNK_HOME/etc/apps/<app>/samples/<sample file name>.
* This stanza is only valid for the following replacementType -> replacement values:
* static -> <string>
* timestamp -> <strptime>
* replaytimestamp -> <strptime>
* random -> ipv4
* random -> ipv6
* random -> mac
* random -> integer[<start>:<end>]
* random -> float[<start.numzerosforprecision>:<end.numzerosforprecision>]
* random -> string(<integer>)
* random -> hex([integer])
* rated -> integer[<start>:<end>]
* rated -> float[<start.numzerosforprecision>:<end.numzerosforprecision>]
* file -> <replacment file name>
* mvfile -> <replacement file name, expects CSV file>:<column number>
disabled = true | false
* Like what it looks like. Will disable event generation for this sample.
sampleDir = <dir>
* Set a different directory to look for samples in.
* if sampleDir is not set, by default, eventgen will loads the sample files from <conf_file_dir>/../samples. conf_file_dir is the directory where eventgen conf file locates.
* You can set sampleDir to some relative path or some absolute path.
* When sampleDir is an absolute path, eventgen will load the sample files from the path directly. But absolute path is not recommended, absolute path makes your configuration not work with different OS.
* When sampleDir is a relative path, eventgen takes conf_file_dir as the base path when resolve sampleDir. For example, sampleDir=../../my_sample will be resolved to <conf_file_dir>/../../my_sample. conf_file_dir is the directory where eventgen conf file locates.
threading = thread | process
* Configurable threading model. Process uses multiprocessing.Process in Python to get around issues with the GIL.
* Defaults to thread
profiler = true | false
* Run eventgen with python profiler on
* Defaults to false
useOutputQueue = true | false
* Disable the use of the output Queue. The output queue functions as a reduce step when you need to maintain a single thread or a limited number of threads outputting data, for instance if you're outputting to a file or to stdout/modular input. Defaults to true. If you can multithread output, for example with splunkstream or s2s type outputs, setting this to false will give an order of magnitude or better performance improvement.
#############################
## OUTPUT RELATED SETTINGS ##
#############################
outputWorkers = <number of worker threads>
* Specifies how many threads or processes to stand up to handle output
* Generally if using TCP based outputs like splunkstream, more could be required
* Defaults to 1
outputMode = modinput | s2s | file | splunkstream | stdout | devnull | spool | httpevent | syslogout | tcpout | udpout
* Specifies how to output log data. Modinput is default.
* If setting spool, should set spoolDir
* If setting file, should set fileName
* If setting splunkstream, should set splunkHost, splunkPort, splunkMethod, splunkUser and splunkPassword if not Splunk embedded
* If setting s2s, should set splunkHost and splunkPort
* If setting syslogout, should set syslogDestinationHost and syslogDestinationPort
syslogDestinationHost = <host>
* Defaults to 127.0.0.1
syslogDestinationPort = <port>
* Defaults to port 1514
* Only supports UDP ports
syslogAddHeader = true | false
* Controls whether syslog messages should be prefixed with an RFC3164 compliant header
including the host value defined for the sample.
* Useful in situations where you want to output generated events to syslog and make it
possible for the receiving syslog server to use the sample's defined host value instead of
the hostname of the host that eventgen is running on.
* Defaults to false
tcpDestinationHost = <host>
* Defaults to 127.0.0.1
tcpDestinationPort = <port>
* Defaults to port 3333
udpDestinationHost = <host>
* Defaults to 127.0.0.1
udpDestinationPort = <port>
* Defaults to port 3333
httpeventServers = <valid json>
* valid json that contains a list of server objects
* valid server objects contain a protocol, a address, a port and a session key
* {"servers":[{ "protocol":"https", "address":"127.0.0.1", "port":"8088", "key":"12345-12345-123123123123123123"}]}
httpeventOutputMode = roundrobin | mirror
* in roundrobin mode, the HEC/Battlecat plugin will output to a random server out of the server pool
* in mirror moded, HEC/Battlecat plugin will mirror the event to every server specified in the server pool
httpeventMaxPayloadSize = <int>
* the max payload size that is currently configured for HTTP event
httpeventWaitResponse = <bool>
* wait for all responses on a generator output before returning the outputter. Defaults to true.
spoolDir = <spool directory>
* Spool directory is the generated files destination directory.
* Only valid in spool outputMode.
* Windows separators should contain double forward slashes '\\' (i.e. $SPLUNK_HOME\\var\\spool\\splunk).
* Unix separators will work on Windows and vice-versa.
* Defaults to $SPLUNK_HOME/var/spool/splunk
spoolFile = <spool file name>
* Spool file is the generated files name.
* Not valid if stanza is a pattern.
* Defaults to <SAMPLE> (sample file name).
fileName = </path/to/file>
* Should set the full path
* Uses a rotating file handler which will rotate the file at a certain size, by default 10 megs
and will by default only save 5 files. See fileMaxBytes and fileBackupFiles
fileMaxBytes = <size in bytes>
* Will rotate a file output at this given size
* Defaults to 10 Megabytes (10485760 bytes)
fileBackupFiles = <number of files>
* Will keep this number of files (.1, .2, etc) after rotation
* Defaults to 5
splunkHost = <host> | <json list of hosts>
* If you specify just one host, will only POST to that host, if you specify a JSON list,
it will POST to multiple hosts in a random distribution. This allows us from one eventgen to
feed an entire cluster of Splunk indexers without needing forwarders.
* JSON list should look like [ "host.name", "host2.name" ]
splunkPort = <port>
* Defaults to the default Splunk management port 8089
splunkMethod = http | https
* Defaults to https
splunkUser = <user>
* User with rights to post to REST endpoint receivers/stream
splunkPass = <pass>
* Password for SplunkUser
projectID = <id>
* Project ID for Splunk Storm
accessToken = <accesstoken>
* Access Token for Splunk Storm
index = <index>
* ONLY VALID WITH outputMode SPLUNKSTREAM
* Splunk index to write events to. Defaults to main if none specified.
source = <source>
* Valid with outputMode=modinput (default) & outputMode=splunkstream & outputMode=httpevent
* Set event source in Splunk to <source>. Defaults to sample file name if none specified.
sourcetype = <sourcetype>
* Valid with outputMode=modinput (default) & outputMode=splunkstream & outputMode=httpevent
* Set event sourcetype in Splunk to <source> Defaults to 'eventgen' if none specified.
host = <host>
* ONLY VALID WITH outputMode SPLUNKSTREAM and SYSLOGOUT
* When outputMode is splunkstream, set event host in Splunk to <host>.
* When outputMode is syslogout and syslogAddHeader is set to true, add initial header with hostname <host>,
see syslogAddHeader for details.
* Defaults to 127.0.0.1 if none specified.
hostRegex = <hostRegex>
* ONLY VALID WITH outputMode SPLUNKSTREAM
* Allows setting the event host via a regex from the actual event itself. Only used if host not set.
maxIntervalsBeforeFlush = <intervals before flushing queue>
* Number of intervals before flushing the queue if the queue hasn't filled to maxQueueLength
* Defaults to 3
maxQueueLength = <maximum items before flushing the queue>
* Number of items before flushing the output queue
* Default is per outputMode specific
###############################
## EVENT GENERATION SETTINGS ##
###############################
generator = default | <plugin>
* Specifies the generator plugin to use. Default generator will give behavior of eventgen pre-3.0
which exclusively uses settings in eventgen.conf to control behavior. Generators in 3.0 are now
pluggable python modules which can be custom code.
generatorWorkers = <number of generator threads>
* Specifies how many threads to use to generate events
* Defaults to 1
rater = config | <plugin>
* Specifies which rater plugin to use. Default rater uses hourOfDayRate, etc, settings to specify
how to affect the count of events being generated. Raters in 3.0 are now pluggable python modules.
mode = sample | replay
* Default is sample, which will generate count (+/- rating) events every configured interval
* Replay will instead read the file and leak out events, replacing timestamps,
splitSample = <integer>
* only works with mode sample
* default value set to 0
* Value of 1 will default to number of threads / processes enabled
* some generators may not have the ability split threads and guarantee transaction order.
sampletype = raw | csv
* Raw are raw events (default)
* CSV are from an outputcsv or export from Splunk.
CSV allows you to override output fields for the sample like host, index, source and sourcetype
from the CSV file. Will read the raw events from a field called _raw. Assumes the CSV file has
a header row which defines field names.
OVERRIDES FOR DEFAULT FIELDS WILL ONLY WITH WITH outputMode SPLUNKSTREAM.
interval = <integer>
* Delay between exections. This number in replay mode occurs after the replay has finished.
* How often to generate sample (in seconds).
* 0 means disabled.
* Defaults to 60 seconds.
delay = <integer>
* Specifies how long to wait until we begin generating events for this sample
* Primarily this is used so we can stagger sets of samples which similar but slightly different data
* Defaults to 0 which is disabled.
autotimestamp = <boolean>
* Will enable autotimestamp feature which detects most common forms of timestamps in your samples with no configuration.
timeMultiple = <float>
* Only valid in mode = replay
* Will slow down the replay of events by <float> factor. For example, allows a 10 minute sample
to play out over 20 minutes with a timeMultiple of 2, or 60 minutes with a timeMultiple of 6.
By the converse, make timeMultiple 0.5 will make the events run twice as fast.
timeField = <field name>
* Only valid in mode = replay
* Will select the field to find the timestamp in. In many cases, time will come from a different
field in the CSV.
timezone = local | <integer>
* If set to 'local', will output local time, if set to '0000' will output UTC time
* Otherwise it must be a timezone offset like +hhmm or -hhmm, for example:
US Eastern Standard (EST) would be: timezone = -0500
US Pacific Daylight (PDT) would be: timezone = -0700
Indian Standard would be timezone = +0530
* Valid range +2359 to -2359 (The last two digits are MINUTES, so they should be within 0-59)
backfill = <time-str>
* Specified in Splunk's relative time language, used to set a time to backfill events
end = <time-str> | <integer>
* Will end execution on a specific time or a number of events
* Can be used to execute only a specified number of intervals or with backfill to generate events over a specific time window.
backfillSearch = <splunk search>
* If outputMode = splunkstream, this will run this search, appending '| head 1', and narrow the
backfill range specified with backfill to when the search has last seen events.
backfillSearchUrl = <url>
* Defaults to splunkMethod://splunkHost:splunkPort/, can override in case you're running
in a cluster.
count = <integer>
* Maximum number of events to generate per sample file
* 0 means replay the entire sample.
* Defaults to 0.
perDayVolume = <float>
* This is used in place of count. The perDayVolume is a size supplied in GB per Day. This value will allow
* eventgen to supply a target datavolume instead of a count for event generation.
* Defaults to Null
bundlelines = true | false
* For outside use cases where you need to take all the lines in a sample file and pretend they are
one event, but count = 0 will not work because you want to replay all the lines more than once.
Also, please note you can also use breaker=\r*\n\r*\n to break the sample file into multi-line
transactions that would work better than this as well. This is also useful where you want to bring
in sampletype = csv and bundle that multiple times.
* If bundlelines = true and the token replacementType is replaytimestamp, we will introduce some randomness
into the times between items in the transaction in microseconds.
* Will override any breaker setting.
hourOfDayRate = <json>
* Takes a JSON hash of 24 hours with float values to rate limit how many events we should see
in a given hour.
* Sample JSON:
{ "0": 0.05, "1": 0.05: "2": 0.07... }
* If a match is not found, will default to count events
* Also multiplied times dayOfWeekRate, minuteOfHourRate, dayOfMonthRate, monthOfYearRate
dayOfWeekRate = <json>
* Takes a JSON hash of 7 days of the week in Splunk format (0 is Sunday)
* Sample JSON:
{ "0": 0.55, "1": 0.97, "2": 0.95, "3": 0.90, "4": 0.97, "5": 1.0, "6": 0.99 }
* If a match is not found, will default to count events
* Also multiplied times hourOfDayRate, minuteOfHourRate, dayOfMonthRate, monthOfYearRate
minuteOfHourRate = <json>
* Takes a JSON hash of 60 minutes of an hour, starting with 0
* Sample JSON:
{ "0": 1, "2": 1...}
* If a match is not found, will default to count events
* Also multiplied times dayOfWeekRate, hourOfDateRate, dayOfMonthRate, monthOfYearRate
dayOfMonthRate = <json>
* Takes a JSON hash of 31 days of the month, starting with 1
* Sample JSON:
{ "1": 1, "2": 1...}
* If a match is not found, will default to count events
* Also multiplied times dayOfWeekRate, hourOfDateRate, minuteOfHourRate, monthOfYearRate
monthOfYearRate = <json>
* Takes a JSON hash of 60 minutes of an hour, starting with 0
* Sample JSON:
{ "0": 1, "2": 1...}
* If a match is not found, will default to count events
* Also multiplied times dayOfWeekRate, hourOfDateRate, minuteOfHourRate, dayOfMonthRate
randomizeCount = <float>
* Will randomize the number of events generated by percentage passed
* Example values: 0.2, 0.5
* Recommend passing 0.2 to give 20% randomization either way (plus or minus)
randomizeEvents = <boolean>
* Will randomize the events found in the sample file before choosing the events.
* NOT SUPPORTED WITH sampletype csv
* NOT SUPPORTED WITH mode = replay OR custom generators like generator = replay
breaker = <regular expression>
* NOT to be confused w/ props.conf LINE_BREAKER.
* PCRE used for flow control.
* If count > 0; data will be generated until number of discovered breakers <= "count".
* If breaker does not match in sample, one iteration of sample will be generated.
* Defaults to [^\r\n\s]+
earliest = <time-str>
* Specifies the earliest random time for generated events.
* If this value is an absolute time, use the dispatch.time_format to format the value.
* Defaults to now.
latest = <time-str>
* Specifies the latest random time for generated events.
* If this value is an absolute time, use the dispatch.time_format to format the value.
* Defaults to now.
#############################
## JINJA TEMPLATE SETTINGS ##
#############################
jinja_template_dir = <str>
* directory where jinja templates can be located.
* If it is not defined, default template directory is <sampleDir>/templates. sampleDir is the parameter which defines the sample file directory.
* You can set it as some relative path or absolute path.
* When it is set as the absolute path, eventgen loads the jinja template from this directory directly.
* When it is set as the relative path, eventgen uses sampleDir as the base when resolving the path. sampleDir is the parameter which defines the sample file directory. For example, if jinja_template_dir=../my_templates, it will be resolved to <sampleDir>/../my_templates.
jinja_target_template = <str>
* root template to load for all sample generation.
jinja_variables = <json>
* json value that contains a dict of kv pairs to pass as options to load inside of the jinja templating engine.
################################
## TOKEN REPLACEMENT SETTINGS ##
################################
token.<n>.token = <regular expression>
* 'n' is a number starting at 0, and increasing by 1.
* PCRE expression used to identify segment for replacement.
* If one or more capture groups are present the replacement will be performed on group 1.
* Defaults to None.
token.<n>.replacementType = static | timestamp | replaytimestamp | random | rated | file | mvfile | integerid
* 'n' is a number starting at 0, and increasing by 1. Stop looking at the filter when 'n' breaks.
* For static, the token will be replaced with the value specified in the replacement setting.
* For timestamp, the token will be replaced with the strptime specified in the replacement setting
* For replaytimestamp, the token will be replaced with the strptime specified in the replacement setting
but the time will not be based on earliest and latest, but will instead be replaced by looking at the
offset of the timestamp in the current event versus the first event, and then adding that time difference
to the timestamp when we started processing the sample. This allows for replaying events with a
new timestamp but to look much like the original transaction. Assumes replacement value is the same
strptime format as the original token we're replacing, otherwise it will fail. First timestamp will
be the value of earliest. NOT TO BE CONFUSED WITH REPLAY MODE. Replay mode replays a whole file
with timing to look like the original file. This will allow a single transaction to be replayed with some randomness.
* For random, the token will be replaced with a type aware value (i.e. valid IPv4 Address).
* For rated, the token will be replaced with a subset of random types (float, integer), which are
rated by hourOfDayRate and dayOfWeekRate.
* For file, the token will be replaced with a random value retrieved from a file specified in the replacement setting.
* For mvfile, the token will be replaced with a random value of a column retrieved from a file specified in the replacement setting. Multiple files can reference the same source file and receive different columns from the same random line.
* For integerid, will use an incrementing integer as the replacement.
* Defaults to None.
token.<n>.replacement = <string> | <strptime> | ["list","of","strptime"] | guid | ipv4 | ipv6 | mac | integer[<start>:<end>] | float[<start>:<end>] | string(<i>) | hex(<i>) | list["list", "of", "values"] | <replacement file name> | <replacement file name>:<column number> | <integer>
* 'n' is a number starting at 0, and increasing by 1. Stop looking at the filter when 'n' breaks.
* For <string>, the token will be replaced with the value specified.
* For <strptime>, a strptime formatted string to replace the timestamp with
* For ["list","of","strptime"], only used with replaytimestamp, a JSON formatted list of strptime
formats to try. Will find the replace with the same format which matches the replayed timestamp.
* For guid, the token will be replaced with a random GUID value.
* For ipv4, the token will be replaced with a random valid IPv4 Address (i.e. 10.10.200.1).
* For ipv6, the token will be replaced with a random valid IPv6 Address (i.e. c436:4a57:5dea:1035:7194:eebb:a210:6361).
* For mac, the token will be replaced with a random valid MAC Address (i.e. 6e:0c:51:c6:c6:3a).
* For integer[<start>:<end>], the token will be replaced with a random integer between
start and end values where <start> is a number greater than 0
and <end> is a number greater than 0 and greater than or equal to <start>. If rated,
will be multiplied times hourOfDayRate and dayOfWeekRate.
* For float[<start>:<end>], the token will be replaced with a random float between
start and end values where <end> is a number greater than or equal to <start>.
For floating point numbers, precision will be based off the precision specified
in <start>. For example, if we specify 1.0, precision will be one digit, if we specify
1.0000, precision will be four digits. If rated, will be multiplied times hourOfDayRate and dayOfWeekRate.
* For string(<i>), the token will be replaced with i number(s) of ASCII characters where 'i' is a number greater than 0.
* For hex(<i>), the token will be replaced with i number of Hexadecimal characters [0-9A-F] where 'i' is a number greater than 0.
* For list, the token will be replaced with a random member of the JSON list provided.
* For <replacement file name>, the token will be replaced with a random line in the replacement file.
* Replacement file name should be a fully qualified path (i.e. $SPLUNK_HOME/etc/apps/windows/samples/users.list).
* Windows separators should contain double forward slashes '\\' (i.e. $SPLUNK_HOME\\etc\\apps\\windows\\samples\\users.list).
* Unix separators will work on Windows and vice-versa.
* Column numbers in mvfile references are indexed at 1, meaning the first column is column 1, not 0.
* <integer> used as the seed for integerid.
* Defaults to None.
host.token = <regular expression>
* PCRE expression used to identify the host name (or partial name) for replacement.
* If one or more capture groups are present the replacement will be performed on group 1.
* Defaults to None.
host.replacement = <replacement file name> | <replacement file name>:<column number>
* For <replacement file name>, the token will be replaced with a random line in the replacement file.
* Replacement file name should be a fully qualified path (i.e. $SPLUNK_HOME/etc/apps/windows/samples/users.list).
* Windows separators should contain double forward slashes '\\' (i.e. $SPLUNK_HOME\\etc\\apps\\windows\\samples\\users.list).
* Unix separators will work on Windows and vice-versa.
* Column numbers in mvfile references are indexed at 1, meaning the first column is column 1, not 0.
* Defaults to None.
|
