forked from facebook/buck
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathslice_trace.py
More file actions
81 lines (66 loc) · 2.66 KB
/
slice_trace.py
File metadata and controls
81 lines (66 loc) · 2.66 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
# Copyright (c) Facebook, Inc. and its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, division, print_function, unicode_literals
import argparse
import codecs
import collections
import json
import math
import os
def parse_args():
description = """Slice JSON .trace file into smaller pieces."""
parser = argparse.ArgumentParser(description=description)
parser.add_argument("trace_file", help="Path to trace file.")
parser.add_argument("--slice_size", help="Size of slice in Mb.", default=230)
parser.add_argument(
"--output_name",
help="Format of slices, {slice} is the slice index.",
default="{orig}_{slice}.trace",
)
parser.add_argument(
"--context_events_count",
help="Number of events to carry over between slices for context.",
default=50,
)
return parser.parse_args()
def dump_slice(args, slice_index, event_list):
orig_file, _ = os.path.splitext(args.trace_file)
slice_file_name = args.output_name.format(slice=slice_index, orig=orig_file)
print("Writing slice to", slice_file_name)
with codecs.open(slice_file_name, "w", "utf-8") as slice_file:
json.dump(event_list, slice_file)
def main():
args = parse_args()
slice_size_mb = int(args.slice_size)
context_events_count = int(args.context_events_count)
trace_size_mb = os.path.getsize(args.trace_file) / (1024 * 1024)
slice_ratio = min(1, slice_size_mb / float(trace_size_mb))
assert slice_ratio > 0
with codecs.open(args.trace_file, "r", "utf-8") as trace:
data = json.load(trace)
# Accurate "enough". The default slice size is less than 256MB.
events_per_trace = int(len(data) * slice_ratio)
print(
"Total events in trace file:", len(data), "Events per trace", events_per_trace
)
slice_index = 0
start_index = 0
while start_index < len(data):
i = max(0, start_index - context_events_count)
j = start_index + events_per_trace + context_events_count
dump_slice(args, slice_index, data[i:j])
slice_index += 1
start_index += events_per_trace
if __name__ == "__main__":
main()