-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathjson_to_csv.py
More file actions
176 lines (149 loc) · 8.16 KB
/
json_to_csv.py
File metadata and controls
176 lines (149 loc) · 8.16 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
import json
import pandas as pd
import moment
import numpy as np
import sys
# This script extracts data from the downloaded JSON file for the experiment Reflective Learning in Organizations
# and converts it into a CSV file, with each column containing answers to each of the questions
# How to run the script:
# python json_to_csv <experiment_id>
experiment_id = sys.argv[1]
results_folder = f'results/{experiment_id}'
with open(results_folder + '/data.json', 'r') as f:
all_data = json.load(f)
next_states = {
'setupQuestions': 'setup',
'Onboarding': 'onboarding',
'Morning-Goals-All': 'goal',
'Pre-Reflection': 'reflection',
'Int-Reflection': 'reflection',
'Post-Test': 'post',
"Pre-Test": "pre",
"Pre-Test-2": "mid",
"Goal-Setting": 'reflection',
'Follow-Up': "followup",
"Update-Times": "update"
}
def extract_continuous_interactions(allAnswers):
# Possible continuous interactions: setup, onboarding, goal-setting, reflection, post-test
interactions = []
current_state = 'start' # setup, onboarding, goal, reflection, post
current_interaction = {}
for answer in allAnswers:
category = answer['qId'].split('.')[0]
givenAnswer = answer['answer'][0]
new_state = next_states[category]
if new_state != current_state:
if 'end' not in current_interaction and 'start' in current_interaction:
current_interaction['end'] = current_interaction['start']
interactions.append(current_interaction)
current_interaction = {
'type': new_state,
'start': answer
}
else:
if current_interaction['start']['answer'][0] in ["[No Response]", "[Repeat Question]"]:
current_interaction['start'] = answer
current_interaction['end'] = answer
current_state = new_state
if 'end' not in current_interaction and 'start' in current_interaction:
current_interaction['end'] = current_interaction['start']
interactions.append(current_interaction)
interactions.pop(0)
return interactions
participants = all_data['participants']
times = []
# Dictionary that has a key for every unique Id
# Each value is also a dictionary, that contains information about the participant and their interactions
part_info_dict = {}
def add_to_day(stages, stage, day, key, value):
if stage not in stages:
stages[stage] = {}
if day not in stages[stage]:
stages[stage][day] = {}
if key in stages[stage][day]:
stages[stage][day][key] += "|" + str(value)
else:
stages[stage][day][key] = str(value).replace("\n",".")
irrelevant_qids = ["readyToStart", "firstRelationshipPrompt", "askAddTaskGoals", "anythingElse", "wantRelationshipGoal",
"continueFromInfo", "continueFromExample", "continueFromRelInfo", "continueFromRelExample",
"addGoalsLater", "askReflectTaskGoals","wantContinue"]
irrelevant_categories = ["setupQuestions"]
df_columns = ["uniqueId", "teamName", "condition", "timezone", "morningTime", "eveningTime",
"stageName", "stageDay",
"goalSetComplete", "goalSetStart", "goalSetLength(s)",
"wantGoalInfo", "wantGoalExample", "firstTaskGoal", "addTaskGoals", "addWorkGoalsLater",
"wantRelGoalInfo", "wantRelGoalExample", "addFirstRelGoal", "addRelGoals", "addRelGoalsLater",
"reflectionComplete", "reflectionStart", "reflectionLength(s)",
"relGoalsProgress", "relMoreWork", "relImpact", "relEmotions", "relPursuit", "relRelevance",
"day1RelPursuit", "day1RelRelevance", "day1RelMetaDescription", "day1RelMetaJudgement",
"dayNRelPursuit", "dayNRelRelevance", "dayNRelMetaDescription", "dayNRelWhatChanged","dayNRelOtherChange",
"dayNRelChangeMoreRelevant", "dayNRelChangeLessRelevant", "dayNRelChangeEquallyRelevant",
"dayNRelCanYouImprove", "dayNRelChangeIdeas", "dayNRelChangeSuggestions",
"relImproveGoalSetting", "relHowImprove", "relPlan", "relPlanRemember",
"taskGoalsProgress", "moreWork", "impact", "emotions", "pursuitSatisfaction", "goalsImportance",
"day1Pursuit", "day1Relevance", "day1MetaDescription", "day1MetaJudgement",
"dayNPursuit", "dayNRelevance", "dayNMetaDescription", "dayNWhatChanged","dayNOtherChange",
"dayNChangeMoreRelevant", "dayNChangeLessRelevant", "dayNChangeEquallyRelevant",
"dayNCanYouImprove", "dayNChangeIdeas", "dayNChangeSuggestions",
"improveGoalSetting", "howImprove", "plan", "planRemember",
"survey"
]
for part in participants:
key = part['uniqueId']
if len(part['stages']['activity']) > 0:
# if part['parameters']['PID'] == 'sribug': continue
part_info_dict[key] = { "userData": {}, "stages": {}}
part_info_dict[key]["userData"]["timezone"] = part['parameters']['timezone']
part_info_dict[key]["userData"]['morningTime'] = part['parameters']['morningTime']
part_info_dict[key]["userData"]['eveningTime'] = part['parameters']['eveningTime']
part_info_dict[key]["userData"]['teamName'] = part['parameters']['PID']
part_info_dict[key]["userData"]['condition'] = part['conditionName']
part_info_dict[key]['stages'] = {}
all_interactions = extract_continuous_interactions(part['answers'])
for interaction in all_interactions:
interaction_delta = moment.date(interaction['end']['answerTimeStamp']) - moment.date(interaction['start']['answerTimeStamp'])
stageName = interaction['start']['stageName']
stageDay = interaction['start']['stageDay']
if interaction['type'] == "goal":
complete = interaction['end']['answer'][0] != "[No Response]"
add_to_day(part_info_dict[key]['stages'], stageName, stageDay, "goalSetComplete", complete)
add_to_day(part_info_dict[key]['stages'], stageName, stageDay, "goalSetStart", interaction['start']['answerTimeStamp'])
add_to_day(part_info_dict[key]['stages'], stageName, stageDay, "goalSetLength(s)", interaction_delta.seconds)
elif interaction['type'] == "reflection":
complete = interaction['end']['answer'][0] != "[No Response]"
add_to_day(part_info_dict[key]['stages'], stageName, stageDay, "reflectionComplete", complete)
add_to_day(part_info_dict[key]['stages'], stageName, stageDay, "reflectionStart", interaction['start']['answerTimeStamp'])
add_to_day(part_info_dict[key]['stages'], stageName, stageDay, "reflectionLength(s)", interaction_delta.seconds)
for answer in part['answers']:
category, qId = answer['qId'].split('.')
if qId not in irrelevant_qids and category not in irrelevant_categories:
add_to_day(part_info_dict[key]['stages'], answer['stageName'], answer['stageDay'], qId, '|'.join(answer['answer']))
df_rows = []
stage_should_days = {
"Onboarding":1,
"Pre-Test":1, "Goal-Setting":2, "Pre-Test-2":1, "Intervention":2, "Post-Test":1, "Follow-Up":1
}
for uniqueId, part in part_info_dict.items():
for stage in stage_should_days.keys():
if stage in part["stages"]:
should_days = max(part["stages"][stage].keys())
else:
should_days = stage_should_days[stage]
days = list(range(1,should_days + 1))
for day in days:
row_dict = {}
if stage in part["stages"] and day in part["stages"][stage]:
row_dict = part["stages"][stage][day].copy()
row_dict["uniqueId"] = uniqueId
row_dict["stageDay"] = day
row_dict["stageName"] = stage
row_dict.update(part["userData"])
df_rows.append(row_dict)
csv_df = pd.DataFrame.from_records(df_rows, columns=df_columns)
display_cols = ["uniqueId", "stageName", "stageDay", "reflectionComplete", "reflectionStart", "reflectionLength(s)","survey"]
#print(json.dumps(part_info_dict, sort_keys=False, indent=4))
print(pd.unique(csv_df.uniqueId))
print(csv_df[display_cols])
print(len(csv_df))
csv_df.to_csv(results_folder + '/csv_data.csv')