Skip to content

Commit 887988a

Browse files
dtmilanodtmilano
committed
Added voice search support
 - Reformatted - Renamed variables
1 parent 0436af0 commit 887988a

2 files changed

Lines changed: 119 additions & 35 deletions

File tree

‎src/com/dtmilano/android/concertina.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,22 @@
1919
2020
'''
2121
import random
22+
import subprocess
23+
import platform
24+
import sys
25+
import time
2226

2327
__author__ = 'diego'
2428
__version__ = '10.6.1'
2529

30+
DEBUG = True
31+
2632
class Concertina:
33+
osName = platform.system()
34+
''' The OS name. We sometimes need specific behavior. '''
35+
isLinux = (osName == 'Linux')
36+
isDarwin = (osName == 'Darwin')
37+
2738
PHRASES = [
2839
"Chicken Wings grow on trees",
2940
"Carrot sticks help the mind think",
@@ -212,6 +223,22 @@ def __init__(self):
212223
def getRandomText():
213224
return random.choice(Concertina.PHRASES)
214225

226+
@staticmethod
227+
def sayRandomText():
228+
text = Concertina.getRandomText()
229+
if Concertina.isLinux:
230+
if DEBUG:
231+
print >> sys.stderr, 'Saying "%s" using festival' % text
232+
pipe = subprocess.Popen(['/usr/bin/festival'])
233+
pipe.communicate('(SayText "%s")' % text)
234+
pipe.terminate()
235+
elif Concertina.isDarwin:
236+
if DEBUG:
237+
print >> sys.stderr, 'Saying "%s"' % text
238+
#subprocess.check_call(['/usr/bin/say', 'OK Google'])
239+
#time.sleep(2)
240+
subprocess.check_call(['/usr/bin/say', text])
241+
215242
@staticmethod
216243
def getRandomEmail():
217244
return random.choice(Concertina.EMAILS)

‎src/com/dtmilano/android/culebron.py‎

Lines changed: 92 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ def __init__(self, vc, device, serialno, printOperation, scale=1, concertina=Fal
219219
self.statusBar.grid(row=5, column=1, columnspan=2)
220220
self.statusBar.set("Always press F1 for help")
221221
self.window.update_idletasks()
222-
self.targetIds = []
222+
self.markedTargetIds = {}
223223
self.isTouchingPoint = self.vc is None
224224
self.coordinatesUnit = Unit.DIP
225225
self.permanentlyDisableEvents = False
@@ -665,7 +665,7 @@ def touchPoint(self, x, y):
665665
time.sleep(5)
666666
self.isTouchingPoint = self.vc is None
667667
self.takeScreenshotAndShowItOnWindow()
668-
#self.hideVignette()
668+
# self.hideVignette()
669669
self.statusBar.clear()
670670
return
671671

@@ -697,7 +697,7 @@ def longTouchPoint(self, x, y):
697697
time.sleep(5)
698698
self.isLongTouchingPoint = False
699699
self.takeScreenshotAndShowItOnWindow()
700-
#self.hideVignette()
700+
# self.hideVignette()
701701
self.statusBar.clear()
702702
return
703703

@@ -1111,7 +1111,7 @@ def markTargets(self):
11111111
print >> sys.stderr, " marktargets: targets=", self.targets
11121112
colors = ["#ff00ff", "#ffff00", "#00ffff"]
11131113

1114-
self.targetIds = []
1114+
self.markedTargetIds = {}
11151115
c = 0
11161116
for (x1, y1, x2, y2) in self.targets:
11171117
if DEBUG:
@@ -1125,17 +1125,22 @@ def markTarget(self, x1, y1, x2, y2, color='#ff00ff'):
11251125
@return the id of the rectangle added
11261126
'''
11271127

1128-
self.areTargetsMarked = True
1129-
return self.targetIds.append(
1130-
self.canvas.create_rectangle(x1 * self.scale, y1 * self.scale, x2 * self.scale, y2 * self.scale, fill=color,
1131-
stipple="gray25"))
1128+
#self.areTargetsMarked = True
1129+
_id = self.canvas.create_rectangle(x1 * self.scale, y1 * self.scale, x2 * self.scale, y2 * self.scale,
1130+
fill=color,
1131+
stipple="gray25")
1132+
self.markedTargetIds[_id] = (x1, y1, x2, y2)
1133+
return _id
1134+
1135+
def unmarkTarget(self, _id):
1136+
self.canvas.delete(_id)
11321137

11331138
def unmarkTargets(self):
11341139
if not self.areTargetsMarked:
11351140
return
1136-
for t in self.targetIds:
1137-
self.canvas.delete(t)
1138-
self.targetIds = []
1141+
for (_id, _) in self.markedTargetIds:
1142+
self.unmarkTarget(_id)
1143+
self.markedTargetIds = {}
11391144
self.areTargetsMarked = False
11401145

11411146
def setDragDialogShowed(self, showed):
@@ -1146,11 +1151,15 @@ def setDragDialogShowed(self, showed):
11461151
self.isGrabbingTouch = False
11471152

11481153
def drawTouchedPoint(self, x, y):
1154+
if DEBUG_CONCERTINA:
1155+
print >> sys.stderr, "drawTouchedPoint(", x, ",", y, ")"
11491156
size = 50
11501157
return self.canvas.create_oval((x - size) * self.scale, (y - size) * self.scale, (x + size) * self.scale,
11511158
(y + size) * self.scale, fill=Color.MAGENTA)
11521159

11531160
def drawDragLine(self, x0, y0, x1, y1):
1161+
if DEBUG_CONCERTINA:
1162+
print >> sys.stderr, "drawDragLine(", x0, ",", y0, ",", x1, ",", y1, ")"
11541163
width = 15
11551164
return self.canvas.create_line(x0 * self.scale, y0 * self.scale, x1 * self.scale, y1 * self.scale, width=width,
11561165
fill=Color.MAGENTA, arrow="last", arrowshape=(50, 50, 30), dash=(50, 25))
@@ -1244,37 +1253,46 @@ def concertinaLoopCallback(self, dontinteract=False):
12441253
k = random.choice(['ENTER', 'BACK', 'HOME', 'MENU'])
12451254
if DEBUG_CONCERTINA:
12461255
print >> sys.stderr, "CONCERTINA: key=" + k
1256+
# DEBUG ONLY!
1257+
# print >> sys.stderr, "Not sending key event"
12471258
self.command(k)
12481259
else:
12491260
# Act on views
1250-
l = len(self.targetViews)
1251-
if l > 0:
1261+
_len = len(self.targetViews)
1262+
if _len > 0:
12521263
i = random.randrange(len(self.targetViews))
1253-
t = self.targetViews[i]
1264+
target = self.targetViews[i]
12541265
z = self.targets[i]
12551266
if DEBUG_CONCERTINA:
1256-
print >> sys.stderr, "CONCERTINA: selected", unicode(t.__smallStr__())
1267+
print >> sys.stderr, "CONCERTINA: selected", unicode(target.__smallStr__())
12571268
print >> sys.stderr, "CONCERTINA: selected", z
1258-
self.markTarget(*z)
1269+
_id = self.markTarget(*z)
12591270
self.window.update_idletasks()
12601271
time.sleep(1)
1261-
# FIXME: we need self.unmakeTarget(*z)
1262-
self.unmarkTargets()
1263-
clazz = t.getClass()
1264-
parent = t.getParent()
1272+
self.unmarkTarget(_id)
1273+
self.window.update_idletasks()
1274+
clazz = target.getClass()
1275+
parent = target.getParent()
12651276
if parent:
12661277
parentClass = parent.getClass()
12671278
else:
12681279
parentClass = None
1269-
isScrollable = t.isScrollable()
1280+
isScrollable = target.isScrollable()
12701281
if DEBUG_CONCERTINA:
12711282
print >> sys.stderr, "CONCERTINA: is scrollable: ", isScrollable
12721283
if parent:
12731284
print >> sys.stderr, "CONCERTINA: is scrollable parent: ", parent.isScrollable()
1285+
cond = (isScrollable or parent.isScrollable() or parentClass == 'android.widget.ScrollView')
1286+
# DEBUG ONLY!
1287+
# print >> sys.stderr, "CONCERTINA: check:", cond
1288+
# if not cond:
1289+
# self.window.after(500, self.concertinaLoopCallback)
1290+
# return
12741291
if clazz == 'android.widget.EditText':
1275-
id = t.getId()
1276-
txt = t.getText()
1277-
if t.isPassword() or re.search('password', id, re.IGNORECASE) or re.search('password', txt, re.IGNORECASE):
1292+
id = target.getId()
1293+
txt = target.getText()
1294+
if target.isPassword() or re.search('password', id, re.IGNORECASE) or re.search('password', txt,
1295+
re.IGNORECASE):
12781296
text = Concertina.getRandomPassword()
12791297
elif re.search('email', id, re.IGNORECASE) or re.search('email', txt, re.IGNORECASE):
12801298
text = Concertina.getRandomEmail()
@@ -1284,27 +1302,57 @@ def concertinaLoopCallback(self, dontinteract=False):
12841302
print >> sys.stderr, "Entering text: ", text
12851303
if not text:
12861304
raise RuntimeError('text is None')
1287-
self.setText(t, text)
1288-
elif isScrollable or parent.isScrollable() or parentClass == 'android.widget.ScrollView':
1305+
self.setText(target, text)
1306+
elif target.getContentDescription() in ['Voice Search', 'Tap to speak']:
1307+
Concertina.sayRandomText()
1308+
time.sleep(5)
1309+
elif random.choice(['SCROLL', 'TOUCH']) == 'SCROLL' and (isScrollable or parent.isScrollable() or parentClass == 'android.widget.ScrollView'):
12891310
# NOTE: The order here is important because some EditText are inside ScrollView's and we want to
12901311
# capture the case of other ScrollViews
1312+
if isScrollable:
1313+
((l, t), (r, b)) = target.getBounds()
1314+
else:
1315+
if DEBUG_CONCERTINA:
1316+
print >> sys.stderr, "CONCERTINA: using parent bounds because it's scrollable"
1317+
((l, t), (r, b)) = parent.getBounds()
12911318
if DEBUG_CONCERTINA:
1292-
print >> sys.stderr, "CONCERTINA: bounds=", t.getBounds()
1293-
((t, l), (b, r)) = t.getBounds()
1319+
print >> sys.stderr, "CONCERTINA: bounds=", ((l, t), (r, b))
12941320
if random.choice(['VERTICAL', 'HORIZONTAL']) == 'VERTICAL':
1295-
sp = (l+(r-l)/2, t+50)
1296-
ep = (l+(r-l)/2, b-50)
1321+
if DEBUG_CONCERTINA:
1322+
print >> sys.stderr, 'VERTICAL'
1323+
sp = (l + (r - l) / 2, t + 50)
1324+
ep = (l + (r - l) / 2, b - 50)
12971325
else:
1298-
sp = (l+50, t+(b-t)/2)
1299-
ep = (r-50, t+(b-t)/2)
1300-
d = 1
1326+
if DEBUG_CONCERTINA:
1327+
print >> sys.stderr, 'HORIZONTAL'
1328+
sp = (l + 50, t + (b - t) / 2)
1329+
ep = (r - 50, t + (b - t) / 2)
1330+
if random.choice(['FORWARD', 'REVERSE']) == 'REVERSE':
1331+
if DEBUG_CONCERTINA:
1332+
print >> sys.stderr, 'REVERSE'
1333+
temp = sp
1334+
sp = ep
1335+
ep = temp
1336+
else:
1337+
if DEBUG_CONCERTINA:
1338+
print >> sys.stderr, 'FORWARD'
1339+
d = 500
13011340
s = 20
1302-
units = Unit.DIP
1341+
_id = self.canvas.create_rectangle(l * self.scale, t * self.scale, r * self.scale,
1342+
b * self.scale,
1343+
fill="#00ffff", stipple="gray12")
1344+
self.window.update_idletasks()
1345+
units = Unit.PX
1346+
self.drawTouchedPoint(sp[0], sp[1])
1347+
self.window.update_idletasks()
1348+
self.drawDragLine(sp[0], sp[1], ep[0], ep[1])
1349+
self.window.update_idletasks()
1350+
time.sleep(5)
13031351
if DEBUG_CONCERTINA:
13041352
print >> sys.stderr, "CONCERTINA: dragging %s %s %s %s %s" % (sp, ep, d, s, units)
13051353
self.drag(sp, ep, d, s, units)
13061354
else:
1307-
self.touchView(t)
1355+
self.touchView(target)
13081356
self.printOperation(None, Operation.SLEEP, Operation.DEFAULT)
13091357
time.sleep(5)
13101358
self.takeScreenshotAndShowItOnWindow()
@@ -1372,6 +1420,7 @@ def onShowViewDetailsChanged(self):
13721420
else:
13731421
self.culebron.hideViewDetails()
13741422

1423+
13751424
class ViewTree(Tkinter.Frame):
13761425
def __init__(self, parent):
13771426
Tkinter.Frame.__init__(self, parent)
@@ -1418,6 +1467,7 @@ def tag_bind(self, tagname, sequence=None, callback=None):
14181467
print >> sys.stderr, 'ViewTree.tag_bind(', tagname, ',', sequence, ',', callback, ')'
14191468
return self.viewTree.tag_bind(tagname, sequence, callback)
14201469

1470+
14211471
class ViewDetails(Tkinter.Frame):
14221472
VIEW_DETAILS = "View Details:\n"
14231473

@@ -1431,6 +1481,7 @@ def __init__(self, parent):
14311481
def set(self, view):
14321482
self.label.configure(text=self.VIEW_DETAILS + view.__str__())
14331483

1484+
14341485
class StatusBar(Tkinter.Frame):
14351486

14361487
def __init__(self, parent):
@@ -1446,6 +1497,7 @@ def clear(self):
14461497
self.label.config(text="")
14471498
self.label.update_idletasks()
14481499

1500+
14491501
class LabeledEntry():
14501502
def __init__(self, parent, text, validate, validatecmd):
14511503
self.f = Tkinter.Frame(parent)
@@ -1463,12 +1515,14 @@ def set(self, text):
14631515
self.entry.delete(0, Tkinter.END)
14641516
self.entry.insert(0, text)
14651517

1518+
14661519
class LabeledEntryWithButton(LabeledEntry):
14671520
def __init__(self, parent, text, buttonText, command, validate, validatecmd):
14681521
LabeledEntry.__init__(self, parent, text, validate, validatecmd)
14691522
self.button = Tkinter.Button(self.f, text=buttonText, command=command)
14701523
self.button.grid(row=1, column=3)
14711524

1525+
14721526
class DragDialog(Tkinter.Toplevel):
14731527

14741528
DEFAULT_DURATION = 1000
@@ -1656,6 +1710,7 @@ def cleanUp(self):
16561710
self.__cleanUpSpId()
16571711
self.__cleanUpEpId()
16581712

1713+
16591714
class ContextMenu(Tkinter.Menu):
16601715
# FIXME: should get rid of the nested classes, otherwise it's not possible to create a parent class
16611716
# SubMenu for UiScrollableSubMenu
@@ -1780,6 +1835,7 @@ def showPopupMenu(self, event):
17801835
# self.grab_release()
17811836
pass
17821837

1838+
17831839
class HelpDialog(Tkinter.Toplevel):
17841840

17851841
def __init__(self, culebron):
@@ -1839,6 +1895,7 @@ def onDismiss(self, event=None):
18391895
self.culebron.canvas.focus_set()
18401896
self.destroy()
18411897

1898+
18421899
class FileDialog():
18431900
def __init__(self, culebron, filename):
18441901
self.parent = culebron.window

0 commit comments

Comments
 (0)