Skip to content

Commit 1765c00

Browse files
iamhopaul123iamhopaul123
authored
feat(stack): add Firelens sidecar deployment support (#992)
<!-- Provide summary of changes --> Partially address #875. With this PR now we can deploy a Copilot service with Firelens as the log driver to deliver logs of the main container to destinations. Users can config Firelens in two ways: 1. Set up configurations in the manifest (unable to sent to multiple destinations though). 2. Configure using fluentbit config file. Note that this requires users to build their own custom fluentbit image and specifying their config file path. Additionally users might need to add permissions to task role if necessary. And they are able to add their own permissions by using Copilot addons feature. Any addons policy will be added to the task role. This PR also adjusts `svc logs` to accommodate Firelens changes. <!-- Issue number, if available. E.g. "Fixes #31", "Addresses #42, 77" --> By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
1 parent 2435d25 commit 1765c00

26 files changed

Lines changed: 211 additions & 135 deletions

File tree

‎e2e/addons/addons_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@ var _ = Describe("addons flow", func() {
208208

209209
for _, logLine := range svcLogs {
210210
Expect(logLine.Message).NotTo(Equal(""))
211-
Expect(logLine.TaskID).NotTo(Equal(""))
211+
Expect(logLine.LogStreamName).NotTo(Equal(""))
212212
Expect(logLine.Timestamp).NotTo(Equal(0))
213213
Expect(logLine.IngestionTime).NotTo(Equal(0))
214214
}

‎e2e/init/init_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ var _ = Describe("init flow", func() {
127127

128128
for _, logLine := range svcLogs {
129129
Expect(logLine.Message).NotTo(Equal(""))
130-
Expect(logLine.TaskID).NotTo(Equal(""))
130+
Expect(logLine.LogStreamName).NotTo(Equal(""))
131131
Expect(logLine.Timestamp).NotTo(Equal(0))
132132
Expect(logLine.IngestionTime).NotTo(Equal(0))
133133
}

‎e2e/internal/client/outputs.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ func toSvcListOutput(jsonInput string) (*SvcListOutput, error) {
6060
}
6161

6262
type SvcLogsOutput struct {
63-
TaskID string `json:"taskID"`
63+
LogStreamName string `json:"logStreamName"`
6464
IngestionTime int64 `json:"ingestionTime"`
6565
Timestamp int64 `json:"timestamp"`
6666
Message string `json:"message"`

‎e2e/multi-env-app/multi_env_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ var _ = Describe("Multiple Env App", func() {
198198

199199
for _, logLine := range svcLogs {
200200
Expect(logLine.Message).NotTo(Equal(""))
201-
Expect(logLine.TaskID).NotTo(Equal(""))
201+
Expect(logLine.LogStreamName).NotTo(Equal(""))
202202
Expect(logLine.Timestamp).NotTo(Equal(0))
203203
Expect(logLine.IngestionTime).NotTo(Equal(0))
204204
}

‎e2e/multi-svc-app/multi_svc_app_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ var _ = Describe("Multiple Service App", func() {
3131
Expect("./copilot").Should(BeADirectory())
3232
})
3333

34-
It("app ls includes new project", func() {
34+
It("app ls includes new application", func() {
3535
apps, err := cli.AppList()
3636
Expect(err).NotTo(HaveOccurred())
3737
Expect(apps).To(ContainSubstring(appName))
@@ -235,7 +235,7 @@ var _ = Describe("Multiple Service App", func() {
235235

236236
for _, logLine := range svcLogs {
237237
Expect(logLine.Message).NotTo(Equal(""))
238-
Expect(logLine.TaskID).NotTo(Equal(""))
238+
Expect(logLine.LogStreamName).NotTo(Equal(""))
239239
Expect(logLine.Timestamp).NotTo(Equal(0))
240240
Expect(logLine.IngestionTime).NotTo(Equal(0))
241241
}

‎e2e/multi-svc-app/www/main.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,15 @@ func SimpleGet(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
1414
w.Write([]byte("www"))
1515
}
1616

17+
func healthCheckHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
18+
log.Println("Health Check Succeeded")
19+
w.WriteHeader(http.StatusOK)
20+
w.Write([]byte("www"))
21+
}
22+
1723
func main() {
1824
router := httprouter.New()
25+
router.GET("/", healthCheckHandler)
1926
router.GET("/www/", SimpleGet)
2027
log.Fatal(http.ListenAndServe(":80", router))
2128
}

‎internal/pkg/aws/cloudwatchlogs/cloudwatchlogs.go‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ import (
1919
const (
2020
// SleepDuration is the sleep time for making the next request for log events.
2121
SleepDuration = 1 * time.Second
22+
23+
logStreamNamePrefix = "copilot/"
2224
)
2325

2426
var (
@@ -124,12 +126,8 @@ func (c *CloudWatchLogs) TaskLogEvents(logGroupName string, streamLastEventTime
124126
}
125127

126128
for _, event := range resp.Events {
127-
taskID, err := parseTaskID(*logStreamName)
128-
if err != nil {
129-
return nil, err
130-
}
131129
log := &Event{
132-
TaskID: taskID,
130+
LogStreamName: trimLogStreamName(*logStreamName),
133131
IngestionTime: aws.Int64Value(event.IngestionTime),
134132
Message: aws.StringValue(event.Message),
135133
Timestamp: aws.Int64Value(event.Timestamp),
@@ -171,11 +169,7 @@ func (c *CloudWatchLogs) LogGroupExists(logGroupName string) (bool, error) {
171169
return false, nil
172170
}
173171

174-
func parseTaskID(logStreamName string) (string, error) {
175-
// logStreamName example: ecs/{name}/1cc0685ad01d4d0f8e4e2c00d1775c56
176-
logStreamNameSplit := strings.Split(logStreamName, "/")
177-
if len(logStreamNameSplit) != 3 {
178-
return "", fmt.Errorf("cannot parse task ID from log stream name: %s", logStreamName)
179-
}
180-
return logStreamNameSplit[2], nil
172+
func trimLogStreamName(logStreamName string) string {
173+
// logStreamName example: copilot/{name}/1cc0685ad01d4d0f8e4e2c00d1775c56
174+
return strings.TrimPrefix(logStreamName, logStreamNamePrefix)
181175
}

‎internal/pkg/aws/cloudwatchlogs/cloudwatchlogs_test.go‎

Lines changed: 40 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,11 @@ func TestLogEvents(t *testing.T) {
4343
OrderBy: aws.String("LastEventTime"),
4444
}).Return(&cloudwatchlogs.DescribeLogStreamsOutput{
4545
LogStreams: []*cloudwatchlogs.LogStream{
46-
&cloudwatchlogs.LogStream{
47-
LogStreamName: aws.String("ecs/mockLogGroup/mockLogStream1"),
46+
{
47+
LogStreamName: aws.String("copilot/mockLogGroup/mockLogStream1"),
4848
},
49-
&cloudwatchlogs.LogStream{
50-
LogStreamName: aws.String("ecs/mockLogGroup/mockLogStream2"),
49+
{
50+
LogStreamName: aws.String("copilot/mockLogGroup/mockLogStream2"),
5151
},
5252
},
5353
}, nil)
@@ -57,10 +57,10 @@ func TestLogEvents(t *testing.T) {
5757
EndTime: aws.Int64(1234568),
5858
Limit: aws.Int64(10),
5959
LogGroupName: aws.String("mockLogGroup"),
60-
LogStreamName: aws.String("ecs/mockLogGroup/mockLogStream1"),
60+
LogStreamName: aws.String("copilot/mockLogGroup/mockLogStream1"),
6161
}).Return(&cloudwatchlogs.GetLogEventsOutput{
6262
Events: []*cloudwatchlogs.OutputLogEvent{
63-
&cloudwatchlogs.OutputLogEvent{
63+
{
6464
Message: aws.String("some log"),
6565
Timestamp: aws.Int64(1),
6666
},
@@ -72,10 +72,10 @@ func TestLogEvents(t *testing.T) {
7272
EndTime: aws.Int64(1234568),
7373
Limit: aws.Int64(10),
7474
LogGroupName: aws.String("mockLogGroup"),
75-
LogStreamName: aws.String("ecs/mockLogGroup/mockLogStream2"),
75+
LogStreamName: aws.String("copilot/mockLogGroup/mockLogStream2"),
7676
}).Return(&cloudwatchlogs.GetLogEventsOutput{
7777
Events: []*cloudwatchlogs.OutputLogEvent{
78-
&cloudwatchlogs.OutputLogEvent{
78+
{
7979
Message: aws.String("other log"),
8080
Timestamp: aws.Int64(0),
8181
},
@@ -84,20 +84,20 @@ func TestLogEvents(t *testing.T) {
8484
},
8585

8686
wantLogEvents: []*Event{
87-
&Event{
88-
TaskID: "mockLogStream2",
89-
Message: "other log",
90-
Timestamp: 0,
87+
{
88+
LogStreamName: "mockLogGroup/mockLogStream2",
89+
Message: "other log",
90+
Timestamp: 0,
9191
},
92-
&Event{
93-
TaskID: "mockLogStream1",
94-
Message: "some log",
95-
Timestamp: 1,
92+
{
93+
LogStreamName: "mockLogGroup/mockLogStream1",
94+
Message: "some log",
95+
Timestamp: 1,
9696
},
9797
},
9898
wantLastEventTime: map[string]int64{
99-
"ecs/mockLogGroup/mockLogStream1": 1,
100-
"ecs/mockLogGroup/mockLogStream2": 0,
99+
"copilot/mockLogGroup/mockLogStream1": 1,
100+
"copilot/mockLogGroup/mockLogStream2": 0,
101101
},
102102
wantErr: nil,
103103
},
@@ -107,7 +107,7 @@ func TestLogEvents(t *testing.T) {
107107
endTime: 1234999,
108108
limit: 10,
109109
lastEventTime: map[string]int64{
110-
"ecs/mockLogGroup/mockLogStream": 1234890,
110+
"copilot/mockLogGroup/mockLogStream": 1234890,
111111
},
112112
mockcloudwatchlogsClient: func(m *mocks.Mockapi) {
113113
m.EXPECT().DescribeLogStreams(&cloudwatchlogs.DescribeLogStreamsInput{
@@ -116,8 +116,8 @@ func TestLogEvents(t *testing.T) {
116116
OrderBy: aws.String("LastEventTime"),
117117
}).Return(&cloudwatchlogs.DescribeLogStreamsOutput{
118118
LogStreams: []*cloudwatchlogs.LogStream{
119-
&cloudwatchlogs.LogStream{
120-
LogStreamName: aws.String("ecs/mockLogGroup/mockLogStream"),
119+
{
120+
LogStreamName: aws.String("copilot/mockLogGroup/mockLogStream"),
121121
},
122122
},
123123
}, nil)
@@ -127,10 +127,10 @@ func TestLogEvents(t *testing.T) {
127127
Limit: aws.Int64(10),
128128
EndTime: aws.Int64(1234999),
129129
LogGroupName: aws.String("mockLogGroup"),
130-
LogStreamName: aws.String("ecs/mockLogGroup/mockLogStream"),
130+
LogStreamName: aws.String("copilot/mockLogGroup/mockLogStream"),
131131
}).Return(&cloudwatchlogs.GetLogEventsOutput{
132132
Events: []*cloudwatchlogs.OutputLogEvent{
133-
&cloudwatchlogs.OutputLogEvent{
133+
{
134134
Message: aws.String("some log"),
135135
Timestamp: aws.Int64(1234892),
136136
},
@@ -139,14 +139,14 @@ func TestLogEvents(t *testing.T) {
139139
},
140140

141141
wantLogEvents: []*Event{
142-
&Event{
143-
TaskID: "mockLogStream",
144-
Message: "some log",
145-
Timestamp: 1234892,
142+
{
143+
LogStreamName: "mockLogGroup/mockLogStream",
144+
Message: "some log",
145+
Timestamp: 1234892,
146146
},
147147
},
148148
wantLastEventTime: map[string]int64{
149-
"ecs/mockLogGroup/mockLogStream": 1234892,
149+
"copilot/mockLogGroup/mockLogStream": 1234892,
150150
},
151151
wantErr: nil,
152152
},
@@ -163,8 +163,8 @@ func TestLogEvents(t *testing.T) {
163163
OrderBy: aws.String("LastEventTime"),
164164
}).Return(&cloudwatchlogs.DescribeLogStreamsOutput{
165165
LogStreams: []*cloudwatchlogs.LogStream{
166-
&cloudwatchlogs.LogStream{
167-
LogStreamName: aws.String("ecs/mockLogGroup/mockLogStream"),
166+
{
167+
LogStreamName: aws.String("copilot/mockLogGroup/mockLogStream"),
168168
},
169169
},
170170
}, nil)
@@ -174,14 +174,14 @@ func TestLogEvents(t *testing.T) {
174174
EndTime: aws.Int64(1234568),
175175
Limit: aws.Int64(1),
176176
LogGroupName: aws.String("mockLogGroup"),
177-
LogStreamName: aws.String("ecs/mockLogGroup/mockLogStream"),
177+
LogStreamName: aws.String("copilot/mockLogGroup/mockLogStream"),
178178
}).Return(&cloudwatchlogs.GetLogEventsOutput{
179179
Events: []*cloudwatchlogs.OutputLogEvent{
180-
&cloudwatchlogs.OutputLogEvent{
180+
{
181181
Message: aws.String("some log"),
182182
Timestamp: aws.Int64(0),
183183
},
184-
&cloudwatchlogs.OutputLogEvent{
184+
{
185185
Message: aws.String("other log"),
186186
Timestamp: aws.Int64(1),
187187
},
@@ -190,14 +190,14 @@ func TestLogEvents(t *testing.T) {
190190
},
191191

192192
wantLogEvents: []*Event{
193-
&Event{
194-
TaskID: "mockLogStream",
195-
Message: "other log",
196-
Timestamp: 1,
193+
{
194+
LogStreamName: "mockLogGroup/mockLogStream",
195+
Message: "other log",
196+
Timestamp: 1,
197197
},
198198
},
199199
wantLastEventTime: map[string]int64{
200-
"ecs/mockLogGroup/mockLogStream": 1,
200+
"copilot/mockLogGroup/mockLogStream": 1,
201201
},
202202
wantErr: nil,
203203
},
@@ -250,7 +250,7 @@ func TestLogEvents(t *testing.T) {
250250
OrderBy: aws.String("LastEventTime"),
251251
}).Return(&cloudwatchlogs.DescribeLogStreamsOutput{
252252
LogStreams: []*cloudwatchlogs.LogStream{
253-
&cloudwatchlogs.LogStream{
253+
{
254254
LogStreamName: aws.String("mockLogStream"),
255255
},
256256
},
@@ -311,7 +311,7 @@ func TestLogGroupExists(t *testing.T) {
311311
LogGroupName: aws.String("mockLogGroup"),
312312
}).Return(&cloudwatchlogs.DescribeLogStreamsOutput{
313313
LogStreams: []*cloudwatchlogs.LogStream{
314-
&cloudwatchlogs.LogStream{
314+
{
315315
LogStreamName: aws.String("mockLogStream"),
316316
},
317317
},

‎internal/pkg/aws/cloudwatchlogs/event.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,12 @@ import (
1313
)
1414

1515
const (
16-
shortTaskIDLength = 7
16+
shortLogStreamNameLength = 25
1717
)
1818

1919
// Event represents a log event.
2020
type Event struct {
21-
TaskID string `json:"taskID"`
21+
LogStreamName string `json:"logStreamName"`
2222
IngestionTime int64 `json:"ingestionTime"`
2323
Message string `json:"message"`
2424
Timestamp int64 `json:"timestamp"`
@@ -41,12 +41,12 @@ func (l *Event) HumanString() string {
4141
for _, code := range warningCodes {
4242
l.Message = strings.ReplaceAll(l.Message, code, color.Yellow.Sprint(code))
4343
}
44-
return fmt.Sprintf("%s %s\n", color.Grey.Sprint(l.shortTaskID()), l.Message)
44+
return fmt.Sprintf("%s %s\n", color.Grey.Sprint(l.shortLogStreamName()), l.Message)
4545
}
4646

47-
func (l *Event) shortTaskID() string {
48-
if len(l.TaskID) < shortTaskIDLength {
49-
return l.TaskID
47+
func (l *Event) shortLogStreamName() string {
48+
if len(l.LogStreamName) < shortLogStreamNameLength {
49+
return l.LogStreamName
5050
}
51-
return l.TaskID[0:shortTaskIDLength]
51+
return l.LogStreamName[0:shortLogStreamNameLength]
5252
}

‎internal/pkg/cli/svc_logs_test.go‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -573,29 +573,29 @@ func TestSvcLogs_Execute(t *testing.T) {
573573
}
574574
logEvents := []*cloudwatchlogs.Event{
575575
{
576-
TaskID: "123456789",
577-
Message: `10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 200 -`,
576+
LogStreamName: "firelens_log_router/fcfe4ab8043841c08162318e5ad805f1",
577+
Message: `10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 200 -`,
578578
},
579579
{
580-
TaskID: "123456789",
581-
Message: `10.0.0.00 - - [01/Jan/1970 01:01:01] "FATA some error" - -`,
580+
LogStreamName: "firelens_log_router/fcfe4ab8043841c08162318e5ad805f1",
581+
Message: `10.0.0.00 - - [01/Jan/1970 01:01:01] "FATA some error" - -`,
582582
},
583583
{
584-
TaskID: "123456789",
585-
Message: `10.0.0.00 - - [01/Jan/1970 01:01:01] "WARN some warning" - -`,
584+
LogStreamName: "firelens_log_router/fcfe4ab8043841c08162318e5ad805f1",
585+
Message: `10.0.0.00 - - [01/Jan/1970 01:01:01] "WARN some warning" - -`,
586586
},
587587
}
588588
moreLogEvents := []*cloudwatchlogs.Event{
589589
{
590-
TaskID: "123456789",
591-
Message: `10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 404 -`,
590+
LogStreamName: "firelens_log_router/fcfe4ab8043841c08162318e5ad805f1",
591+
Message: `10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 404 -`,
592592
},
593593
}
594-
logEventsHumanString := `1234567 10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 200 -
595-
1234567 10.0.0.00 - - [01/Jan/1970 01:01:01] "FATA some error" - -
596-
1234567 10.0.0.00 - - [01/Jan/1970 01:01:01] "WARN some warning" - -
594+
logEventsHumanString := `firelens_log_router/fcfe4 10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 200 -
595+
firelens_log_router/fcfe4 10.0.0.00 - - [01/Jan/1970 01:01:01] "FATA some error" - -
596+
firelens_log_router/fcfe4 10.0.0.00 - - [01/Jan/1970 01:01:01] "WARN some warning" - -
597597
`
598-
logEventsJSONString := "{\"taskID\":\"123456789\",\"ingestionTime\":0,\"message\":\"10.0.0.00 - - [01/Jan/1970 01:01:01] \\\"GET / HTTP/1.1\\\" 200 -\",\"timestamp\":0}\n{\"taskID\":\"123456789\",\"ingestionTime\":0,\"message\":\"10.0.0.00 - - [01/Jan/1970 01:01:01] \\\"FATA some error\\\" - -\",\"timestamp\":0}\n{\"taskID\":\"123456789\",\"ingestionTime\":0,\"message\":\"10.0.0.00 - - [01/Jan/1970 01:01:01] \\\"WARN some warning\\\" - -\",\"timestamp\":0}\n"
598+
logEventsJSONString := "{\"logStreamName\":\"firelens_log_router/fcfe4ab8043841c08162318e5ad805f1\",\"ingestionTime\":0,\"message\":\"10.0.0.00 - - [01/Jan/1970 01:01:01] \\\"GET / HTTP/1.1\\\" 200 -\",\"timestamp\":0}\n{\"logStreamName\":\"firelens_log_router/fcfe4ab8043841c08162318e5ad805f1\",\"ingestionTime\":0,\"message\":\"10.0.0.00 - - [01/Jan/1970 01:01:01] \\\"FATA some error\\\" - -\",\"timestamp\":0}\n{\"logStreamName\":\"firelens_log_router/fcfe4ab8043841c08162318e5ad805f1\",\"ingestionTime\":0,\"message\":\"10.0.0.00 - - [01/Jan/1970 01:01:01] \\\"WARN some warning\\\" - -\",\"timestamp\":0}\n"
599599
testCases := map[string]struct {
600600
inputApp string
601601
inputSvc string
@@ -671,10 +671,10 @@ func TestSvcLogs_Execute(t *testing.T) {
671671
},
672672

673673
wantedError: nil,
674-
wantedContent: `1234567 10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 200 -
675-
1234567 10.0.0.00 - - [01/Jan/1970 01:01:01] "FATA some error" - -
676-
1234567 10.0.0.00 - - [01/Jan/1970 01:01:01] "WARN some warning" - -
677-
1234567 10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 404 -
674+
wantedContent: `firelens_log_router/fcfe4 10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 200 -
675+
firelens_log_router/fcfe4 10.0.0.00 - - [01/Jan/1970 01:01:01] "FATA some error" - -
676+
firelens_log_router/fcfe4 10.0.0.00 - - [01/Jan/1970 01:01:01] "WARN some warning" - -
677+
firelens_log_router/fcfe4 10.0.0.00 - - [01/Jan/1970 01:01:01] "GET / HTTP/1.1" 404 -
678678
`,
679679
},
680680
"returns error if fail to get event logs": {

0 commit comments

Comments
 (0)