Skip to content

Commit 703aa28

Browse files
andyburkeandyburke
committed
Merge pull request andyburke#44 from jjay/master
StreamedWWWForm for memory efficient files upload
2 parents d1a375c + e4c2975 commit 703aa28

3 files changed

Lines changed: 261 additions & 9 deletions

File tree

‎src/FormDataStream.cs‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
using System;
2+
using System.IO;
3+
using System.Text;
4+
using System.Collections.Generic;
5+
namespace HTTP {
6+
7+
public class FormPart {
8+
string fieldName;
9+
string mimeType;
10+
byte[] header;
11+
Stream contents;
12+
int position = 0;
13+
14+
public FormPart(string fieldName, string mimeType, string boundary, Stream contents, string fileName=null){
15+
string filenameheader = "";
16+
if (fileName != null){
17+
filenameheader = "; filename=\"" + fileName +"\"";
18+
}
19+
header = Encoding.ASCII.GetBytes(
20+
"\r\n--" + boundary + "\r\n" +
21+
"Content-Type: " + mimeType + "\r\n" +
22+
"Content-disposition: form-data; name=\"" + fieldName + "\"" + filenameheader + "\r\n\r\n"
23+
);
24+
this.contents = contents;
25+
}
26+
public long Length {
27+
get {
28+
return header.Length + contents.Length;
29+
}
30+
}
31+
public int Read(byte[] buffer, int offset, int size){
32+
int writed = 0;
33+
int bytesToWrite;
34+
if (position < header.Length){
35+
bytesToWrite = (int)(header.Length - position) > size ? size : (int)(header.Length - position);
36+
Array.Copy (
37+
header, // from header
38+
position, // started from position
39+
buffer, // to buffer
40+
offset, // started with offset
41+
bytesToWrite
42+
);
43+
writed += bytesToWrite;
44+
position += bytesToWrite;
45+
}
46+
if (writed >= size){
47+
return writed;
48+
}
49+
bytesToWrite = contents.Read(buffer, writed + offset, size - writed);
50+
writed += bytesToWrite;
51+
position += bytesToWrite;
52+
return writed;
53+
}
54+
}
55+
56+
public class FormDataStream: Stream {
57+
long position = 0;
58+
List<FormPart> parts = new List<FormPart>();
59+
bool dirty = false;
60+
byte[] footer;
61+
string boundary;
62+
63+
public FormDataStream(string boundary){
64+
this.boundary = boundary;
65+
footer = Encoding.ASCII.GetBytes("\r\n--" + boundary + "--\r\n");
66+
}
67+
68+
public override bool CanRead { get { return true; } }
69+
public override bool CanSeek { get { return false; } }
70+
public override bool CanTimeout { get { return false; } }
71+
public override bool CanWrite { get { return false; } }
72+
public override int ReadTimeout { get { return 0; } set { } }
73+
public override int WriteTimeout { get { return 0; } set { } }
74+
public override long Position {
75+
get {
76+
return position;
77+
}
78+
set {
79+
throw new NotImplementedException("FormDataStream is non-seekable stream");
80+
}
81+
}
82+
public override long Length {
83+
get {
84+
if (parts.Count == 0){
85+
return 0;
86+
}
87+
dirty = true;
88+
long len = 0;
89+
foreach (var part in parts){
90+
len += part.Length;
91+
}
92+
return len + footer.Length;
93+
}
94+
}
95+
96+
public override void Flush(){
97+
throw new NotImplementedException("FormDataStream is readonly stream");
98+
}
99+
100+
public override int Read(byte[] buffer, int offset, int count){
101+
dirty = true;
102+
int writed = 0;
103+
int bytesToWrite = 0;
104+
105+
// write parts
106+
long partsSize = 0;
107+
foreach (var part in parts){
108+
partsSize += part.Length;
109+
if (position > partsSize){
110+
continue;
111+
}
112+
bytesToWrite = part.Read(buffer, writed + offset, count - writed);
113+
writed += bytesToWrite;
114+
position += bytesToWrite;
115+
if (writed >= count){
116+
return count;
117+
}
118+
}
119+
120+
121+
// write footer
122+
bytesToWrite = count - writed > footer.Length? footer.Length : count - writed;
123+
Array.Copy (footer, 0, buffer, writed + offset, bytesToWrite);
124+
position += bytesToWrite;
125+
writed += bytesToWrite;
126+
return writed;
127+
}
128+
129+
public override long Seek(long amount, SeekOrigin origin){
130+
throw new NotImplementedException("FormDataStream is non-seekable stream");
131+
}
132+
133+
public override void SetLength (long len){
134+
throw new NotImplementedException("FormDataStream is readonly stream");
135+
}
136+
137+
public override void Write(byte[] source, int offset, int count){
138+
throw new NotImplementedException("FormDataStream is readonly stream");
139+
}
140+
141+
public void AddPart(string fieldName, string mimeType, Stream contents, string fileName=null){
142+
if (dirty){
143+
throw new InvalidOperationException("You can't change form data, form already readed");
144+
}
145+
parts.Add(new FormPart(fieldName, mimeType, boundary, contents, fileName));
146+
}
147+
148+
public void AddPart(FormPart part){
149+
if (dirty){
150+
throw new InvalidOperationException("You can't change form data, form already readed");
151+
}
152+
parts.Add(part);
153+
}
154+
}
155+
156+
}
157+
158+

‎src/Request.cs‎

Lines changed: 41 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ public enum RequestState {
2626

2727
public class Request
2828
{
29+
2930
public static bool LogAllRequests = false;
3031
public static bool VerboseLogging = false;
3132
public static string unityVersion = Application.unityVersion;
@@ -34,7 +35,7 @@ public class Request
3435
public CookieJar cookieJar = CookieJar.Instance;
3536
public string method = "GET";
3637
public string protocol = "HTTP/1.1";
37-
public byte[] bytes;
38+
public Stream byteStream;
3839
public Uri uri;
3940
public static byte[] EOL = { (byte)'\r', (byte)'\n' };
4041
public Response response = null;
@@ -46,6 +47,7 @@ public class Request
4647
public RequestState state = RequestState.Waiting;
4748
public long responseTime = 0; // in milliseconds
4849
public bool synchronous = false;
50+
public int bufferSize = 4 * 1024;
4951

5052
public Action< HTTP.Request > completedCallback = null;
5153

@@ -69,25 +71,36 @@ public Request (string method, string uri, byte[] bytes)
6971
{
7072
this.method = method;
7173
this.uri = new Uri (uri);
72-
this.bytes = bytes;
74+
this.byteStream = new MemoryStream(bytes);
75+
}
76+
77+
public Request(string method, string uri, StreamedWWWForm form){
78+
this.method = method;
79+
this.uri = new Uri (uri);
80+
this.byteStream = form.stream;
81+
foreach ( DictionaryEntry entry in form.headers )
82+
{
83+
this.AddHeader( (string)entry.Key, (string)entry.Value );
84+
}
7385
}
7486

7587
public Request( string method, string uri, WWWForm form )
7688
{
7789
this.method = method;
7890
this.uri = new Uri (uri);
79-
this.bytes = form.data;
91+
this.byteStream = new MemoryStream(form.data);
8092
foreach ( DictionaryEntry entry in form.headers )
8193
{
8294
this.AddHeader( (string)entry.Key, (string)entry.Value );
8395
}
96+
8497
}
8598

8699
public Request( string method, string uri, Hashtable data )
87100
{
88101
this.method = method;
89102
this.uri = new Uri( uri );
90-
this.bytes = Encoding.UTF8.GetBytes( JSON.JsonEncode( data ) );
103+
this.byteStream = new MemoryStream(Encoding.UTF8.GetBytes( JSON.JsonEncode( data ) ));
91104
this.AddHeader( "Content-Type", "application/json" );
92105
}
93106

@@ -195,15 +208,26 @@ private void GetResponse() {
195208
}
196209

197210
} catch (Exception e) {
211+
#if !UNITY_EDITOR
198212
Console.WriteLine ("Unhandled Exception, aborting request.");
199213
Console.WriteLine (e);
214+
#else
215+
Debug.LogError("Unhandled Exception, aborting request.");
216+
Debug.LogException(e);
217+
#endif
200218
exception = e;
201219
response = null;
202220
}
221+
203222
state = RequestState.Done;
204223
isDone = true;
205224
responseTime = curcall.ElapsedMilliseconds;
206225

226+
if ( byteStream != null )
227+
{
228+
byteStream.Close();
229+
}
230+
207231
if ( completedCallback != null )
208232
{
209233
if (synchronous) {
@@ -267,8 +291,8 @@ public virtual void Send( Action< HTTP.Request > callback = null)
267291
SetHeader( "cookie", cookieString );
268292
}
269293

270-
if ( bytes != null && bytes.Length > 0 && GetHeader ("Content-Length") == "" ) {
271-
SetHeader( "Content-Length", bytes.Length.ToString() );
294+
if ( byteStream != null && byteStream.Length > 0 && GetHeader ("Content-Length") == "" ) {
295+
SetHeader( "Content-Length", byteStream.Length.ToString() );
272296
}
273297

274298
if ( GetHeader( "User-Agent" ) == "" ) {
@@ -298,7 +322,7 @@ public virtual void Send( Action< HTTP.Request > callback = null)
298322
}
299323

300324
public string Text {
301-
set { bytes = System.Text.Encoding.UTF8.GetBytes (value); }
325+
set { byteStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes (value)); }
302326
}
303327

304328
public static bool ValidateServerCertificate (object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors)
@@ -328,8 +352,16 @@ void WriteToStream( Stream outputStream )
328352

329353
stream.Write( EOL );
330354

331-
if ( bytes != null && bytes.Length > 0 ) {
332-
stream.Write( bytes );
355+
if (byteStream == null){
356+
return;
357+
}
358+
359+
long numBytesToRead = byteStream.Length;
360+
byte[] buffer = new byte[bufferSize];
361+
while (numBytesToRead > 0){
362+
int readed = byteStream.Read(buffer, 0, bufferSize);
363+
stream.Write(buffer, 0, readed);
364+
numBytesToRead -= readed;
333365
}
334366
}
335367

‎src/StreamedWWWForm.cs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
using System;
2+
using System.IO;
3+
using System.Text;
4+
using System.Collections;
5+
6+
namespace HTTP
7+
{
8+
public class StreamedWWWForm {
9+
string boundary;
10+
public FormDataStream stream;
11+
12+
public Hashtable headers {
13+
get {
14+
return new Hashtable {
15+
{ "Content-Type", "multipart/form-data; boundary=\"" + boundary + "\""}
16+
};
17+
}
18+
}
19+
20+
public StreamedWWWForm(){
21+
byte[] bytes = new byte[40];
22+
var random = new Random();
23+
for (int i=0; i<40; i++){
24+
bytes[i] = (byte)(48 + random.Next(62));
25+
if (bytes[i] > 57){
26+
bytes[i] += 7;
27+
}
28+
if (bytes[i] > 90){
29+
bytes[i] += 6;
30+
}
31+
}
32+
boundary = Encoding.ASCII.GetString(bytes);
33+
stream = new FormDataStream(boundary);
34+
}
35+
36+
public void AddField(string fieldName, string fieldValue){
37+
var contentStream = new MemoryStream(Encoding.UTF8.GetBytes(fieldValue));
38+
stream.AddPart(fieldName, "text/plain; charset=\"utf-8\"", contentStream);
39+
40+
}
41+
public void AddBinaryData(string fieldName, byte[] contents=null, string mimeType = null){
42+
var contentStream = new MemoryStream(contents);
43+
if (mimeType == null){
44+
mimeType = "application/octet-stream";
45+
}
46+
stream.AddPart(fieldName, mimeType, contentStream, fieldName + ".dat");
47+
}
48+
public void AddBinaryData(string fieldName, Stream contents=null, string mimeType = null){
49+
if (mimeType == null){
50+
mimeType = "application/octet-stream";
51+
}
52+
stream.AddPart(fieldName, mimeType, contents, fieldName + ".dat");
53+
}
54+
public void AddFile(string fieldName, string path, string mimeType=null){
55+
if (mimeType == null){
56+
mimeType = "application/octet-stream";
57+
}
58+
var contents = new FileInfo(path).Open(FileMode.Open);
59+
stream.AddPart(fieldName, mimeType, contents, fieldName + ".dat");
60+
}
61+
}
62+
}

0 commit comments

Comments
 (0)