Version 1.19beta1

Changes to WiFi server and interface code for better network
reliability. FTP and MDNS now working on Duet WiFi.
Changed heater tuning to allow more time for the temperature to rise
when tuning a bed or chamber heater
Toned down the temperaure warning message

src/DuetNG/DuetEthernet/Network.cpp

@@ -135,6 +135,7 @@ void Network::ShutdownProtocol(Protocol protocol)
 	{
 		r->Terminate(protocol);
 	}
+
 	switch(protocol)
 	{
 	case HttpProtocol:
@@ -464,7 +465,7 @@ void Network::SetHostname(const char *name)
 		}
 	}
 
-	if (i)
+	if (i != 0)
 	{
 		hostname[i] = 0;
 	}

src/DuetNG/DuetEthernet/Network.h

@@ -2,7 +2,7 @@
 
 RepRapFirmware - Network: RepRapPro Ormerod with Duet controller
 
-Separated out from Platform.h by dc42 and extended by zpl
+Separated out from Platform.h by dc42 and extended by chrishamm
 
 ****************************************************************************************************/
 

src/DuetNG/DuetWiFi/Network.cpp

@@ -27,7 +27,7 @@
 
 #include "matrix.h"
 
-const uint32_t WifiResponseTimeoutMillis = 1000;
+const uint32_t WifiResponseTimeoutMillis = 200;
 const uint32_t WiFiStartupMillis = 300;
 
 const unsigned int MaxHttpConnections = 4;
@@ -89,9 +89,9 @@ static void EspTransferRequestIsr()
 /*-----------------------------------------------------------------------------------*/
 // WiFi interface class
 
-Network::Network(Platform& p) : platform(p), nextResponderToPoll(nullptr), uploader(nullptr), currentSocket(0),
-		state(NetworkState::disabled), requestedMode(WiFiState::disabled), currentMode(WiFiState::disabled), activated(false), espStatusChanged(false),
-		spiTxUnderruns(0), spiRxOverruns(0)
+Network::Network(Platform& p) : platform(p), nextResponderToPoll(nullptr), uploader(nullptr), currentSocket(0), ftpDataPort(0),
+		state(NetworkState::disabled), requestedMode(WiFiState::disabled), currentMode(WiFiState::disabled), activated(false),
+		espStatusChanged(false), spiTxUnderruns(0), spiRxOverruns(0)
 {
 	for (size_t i = 0; i < NumProtocols; ++i)
 	{
@@ -154,12 +154,7 @@ void Network::EnableProtocol(int protocol, int port, int secure, StringRef& repl
 			if (state == NetworkState::active)
 			{
 				StartProtocol(protocol);
-#if 0	// mdns not implemented yet, if we are going to implement it then we need to send the WiFi module a command to do it here and pass the protocol set
-				if (state == NetworkState::active)
-				{
-					DoMdnsAnnounce();
-				}
-#endif
+				// mDNS announcement is done by the WiFi Server firmware
 			}
 		}
 		ReportOneProtocol(protocol, reply);
@@ -192,15 +187,15 @@ void Network::StartProtocol(Protocol protocol)
 	switch(protocol)
 	{
 	case HttpProtocol:
-		SendListenCommand(portNumbers[protocol], MaxHttpConnections);
+		SendListenCommand(portNumbers[protocol], protocol, MaxHttpConnections);
 		break;
 
 	case FtpProtocol:
-		SendListenCommand(portNumbers[protocol], 2);
+		SendListenCommand(portNumbers[protocol], protocol, 1);
 		break;
 
 	case TelnetProtocol:
-		SendListenCommand(portNumbers[protocol], 1);
+		SendListenCommand(portNumbers[protocol], protocol, 1);
 		break;
 
 	default:
@@ -217,12 +212,12 @@ void Network::ShutdownProtocol(Protocol protocol)
 	switch(protocol)
 	{
 	case HttpProtocol:
-		SendListenCommand(portNumbers[protocol], 0);
+		SendListenCommand(portNumbers[protocol], protocol, 0);
 		TerminateSockets(portNumbers[protocol]);
 		break;
 
 	case FtpProtocol:
-		SendListenCommand(portNumbers[protocol], 0);
+		SendListenCommand(portNumbers[protocol], protocol, 0);
 		TerminateSockets(portNumbers[protocol]);
 		if (ftpDataPort != 0)
 		{
@@ -231,7 +226,7 @@ void Network::ShutdownProtocol(Protocol protocol)
 		break;
 
 	case TelnetProtocol:
-		SendListenCommand(portNumbers[protocol], 0);
+		SendListenCommand(portNumbers[protocol], protocol, 0);
 		TerminateSockets(portNumbers[protocol]);
 		break;
 
@@ -364,7 +359,8 @@ void Network::Start()
 
 	// Set the data request pin to be an input
 	pinMode(EspTransferRequestPin, INPUT_PULLUP);
-	// we don't use an interrupt for this yetr
+	// FIXME: Attaching the ISR here is a possible reason for watchdog resets (if the ESP firmware is broken),
+	// but without this call the WiFi chip doesn't boot up any more if the firmware image is valid
 	attachInterrupt(EspTransferRequestPin, EspTransferRequestIsr, RISING);
 
 	// The ESP takes about 300ms before it starts talking to us, so don't wait for it here, do that in Spin()
@@ -380,7 +376,9 @@ void Network::Stop()
 	if (state != NetworkState::disabled)
 	{
 		digitalWrite(SamTfrReadyPin, LOW);			// tell the ESP we can't receive
-		digitalWrite(EspResetPin, LOW);	// put the ESP back into reset
+		digitalWrite(EspResetPin, LOW);				// put the ESP back into reset
+		detachInterrupt(EspTransferRequestPin);		// ignore IRQs from the transfer request pin
+
 		NVIC_DisableIRQ(SPI_IRQn);
 		spi_disable(SPI);
 		spi_dma_check_rx_complete();
@@ -405,7 +403,8 @@ void Network::Spin(bool full)
 			{
 				// Setup the SPI controller in slave mode and assign the CS pin to it
 				platform.Message(HOST_MESSAGE, "WiFi module started\n");
-				SetupSpi();						// set up the SPI subsystem
+				SetupSpi();									// set up the SPI subsystem
+
 				state = NetworkState::active;
 				currentMode = WiFiState::idle;				// wifi module is running but inactive
 
@@ -414,6 +413,19 @@ void Network::Spin(bool full)
 				if (SendCommand(NetworkCommand::networkGetStatus, 0, 0, nullptr, 0, status) > 0)
 				{
 					SafeStrncpy(wiFiServerVersion, status.Value().versionText, ARRAY_SIZE(wiFiServerVersion));
+
+					// Set the hostname before anything else is done
+					if (SendCommand(NetworkCommand::networkSetHostName, 0, 0, hostname, HostNameLength, nullptr, 0) != ResponseEmpty)
+					{
+						reprap.GetPlatform().Message(GENERIC_MESSAGE, "Error: Could not set WiFi hostname\n");
+					}
+				}
+				else
+				{
+					// Something went wrong, maybe a bad firmware image was flashed
+					// Disable the WiFi chip again in this case
+					platform.Message(HOST_MESSAGE, "Error: Failed to initialise WiFi module!\n");
+					Stop();
 				}
 			}
 		}
@@ -624,6 +636,12 @@ void Network::Diagnostics(MessageType mtype)
 				platform.MessageF(mtype, "WiFi signal strength %ddb\n", r.rssi);
 			}
 			// status, ssid and hostName not displayed
+
+			// Print LwIP stats and other values over the ESP's UART line
+			if (SendCommand(NetworkCommand::diagnostics, 0, 0, nullptr, 0, nullptr, 0) != ResponseEmpty)
+			{
+				platform.Message(mtype, "Failed to request ESP stats\n");
+			}
 		}
 		else
 		{
@@ -631,7 +649,12 @@ void Network::Diagnostics(MessageType mtype)
 		}
 	}
 	HttpResponder::CommonDiagnostics(mtype);
-	platform.Message(mtype, "Responder states:");
+	platform.Message(mtype, "Socket states: ");
+	for (size_t i = 0; i < NumTcpSockets; i++)
+	{
+		platform.MessageF(mtype, " %d", sockets[i].State());
+	}
+	platform.Message(mtype, "\nResponder states:");
 	for (NetworkResponder *r = responders; r != nullptr; r = r->GetNext())
 	{
 		r->Diagnostics(mtype);
@@ -724,6 +747,7 @@ void Network::SetIPAddress(const uint8_t p_ipAddress[], const uint8_t p_netmask[
 // Set the DHCP hostname. Removes all whitespaces and converts the name to lower-case.
 void Network::SetHostname(const char *name)
 {
+	// Filter out illegal characters
 	size_t i = 0;
 	while (*name && i < ARRAY_UPB(hostname))
 	{
@@ -739,7 +763,7 @@ void Network::SetHostname(const char *name)
 		}
 	}
 
-	if (i)
+	if (i != 0)
 	{
 		hostname[i] = 0;
 	}
@@ -747,6 +771,15 @@ void Network::SetHostname(const char *name)
 	{
 		strcpy(hostname, HOSTNAME);
 	}
+
+	// Update the hostname if possible
+	if (state == NetworkState::active)
+	{
+		if (SendCommand(NetworkCommand::networkSetHostName, 0, 0, hostname, HostNameLength, nullptr, 0) != ResponseEmpty)
+		{
+			platform.Message(GENERIC_MESSAGE, "Error: Could not set WiFi hostname\n");
+		}
+	}
 }
 
 void Network::InitSockets()
@@ -795,28 +828,48 @@ void Network::UpdateSocketStatus(uint16_t connectedSockets, uint16_t otherEndClo
 // Find a responder to process a new connection
 bool Network::FindResponder(Socket *skt, Port localPort)
 {
-	for (size_t i = 0; i < NumProtocols; ++i)
+	// Get the right protocol
+	Protocol protocol;
+	if (localPort == ftpDataPort)
 	{
-		if (protocolEnabled[i] && portNumbers[i] == localPort)
+		protocol = FtpDataProtocol;
+	}
+	else
+	{
+		bool protocolFound = false;
+		for (size_t i = 0; i < NumProtocols; ++i)
 		{
-			for (NetworkResponder *r = responders; r != nullptr; r = r->GetNext())
+			if (protocolEnabled[i] && portNumbers[i] == localPort)
 			{
-				if (r->Accept(skt, i))
-				{
-					return true;
-				}
+				protocol = i;
+				protocolFound = true;
+				break;
 			}
-			return false;	// no free responder, or protocol disabled
+		}
+
+		if (!protocolFound)
+		{
+			// Protocol is disabled
+			return false;
 		}
 	}
-	return false;			// unknown port number
+
+	// Try to find a responder to deal with this connection
+	for (NetworkResponder *r = responders; r != nullptr; r = r->GetNext())
+	{
+		if (r->Accept(skt, protocol))
+		{
+			return true;
+		}
+	}
+	return false;
 }
 
 // Open the FTP data port
 void Network::OpenDataPort(Port port)
 {
 	ftpDataPort = port;
-	SendListenCommand(ftpDataPort, 1);
+	SendListenCommand(ftpDataPort, FtpDataProtocol, 1);
 }
 
 // Close FTP data port and purge associated resources
@@ -824,8 +877,14 @@ void Network::CloseDataPort()
 {
 	if (ftpDataPort != 0)
 	{
-		SendListenCommand(ftpDataPort, 0);
-		TerminateSockets(ftpDataPort);				// if the FTP responder wants to close the socket cleanly, it should have closed it itself
+		SendListenCommand(ftpDataPort, FtpDataProtocol, 0);
+		for (SocketNumber skt = 0; skt < NumTcpSockets; ++skt)
+		{
+			if (sockets[skt].GetLocalPort() == ftpDataPort)
+			{
+				sockets[skt].TerminatePolitely();
+			}
+		}
 		ftpDataPort = 0;
 	}
 }
@@ -1140,7 +1199,7 @@ int32_t Network::SendCommand(NetworkCommand cmd, SocketNumber socketNum, uint8_t
 
 	if (response < 0 && reprap.Debug(moduleNetwork))
 	{
-		debugPrintf("Network command %d socket %u returned error %d\n", (int)cmd, response);
+		debugPrintf("Network command %d socket %u returned error %d\n", (int)cmd, socketNum, response);
 	}
 
 #if 0
@@ -1154,10 +1213,11 @@ int32_t Network::SendCommand(NetworkCommand cmd, SocketNumber socketNum, uint8_t
 	return response;
 }
 
-void Network::SendListenCommand(Port port, unsigned int maxConnections)
+void Network::SendListenCommand(Port port, Protocol protocol, unsigned int maxConnections)
 {
 	ListenOrConnectData lcb;
 	lcb.port = port;
+	lcb.protocol = protocol;
 	lcb.remoteIp = AnyIp;
 	lcb.maxConnections = maxConnections;
 	SendCommand(NetworkCommand::networkListen, 0, 0, &lcb, sizeof(lcb), nullptr, 0);
@@ -1173,6 +1233,7 @@ void Network::GetNewStatus()
 	Receiver<MessageResponse> rcvr;
 
 	espStatusChanged = false;
+
 	const int32_t rslt = SendCommand(NetworkCommand::networkGetLastError, 0, 0, nullptr, 0, rcvr);
 	rcvr.Value().messageBuffer[ARRAY_UPB(rcvr.Value().messageBuffer)] = 0;
 	if (rslt < 0)

src/DuetNG/DuetWiFi/Network.h

@@ -2,7 +2,7 @@
 
 RepRapFirmware - Network: RepRapPro Ormerod with Duet controller
 
-Separated out from Platform.h by dc42 and extended by zpl
+Separated out from Platform.h by dc42 and extended by chrishamm
 
 ****************************************************************************************************/
 
@@ -119,7 +119,7 @@ private:
 
 	void SetupSpi();
 
-	void SendListenCommand(Port port, unsigned int maxConnections);
+	void SendListenCommand(Port port, Protocol protocol, unsigned int maxConnections);
 	void GetNewStatus();
 
 	static const char* TranslateEspResetReason(uint32_t reason);

src/DuetNG/DuetWiFi/Socket.cpp

@@ -10,10 +10,6 @@
 #include "RepRap.h"
 #include "Network.h"
 
-// TODO fif inefficiencies in this code and other parts of the WiFi interface at present:
-// 1. We keep polling all sockets instead of using the summary status to see which ones are active
-// 2. Send() in the NetworkResponder classes don't use the "Send and close" facility, they send a separate Close command
-
 const uint32_t FindResponderTimeout = 2000;			// how long we wait for a responder to become available
 const unsigned int MaxBuffersPerSocket = 4;
 
@@ -40,6 +36,7 @@ void Socket::Close()
 			return;
 		}
 	}
+
 	if (reprap.Debug(moduleNetwork))
 	{
 		debugPrintf("close failed, ir wrong state\n");
@@ -48,16 +45,27 @@ void Socket::Close()
 }
 
 // Terminate a connection immediately
-// The WiFi server code is written so that terminating a socket should always work, even if the socket isn't in use.
-// So we can call this after any sort of error on a socket.
+// We can call this after any sort of error on a socket as long as it is in use.
 void Socket::Terminate()
 {
-	const int32_t reply = reprap.GetNetwork().SendCommand(NetworkCommand::connAbort, socketNum, 0, nullptr, 0, nullptr, 0);
-	state = (reply != 0) ? SocketState::broken : SocketState::inactive;
+	if (state != SocketState::inactive)
+	{
+		const int32_t reply = reprap.GetNetwork().SendCommand(NetworkCommand::connAbort, socketNum, 0, nullptr, 0, nullptr, 0);
+		state = (reply != 0) ? SocketState::broken : SocketState::inactive;
+	}
 	DiscardReceivedData();
 	txBufferSpace = 0;
 }
 
+// Called to terminate the connection unless it is already being closed
+void Socket::TerminatePolitely()
+{
+	if (state != SocketState::clientDisconnecting && state != SocketState::closing)
+	{
+		Terminate();
+	}
+}
+
 // Return true if there is or may soon be more data to read
 bool Socket::CanRead() const
 {
@@ -137,13 +145,38 @@ void Socket::Poll(bool full)
 
 	switch (resp.Value().state)
 	{
+	case ConnState::otherEndClosed:
+		// Check for further incoming packets before this socket is finally closed.
+		// This must be done to ensure that FTP uploads are not cut off.
+		ReceiveData(resp.Value().bytesAvailable != 0);
+
+		if (state == SocketState::clientDisconnecting)
+		{
+			// We already got here before, so close the connection once and for all
+			Close();
+			break;
+		}
+		else if (state != SocketState::inactive)
+		{
+			state = SocketState::clientDisconnecting;
+			if (reprap.Debug(moduleNetwork))
+			{
+				debugPrintf("Client disconnected on socket %u\n", socketNum);
+			}
+			break;
+		}
+		// We can get here if a client has sent very little data and then instantly closed
+		// the connection, e.g. when an FTP client transferred very small files over the
+		// data port. In such cases we must notify the responder about this transmission!
+		// no break
+
 	case ConnState::connected:
 		if (full && state != SocketState::connected)
 		{
 			// It's a new connection
 			if (reprap.Debug(moduleNetwork))
 			{
-				debugPrintf("New conn on socket %u\n", socketNum);
+				debugPrintf("New conn on socket %u for local port %u\n", socketNum, localPort);
 			}
 			localPort = resp.Value().localPort;
 			remotePort = resp.Value().remotePort;
@@ -155,7 +188,7 @@ void Socket::Poll(bool full)
 			}
 			if (network.FindResponder(this, localPort))
 			{
-				state = SocketState::connected;
+				state = (resp.Value().state == ConnState::connected) ? SocketState::connected : SocketState::clientDisconnecting;
 				if (reprap.Debug(moduleNetwork))
 				{
 					debugPrintf("Found responder\n", socketNum);
@@ -174,29 +207,10 @@ void Socket::Poll(bool full)
 		if (state == SocketState::connected)
 		{
 			txBufferSpace = resp.Value().writeBufferSpace;
-			if (resp.Value().bytesAvailable != 0)
-			{
-				ReceiveData();
-			}
+			ReceiveData(resp.Value().bytesAvailable != 0);
 		}
 		break;
 
-	case ConnState::otherEndClosed:
-		// Check for further incoming packets before this socket is finally closed.
-		// This must be done to ensure that FTP uploads are not cut off.
-		if (resp.Value().bytesAvailable != 0)
-		{
-			ReceiveData();
-		}
-
-		if (state != SocketState::clientDisconnecting && reprap.Debug(moduleNetwork))
-		{
-			debugPrintf("Client disconnected on socket %u\n", socketNum);
-		}
-
-		state = SocketState::clientDisconnecting;
-		break;
-
 	default:
 		if (state == SocketState::connected || state == SocketState::waitingForResponder)
 		{
@@ -205,6 +219,11 @@ void Socket::Poll(bool full)
 			{
 				debugPrintf("Unexpected state change on socket %u\n", socketNum);
 			}
+			state = SocketState::broken;
+		}
+		else if (state == SocketState::closing)
+		{
+			// Socket closed
 			state = SocketState::inactive;
 		}
 		break;
@@ -213,35 +232,51 @@ void Socket::Poll(bool full)
 	needsPolling = false;
 }
 
-// Try to receive more incoming data from the socket
-void Socket::ReceiveData()
+// Try to receive more incoming data from the socket.
+void Socket::ReceiveData(uint16_t bytesAvailable)
 {
-	if (NetworkBuffer::Count(receivedData) < MaxBuffersPerSocket)
+	if (bytesAvailable != 0)
 	{
-//		debugPrintf("%u available\n", len);
-		// There is data available, so allocate a buffer
-		//TODO: if there is already a buffer and it is in an appropriate state (i.e. receiving) and it has enough room, we could just append the data
-		NetworkBuffer * const buf = NetworkBuffer::Allocate();
-		if (buf != nullptr)
+//		debugPrintf("%u available\n", bytesAvailable);
+		// First see if we already have a buffer with enough room
+		NetworkBuffer *const lastBuffer = NetworkBuffer::FindLast(receivedData);
+		if (lastBuffer != nullptr && (bytesAvailable <= lastBuffer->SpaceLeft() || (lastBuffer->SpaceLeft() != 0 && NetworkBuffer::Count(receivedData) >= MaxBuffersPerSocket)))
 		{
-			//TODO if an error occurs in the following transaction and 'full' is not set, it would be better not to call Platform::Message in SendCommand()
-			const int32_t ret = reprap.GetNetwork().SendCommand(NetworkCommand::connRead, socketNum, 0, nullptr, 0, buf->Data(), NetworkBuffer::bufferSize);
-			if (ret > 0)
+			// Read data into the existing buffer
+			const size_t maxToRead = min<size_t>(lastBuffer->SpaceLeft(), MaxDataLength);
+			const int32_t ret = reprap.GetNetwork().SendCommand(NetworkCommand::connRead, socketNum, 0, nullptr, 0, lastBuffer->UnwrittenData(), maxToRead);
+			if (ret > 0 && (size_t)ret <= maxToRead)
 			{
-				buf->dataLength = (size_t)ret;
-				buf->readPointer = 0;
-				NetworkBuffer::AppendToList(&receivedData, buf);
+				lastBuffer->dataLength += (size_t)ret;
 				if (reprap.Debug(moduleNetwork))
 				{
-					debugPrintf("Received %u bytes\n", buf->dataLength);
+					debugPrintf("Received %u bytes\n", (unsigned int)ret);
 				}
 			}
-			else
-			{
-				buf->Release();
-			}
 		}
-//		else debugPrintf("no buffer\n");
+		else if (NetworkBuffer::Count(receivedData) < MaxBuffersPerSocket)
+		{
+			NetworkBuffer * const buf = NetworkBuffer::Allocate();
+			if (buf != nullptr)
+			{
+				const size_t maxToRead = min<size_t>(NetworkBuffer::bufferSize, MaxDataLength);
+				const int32_t ret = reprap.GetNetwork().SendCommand(NetworkCommand::connRead, socketNum, 0, nullptr, 0, buf->Data(), maxToRead);
+				if (ret > 0 && (size_t)ret <= maxToRead)
+				{
+					buf->dataLength = (size_t)ret;
+					NetworkBuffer::AppendToList(&receivedData, buf);
+					if (reprap.Debug(moduleNetwork))
+					{
+						debugPrintf("Received %u bytes\n", (unsigned int)ret);
+					}
+				}
+				else
+				{
+					buf->Release();
+				}
+			}
+//			else debugPrintf("no buffer\n");
+		}
 	}
 }
 
@@ -255,13 +290,13 @@ void Socket::DiscardReceivedData()
 }
 
 // Send the data, returning the length buffered
-//TODO only try to send data if the status indicates rthat it is possible
 size_t Socket::Send(const uint8_t *data, size_t length)
 {
 	if (state == SocketState::connected && txBufferSpace != 0)
 	{
-		const int32_t reply = reprap.GetNetwork().SendCommand(NetworkCommand::connWrite, socketNum, 0, data, min<size_t>(length, txBufferSpace), nullptr, 0);
-		if (reply >= 0)
+		const size_t lengthToSend = min<size_t>(length, min<size_t>(txBufferSpace, MaxDataLength));
+		const int32_t reply = reprap.GetNetwork().SendCommand(NetworkCommand::connWrite, socketNum, 0, data, lengthToSend, nullptr, 0);
+		if (reply >= 0 && (size_t)reply <= lengthToSend)
 		{
 			txBufferSpace -= (size_t)reply;
 			return (size_t)reply;
@@ -270,7 +305,7 @@ size_t Socket::Send(const uint8_t *data, size_t length)
 		{
 			debugPrintf("Send failed, terminating\n");
 		}
-		Terminate();							// something is not right, so terminate the socket for safety
+		state = SocketState::broken;							// something is not right, terminate the socket soon
 	}
 	return 0;
 }
@@ -283,7 +318,11 @@ void Socket::Send()
 		const int32_t reply = reprap.GetNetwork().SendCommand(NetworkCommand::connWrite, socketNum, MessageHeaderSamToEsp::FlagPush, nullptr, 0, nullptr, 0);
 		if (reply < 0)
 		{
-			Terminate();						// something is not right, so terminate the socket for safety
+			if (reprap.Debug(moduleNetwork))
+			{
+				debugPrintf("Send failed, terminating\n");
+			}
+			state = SocketState::broken;						// something is not right, terminate the socket soon
 		}
 	}
 }

src/DuetNG/DuetWiFi/Socket.h

@@ -16,12 +16,14 @@ class Socket
 public:
 	Socket();
 	void Init(SocketNumber n);
+	int State() const { return (int)state; }				// used only for reporting debug info, hence the 'int' return
 	void Poll(bool full);
 	Port GetLocalPort() const { return localPort; }
 	uint32_t GetRemoteIP() const { return remoteIp; }
 	Port GetRemotePort() const { return remotePort; }
 	void Close();
 	void Terminate();
+	void TerminatePolitely();
 	bool ReadChar(char& c);
 	bool ReadBuffer(const uint8_t *&buffer, size_t &len);
 	void Taken(size_t len);
@@ -44,7 +46,7 @@ private:
 	};
 
 	void ReInit();
-	void ReceiveData();
+	void ReceiveData(uint16_t bytesAvailable);
 	void DiscardReceivedData();
 
 	Port localPort, remotePort;							// The local and remote ports

src/DuetNG/FtpResponder.cpp

@@ -23,6 +23,7 @@ bool FtpResponder::Accept(Socket *s, Protocol protocol)
 		// Make sure we can get an output buffer before we accept the connection, or we won't be able to reply
 		if (outBuf != nullptr || OutputBuffer::Allocate(outBuf))
 		{
+			clientPointer = 0;
 			skt = s;
 			if (reprap.Debug(moduleWebserver))
 			{
@@ -352,9 +353,8 @@ void FtpResponder::DoUpload()
 			GetPlatform().MessageF(HOST_MESSAGE, "Writing %u bytes of upload data\n", len);
 		}
 
-		const bool success = fileBeingUploaded.Write(reinterpret_cast<const char*>(buffer), len);
 		dataSocket->Taken(len);
-		if (!success)
+		if (!fileBeingUploaded.Write(buffer, len))
 		{
 			uploadError = true;
 			GetPlatform().Message(GENERIC_MESSAGE, "Error: Could not write upload data!\n");

src/DuetNG/FtpResponder.h

@@ -2,7 +2,7 @@
  * FtpResponder.h
  *
  *  Created on: 14 Apr 2017
- *      Author: David
+ *      Authors: David and Christian
  */
 
 #ifndef SRC_DUETNG_DUETETHERNET_FTPRESPONDER_H_

src/DuetNG/HttpResponder.cpp

@@ -1012,18 +1012,10 @@ void HttpResponder::ProcessMessage()
 					return;
 				}
 
-				// We cannot upload more than one file at once
-				if (!GetUploadLock())
-				{
-					RejectMessage("cannot upload more than one file at once");
-					return;
-				}
-
 				// Start a new file upload
 				FileStore *file = GetPlatform().GetFileStore(FS_PREFIX, qualifiers[0].value, true);
 				if (file == nullptr)
 				{
-					ReleaseUploadLock();
 					RejectMessage("could not create file");
 					return;
 
@@ -1090,36 +1082,20 @@ void HttpResponder::RejectMessage(const char* response, unsigned int code)
 // It tries to process a chunk of uploaded data and changes the state if finished.
 void HttpResponder::DoUpload()
 {
-	if (uploadedBytes == 0)
-	{
-		writeBufIndex = 0;
-	}
-
 	const uint8_t *buffer;
 	size_t len;
 	if (skt->ReadBuffer(buffer, len))
 	{
-		// Write data in sector-aligned chunks. This also means that the buffer in fatfs is only used to hold the FAT.
-		// Buffer size must be a multiple of the 512b sector size.
-		char* const writeBuf = reinterpret_cast<char *>(writeBufStorage);
-		const size_t lengthToCopy = min<size_t>(writeBufLength - writeBufIndex, len);
-		memcpy(writeBuf + writeBufIndex, buffer, lengthToCopy);
-		writeBufIndex += lengthToCopy;
-		uploadedBytes += lengthToCopy;
-		buffer += lengthToCopy;
-		skt->Taken(lengthToCopy);
-		if (writeBufIndex == writeBufLength || uploadedBytes >= postFileLength)
+		skt->Taken(len);
+		uploadedBytes += len;
+
+		if (!fileBeingUploaded.Write(buffer, len))
 		{
-			const bool success = fileBeingUploaded.Write(writeBuf, writeBufIndex);
-			writeBufIndex = 0;
-			if (!success)
-			{
-				uploadError = true;
-				GetPlatform().Message(GENERIC_MESSAGE, "Error: Could not write upload data!\n");
-				CancelUpload();
-				SendJsonResponse("upload");
-				return;
-			}
+			uploadError = true;
+			GetPlatform().Message(GENERIC_MESSAGE, "Error: Could not write upload data!\n");
+			CancelUpload();
+			SendJsonResponse("upload");
+			return;
 		}
 	}
 

src/DuetNG/NetworkBuffer.cpp

@@ -91,6 +91,19 @@ void NetworkBuffer::Empty()
 	*list = b;
 }
 
+// Find the last buffer in a list
+/*static*/ NetworkBuffer *NetworkBuffer::FindLast(NetworkBuffer *list)
+{
+	if (list != nullptr)
+	{
+		while (list->next != nullptr)
+		{
+			list = list->next;
+		}
+	}
+	return list;
+}
+
 /*static*/ NetworkBuffer *NetworkBuffer::Allocate()
 {
 	NetworkBuffer *ret = freelist;

src/DuetNG/NetworkBuffer.h

@@ -40,6 +40,9 @@ public:
 	// Return the length available for writing
 	size_t SpaceLeft() const { return bufferSize - dataLength; }
 
+	// Return a pointer to the space available for writing
+	uint8_t* UnwrittenData() { return Data() + dataLength; }
+
 	// Append some data, returning the amount appended
 	size_t AppendData(const uint8_t *source, size_t length);
 
@@ -52,6 +55,9 @@ public:
 	// Append a buffer to a list
 	static void AppendToList(NetworkBuffer **list, NetworkBuffer *b);
 
+	// Find the last buffer in a list
+	static NetworkBuffer *FindLast(NetworkBuffer *list);
+
 	// Allocate a buffer
 	static NetworkBuffer *Allocate();
 

src/DuetNG/NetworkResponder.cpp

@@ -178,16 +178,25 @@ void NetworkResponder::ConnectionLost()
 	CancelUpload();
 	OutputBuffer::ReleaseAll(outBuf);
 	outStack->ReleaseAll();
+
 	if (fileBeingSent != nullptr)
 	{
 		fileBeingSent->Close();
 		fileBeingSent = nullptr;
 	}
+
+	if (fileBuffer != nullptr)
+	{
+		fileBuffer->Release();
+		fileBuffer = nullptr;
+	}
+
 	if (skt != nullptr)
 	{
 		skt->Terminate();
 		skt = nullptr;
 	}
+
 	responderState = ResponderState::free;
 }
 
@@ -203,17 +212,13 @@ void NetworkResponder::StartUpload(FileStore *file, const char *fileName)
 // If this responder has an upload in progress, cancel it
 void NetworkResponder::CancelUpload()
 {
-	if (uploadLock.IsOwnedBy(this))
+	if (fileBeingUploaded.IsLive())
 	{
-		if (fileBeingUploaded.IsLive())
+		fileBeingUploaded.Close();
+		if (filenameBeingUploaded[0] != 0)
 		{
-			fileBeingUploaded.Close();
-			if (filenameBeingUploaded[0] != 0)
-			{
-				GetPlatform().GetMassStorage()->Delete(FS_PREFIX, filenameBeingUploaded);
-			}
+			GetPlatform().GetMassStorage()->Delete(FS_PREFIX, filenameBeingUploaded);
 		}
-		uploadLock.Release(this);
 	}
 }
 
@@ -256,12 +261,6 @@ void NetworkResponder::FinishUpload(uint32_t fileLength, time_t fileLastModified
 
 	// Clean up again
 	filenameBeingUploaded[0] = 0;
-	uploadLock.Release(this);
 }
 
-// NetworkResponder static data
-NetworkResponderLock NetworkResponder::uploadLock;
-uint32_t NetworkResponder::writeBufStorage[writeBufLength/4];
-size_t NetworkResponder::writeBufIndex;
-
 // End

src/DuetNG/NetworkResponder.h

@@ -72,9 +72,6 @@ protected:
 	virtual void SendData();
 	virtual void ConnectionLost();
 
-	bool GetUploadLock() const { return uploadLock.Acquire(this); }
-	void ReleaseUploadLock() const { return uploadLock.Release(this); }
-
 	void StartUpload(FileStore *file, const char *fileName);
 	void FinishUpload(uint32_t fileLength, time_t fileLastModified);
 	virtual void CancelUpload();
@@ -101,11 +98,6 @@ protected:
 	uint32_t postFileLength, uploadedBytes;				// How many POST bytes do we expect and how many have already been written?
 	time_t fileLastModified;
 	bool uploadError;
-
-	static NetworkResponderLock uploadLock;
-	static const size_t writeBufLength = 8192;
-	static uint32_t writeBufStorage[writeBufLength/4];	// aligned buffer for file writes
-	static size_t writeBufIndex;
 };
 
 #endif /* SRC_DUETNG_DUETETHERNET_NETWORKRESPONDER_H_ */

src/DuetNG/TelnetResponder.cpp

@@ -61,6 +61,7 @@ bool TelnetResponder::Spin()
 			// Discard the setup message
 			char c;
 			while (skt->ReadChar(c)) { }
+			connectTime = 0;
 			return true;
 		}
 

src/Heating/Pid.cpp

@@ -20,21 +20,16 @@ const uint32_t TempSettleTimeout = 20000;	// how long we allow the initial tempe
 float *PID::tuningTempReadings = nullptr;	// the readings from the heater being tuned
 float PID::tuningStartTemp;					// the temperature when we turned on the heater
 float PID::tuningPwm;						// the PWM to use
-float PID::tuningTargetTemp;					// the maximum temperature we are allowed to reach
+float PID::tuningTargetTemp;				// the maximum temperature we are allowed to reach
 uint32_t PID::tuningBeginTime;				// when we started the tuning process
 uint32_t PID::tuningPhaseStartTime;			// when we started the current tuning phase
 uint32_t PID::tuningReadingInterval;		// how often we are sampling
 size_t PID::tuningReadingsTaken;			// how many samples we have taken
 
-#ifdef NEW_TUNING
 float PID::tuningHeaterOffTemp;				// the temperature when we turned the heater off
 float PID::tuningPeakTemperature;			// the peak temperature reached, averaged over 3 readings (so slightly less than the true peak)
 uint32_t PID::tuningHeatingTime;			// how long we had the heating on for
 uint32_t PID::tuningPeakDelay;				// how many milliseconds the temperature continues to rise after turning the heater off
-#else
-float PID::tuningTimeOfFastestRate;			// how long after turn-on the fastest temperature rise occurred
-float PID::tuningFastestRate;				// the fastest temperature rise
-#endif
 
 // Member functions and constructors
 
@@ -101,7 +96,7 @@ bool PID::SetModel(float gain, float tc, float td, float maxPwm, bool usePid)
 			if (gain > safeGain)
 			{
 				platform.MessageF(GENERIC_MESSAGE,
-						"Warning: Heater %u appears to be over-powered and a fire risk! If left on at full power, its temperature is predicted to reach %uC.\n",
+						"Warning: Heater %u appears to be over-powered. If left on at full power, its temperature is predicted to reach %uC.\n",
 						heater, (unsigned int)gain + 20);
 			}
 		}
@@ -599,7 +594,6 @@ void PID::DoTuningStep()
 	tuningTempReadings[tuningReadingsTaken] = temperature;
 	++tuningReadingsTaken;
 
-#ifdef NEW_TUNING
 	switch(mode)
 	{
 	case HeaterMode::tuning0:
@@ -627,15 +621,17 @@ void PID::DoTuningStep()
 	case HeaterMode::tuning1:
 		// Heating up
 		{
+			const bool isBedOrChamberHeater = (heater == reprap.GetHeat().GetBedHeater() || heater == reprap.GetHeat().GetChamberHeater());
 			const uint32_t heatingTime = millis() - tuningPhaseStartTime;
-			if (heatingTime > (uint32_t)(model.GetDeadTime() * SecondsToMillis) + (30 * 1000) && (temperature - tuningStartTemp) < 3.0)
+			const float extraTimeAllowed = (isBedOrChamberHeater) ? 60.0 : 30.0;
+			if (heatingTime > (uint32_t)((model.GetDeadTime() + extraTimeAllowed) * SecondsToMillis) && (temperature - tuningStartTemp) < 3.0)
 			{
 				platform.Message(GENERIC_MESSAGE, "Auto tune cancelled because temperature is not increasing\n");
 				break;
 			}
 
-			const uint32_t timeoutMinutes = (heater == reprap.GetHeat().GetBedHeater() || heater == reprap.GetHeat().GetChamberHeater()) ? 20 : 5;
-			if (heatingTime >= timeoutMinutes * 60 * 1000)
+			const uint32_t timeoutMinutes = (isBedOrChamberHeater) ? 20 : 5;
+			if (heatingTime >= timeoutMinutes * 60 * (uint32_t)SecondsToMillis)
 			{
 				platform.Message(GENERIC_MESSAGE, "Auto tune cancelled because target temperature was not reached\n");
 				break;
@@ -716,98 +712,6 @@ void PID::DoTuningStep()
 
 	// If we get here, we have finished
 	SwitchOff();								// sets mode and lastPWM, also deletes tuningTempReadings
-#else
-	if (temperature > tuningMaxTemp)
-	{
-		platform.MessageF(GENERIC_MESSAGE,
-				"Auto tune of heater %u with P=%.2f S=%.1f cancelled because temperature limit exceeded. Use lower P or higher S in m303 command.\n",
-				heater, tuningPwm, tuningTargetTemp);
-	}
-	else
-	{
-		switch(mode)
-		{
-		case HeaterMode::tuning0:
-			// Waiting for initial temperature to settle after any thermostatic fans have turned on
-			if (ReadingsStable(6000/platform.HeatSampleInterval(), 0.5))	// expect temperature to be stable within a 0.5C band for 6 seconds
-			{
-				// Starting temperature is stable, so move on
-				tuningReadingsTaken = 1;
-				tuningTempReadings[0] = tuningStartTemp = temperature;
-				timeSetHeating = tuningPhaseStartTime = millis();
-				lastPwm = tuningPwm;										// turn on heater at specified power
-				tuningReadingInterval = platform.HeatSampleInterval();		// reset sampling interval
-				mode = HeaterMode::tuning1;
-				return;
-			}
-			if (millis() - tuningPhaseStartTime < 20000)
-			{
-				// Allow up to 20 seconds for starting temperature to settle
-				return;
-			}
-			platform.Message(GENERIC_MESSAGE, "Auto tune cancelled because starting temperature is not stable\n");
-			break;
-
-		case HeaterMode::tuning1:
-			if (millis() - tuningPhaseStartTime > (uint32_t)(model.GetDeadTime() * SecondsToMillis) + 30000 && (temperature - tuningStartTemp) < 3.0)
-			{
-				platform.Message(GENERIC_MESSAGE, "Auto tune cancelled because temperature is not increasing\n");
-				break;
-			}
-
-			// If we have a reasonable number of readings, try to identify the point of maximum rate of temperature increase
-			if (tuningReadingsTaken >= 50)
-			{
-				const size_t index = GetMaxRateIndex();
-				if (index <= tuningReadingsTaken/2)
-				{
-					// We have found the point of inflection, so start the next phase
-					if (reprap.Debug(moduleHeat))
-					{
-						DisplayBuffer("At phase 1 end");
-					}
-					tuningTimeOfFastestRate = index * tuningReadingInterval * MillisToSeconds;
-					tuningFastestRate = (tuningTempReadings[index + 2] - tuningTempReadings[index - 2]) / (tuningReadingInterval * 4 * MillisToSeconds);
-
-					// Move the readings down so as to start at the max rate index
-					tuningPhaseStartTime += index * tuningReadingInterval;
-					tuningReadingsTaken -= index;
-					for (size_t i = 0; i < tuningReadingsTaken; ++i)
-					{
-						tuningTempReadings[i] = tuningTempReadings[i + index];
-					}
-					mode = HeaterMode::tuning2;
-				}
-			}
-			return;
-
-		case HeaterMode::tuning2:
-			// Note: there is no check for temperature increasing here, because it may increase very slowly towards the end of the tuning process.
-			{
-				// In the following, the figure of 2.75 was chosen because a value of 2 is too low to handle the bed heater
-				// with embedded thermistor on my Kossel (reservoir effect)
-				if (ReadingsStable(tuningReadingsTaken/2, (temperature - tuningTempReadings[0]) * 0.2))		// if we have been going for ~2.75 time constants
-				{
-					// We have been heating for long enough, so we can do a fit
-					FitCurve();
-					break;
-				}
-				else
-				{
-					return;
-				}
-			}
-			break;
-
-		default:
-			// Should not happen, but if it does then quit
-			break;
-		}
-	}
-
-	// If we get here, we have finished
-	SwitchOff();								// sets mode and lastPWM, also deletes tuningTempReadings
-#endif
 }
 
 // Return true if the last 'numReadings' readings are stable
@@ -830,8 +734,6 @@ void PID::DoTuningStep()
 	return maxReading - minReading <= maxDiff;
 }
 
-#ifdef NEW_TUNING
-
 // Calculate which reading gave us the peak temperature.
 // Return -1 if peak not identified yet, 0 if we failed to find a peak, else the index of the peak (which can't be zero because we always average 3 readings)
 /*static*/ int PID::GetPeakTempIndex()
@@ -916,78 +818,6 @@ void PID::CalculateModel()
 	}
 }
 
-#else
-
-// Return the index in the temperature readings of the maximum rate of increase
-// In view of the poor resolution of most thermistors at low temperatures, we measure over 4 time intervals instead of 2.
-/*static*/ size_t PID::GetMaxRateIndex()
-{
-	size_t index = 2;
-	float maxIncrease = 0.0;
-	for (size_t i = 2; i + 2 < tuningReadingsTaken; ++i)
-	{
-		const float increase = tuningTempReadings[i + 2] - tuningTempReadings[i - 2];
-		if (increase > maxIncrease)
-		{
-			maxIncrease = increase;
-			index = i;
-		}
-	}
-	return index;
-}
-
-// Calculate G, td and tc from the accumulated readings and print the auto tune success/fail message
-// Before calling this we must have already identified the point of inflection and re-based the readings to start at it.
-void PID::FitCurve()
-{
-	if (reprap.Debug(moduleHeat))
-	{
-		DisplayBuffer("At completion");
-	}
-
-	// We need 3 points equally-spaced in time to do the calculation
-	const size_t lowIndex = 0;
-	size_t highIndex = tuningReadingsTaken - 1;
-	if ((highIndex - lowIndex) % 2 != 0)
-	{
-		--highIndex;
-	}
-
-	// Calculate tc, td, G
-	const float T1 = tuningTempReadings[lowIndex] - tuningStartTemp;
-	const size_t midIndex = (lowIndex + highIndex)/2;
-	const float T2 = tuningTempReadings[midIndex] - tuningStartTemp;
-	const float T3 = tuningTempReadings[highIndex] - tuningStartTemp;
-	//const float t1 = lowIndex * tuningReadingInterval * MillisToSeconds;
-	const float dt = (midIndex - lowIndex) * tuningReadingInterval * MillisToSeconds;
-	const float t3 = highIndex * tuningReadingInterval * MillisToSeconds;
-	const float tc = dt/log((T2 - T1)/(T3 - T2));
-
-	// In theory we should calculate the delay time like this:
-	// td = tc * log((T3 - T1)/(T3 * exp(-t1/tc) - T1 * exp(-t3/tc)));
-	// However, the calculated td is highly inaccurate in some situations e.g. bed heater with reservoir effect.
-	// Better to estimate it from the point of inflection.
-	const float td = tuningTimeOfFastestRate - (tuningTempReadings[0] - tuningStartTemp)/tuningFastestRate;
-
-	// I'm not sure which td value we should use to calculate the gain, but it probably doesn't make much difference because usually, td << t3.
-	const float gain = T3 / (lastPwm * (1.0 - exp((td - t3)/tc)));
-
-	tuned = SetModel(gain, tc, td, model.GetMaxPwm(), true);
-	if (tuned)
-	{
-		platform.MessageF(GENERIC_MESSAGE,
-				"Auto tune heater %d with PWM=%.2f completed in %u sec, maximum temperature reached %.1fC\n"
-				"Use M307 H%d to see the result\n",
-				heater, tuningPwm, (millis() - tuningBeginTime)/(uint32_t)SecondsToMillis, tuningTempReadings[tuningReadingsTaken - 1], heater);
-	}
-	else
-	{
-		platform.MessageF(GENERIC_MESSAGE, "Auto tune of heater %u failed due to bad curve fit (G=%.1f, tc=%.1f, td=%.1f)\n", heater, gain, tc, td);
-	}
-}
-
-#endif
-
 void PID::DisplayBuffer(const char *intro)
 {
 	OutputBuffer *buf;

src/Roland.h

@@ -26,6 +26,7 @@ Licence: GPL
 // See: http://www.rolanddg.com/product/3d/3d/mdx-20_15/mdx-20_15.html
 //      http://altlab.org/d/content/m/pangelo/ideas/rml_command_guide_en_v100.pdf
 
+#include "RepRapFirmware.h"
 #include "Core.h"
 #include "Platform.h"
 

src/Scanner.cpp

@@ -168,23 +168,13 @@ void Scanner::Spin()
 			break;
 
 		case ScannerState::Uploading:
-		{
-			// Copy incoming scan data from USB to the buffer
-			size_t bytesToCopy = min<size_t>(SERIAL_MAIN_DEVICE.available(), ScanBufferSize - bufferPointer);
-			bytesToCopy = min<size_t>(bytesToCopy, uploadBytesLeft);
-
-			for(size_t i = 0; i < bytesToCopy; i++)
+			// Write incoming scan data from USB to the file
+			while (SERIAL_MAIN_DEVICE.available() > 0)
 			{
 				char b = static_cast<char>(SERIAL_MAIN_DEVICE.read());
-				buffer[bufferPointer++] = b;
-			}
-			uploadBytesLeft -= bytesToCopy;
-
-			// When this buffer is full or the upload is complete, write the next chunk
-			if (uploadBytesLeft == 0 || bufferPointer == ScanBufferSize)
-			{
-				if (fileBeingUploaded->Write(buffer, bufferPointer))
+				if (fileBeingUploaded->Write(&b, sizeof(char)))
 				{
+					uploadBytesLeft--;
 					if (uploadBytesLeft == 0)
 					{
 						if (reprap.Debug(moduleScanner))
@@ -196,9 +186,8 @@ void Scanner::Spin()
 						fileBeingUploaded = nullptr;
 
 						SetState(ScannerState::Idle);
+						break;
 					}
-
-					bufferPointer = 0;
 				}
 				else
 				{
@@ -208,10 +197,10 @@ void Scanner::Spin()
 
 					platform.Message(GENERIC_MESSAGE, "Error: Could not write scan file\n");
 					SetState(ScannerState::Idle);
+					break;
 				}
 			}
 			break;
-		}
 
 		default:
 			// Pick up incoming commands only if the GCodeBuffer is idle.

src/Scanner.h

@@ -13,7 +13,7 @@
 
 #if SUPPORT_SCANNER
 
-const size_t ScanBufferSize = 512;						// Buffer for incoming codes and upload chunks
+const size_t ScanBufferSize = 128;						// Buffer for incoming commands
 
 enum class ScannerState
 {

src/Storage/FileData.h

@@ -46,7 +46,7 @@ public:
 		return f->Read(b);
 	}
 
-	int Read(char* buf, size_t nBytes)
+	int Read(char *buf, size_t nBytes)
 	{
 		return f->Read(buf, nBytes);
 	}
@@ -56,7 +56,12 @@ public:
 		return f->Write(b);
 	}
 
-	bool Write(const char *s, unsigned int len)
+	bool Write(const char *s, size_t len)
+	{
+		return f->Write(s, len);
+	}
+
+	bool Write(const uint8_t *s, size_t len)
 	{
 		return f->Write(s, len);
 	}

src/Storage/FileStore.cpp

@@ -8,7 +8,7 @@
 
 uint32_t FileStore::longestWriteTime = 0;
 
-FileStore::FileStore(Platform* p) : platform(p)
+FileStore::FileStore(Platform* p) : platform(p), writeBuffer(nullptr)
 {
 }
 
@@ -45,9 +45,9 @@ bool FileStore::Open(const char* directory, const char* fileName, bool write)
 								: fileName;
 	writing = write;
 
-	// Try to create the path of this file if we want to write to it
 	if (writing)
 	{
+		// Try to create the path of this file if we want to write to it
 		char filePathBuffer[FILENAME_LENGTH];
 		StringRef filePath(filePathBuffer, FILENAME_LENGTH);
 		filePath.copy(location);
@@ -73,6 +73,9 @@ bool FileStore::Open(const char* directory, const char* fileName, bool write)
 				filePath[i] = '/';
 			}
 		}
+
+		// Also try to allocate a write buffer so we can perform faster writes
+		writeBuffer = platform->GetMassStorage()->AllocateWriteBuffer();
 	}
 
 	FRESULT openReturn = f_open(&file, location, (writing) ?  FA_CREATE_ALWAYS | FA_WRITE : FA_OPEN_EXISTING | FA_READ);
@@ -149,6 +152,12 @@ bool FileStore::Close()
 		ok = Flush();
 	}
 
+	if (writeBuffer != nullptr)
+	{
+		platform->GetMassStorage()->ReleaseWriteBuffer(writeBuffer);
+		writeBuffer = nullptr;
+	}
+
 	FRESULT fr = f_close(&file);
 	inUse = false;
 	writing = false;
@@ -277,16 +286,47 @@ bool FileStore::Write(const char *s, size_t len)
 		return false;
 	}
 
-	size_t bytesWritten;
-	uint32_t time = micros();
-
-	FRESULT writeStatus = f_write(&file, s, len, &bytesWritten);
-	time = micros() - time;
-	if (time > longestWriteTime)
+	size_t totalBytesWritten = 0;
+	FRESULT writeStatus = FR_OK;
+	if (writeBuffer == nullptr)
 	{
-		longestWriteTime = time;
+		uint32_t time = micros();
+		writeStatus = f_write(&file, s, len, &totalBytesWritten);
+		time = micros() - time;
+		if (time > longestWriteTime)
+		{
+			longestWriteTime = time;
+		}
 	}
-	if ((writeStatus != FR_OK) || (bytesWritten != len))
+	else
+	{
+		do
+		{
+			size_t bytesStored = writeBuffer->Store(s + totalBytesWritten, len - totalBytesWritten);
+			if (writeBuffer->BytesLeft() == 0)
+			{
+				size_t bytesToWrite = writeBuffer->BytesStored(), bytesWritten;
+				uint32_t time = micros();
+				writeStatus = f_write(&file, writeBuffer->Data(), bytesToWrite, &bytesWritten);
+				time = micros() - time;
+				if (time > longestWriteTime)
+				{
+					longestWriteTime = time;
+				}
+				writeBuffer->DataTaken();
+
+				if (bytesToWrite != bytesWritten)
+				{
+					// Something went wrong
+					break;
+				}
+			}
+			totalBytesWritten += bytesStored;
+		}
+		while (writeStatus == FR_OK && totalBytesWritten != len);
+	}
+
+	if ((writeStatus != FR_OK) || (totalBytesWritten != len))
 	{
 		platform->Message(GENERIC_MESSAGE, "Error: Cannot write to file. Drive may be full.\n");
 		return false;
@@ -301,6 +341,26 @@ bool FileStore::Flush()
 		platform->Message(GENERIC_MESSAGE, "Error: Attempt to flush a non-open file.\n");
 		return false;
 	}
+
+	if (writeBuffer != nullptr)
+	{
+		size_t bytesToWrite = writeBuffer->BytesStored(), bytesWritten;
+		uint32_t time = micros();
+		FRESULT writeStatus = f_write(&file, writeBuffer->Data(), bytesToWrite, &bytesWritten);
+		time = micros() - time;
+		if (time > longestWriteTime)
+		{
+			longestWriteTime = time;
+		}
+		writeBuffer->DataTaken();
+
+		if ((writeStatus != FR_OK) || (bytesToWrite != bytesWritten))
+		{
+			platform->Message(GENERIC_MESSAGE, "Error: Cannot write to file. Drive may be full.\n");
+			return false;
+		}
+	}
+
 	return f_sync(&file) == FR_OK;
 }
 

src/Storage/FileStore.h

@@ -6,9 +6,8 @@
 #include "Core.h"
 #include "Libraries/Fatfs/ff.h"
 
-const size_t FileBufLen = 256;						// 512 would be more efficient, but need to free up some RAM first
-
 class Platform;
+class FileWriteBuffer;
 
 class FileStore
 {
@@ -18,6 +17,7 @@ public:
 	int ReadLine(char* buf, size_t nBytes);			// As Read but stop after '\n' or '\r\n' and null-terminate
 	bool Write(char b);								// Write 1 byte
 	bool Write(const char *s, size_t len);			// Write a block of len bytes
+	bool Write(const uint8_t *s, size_t len);		// Write a block of len bytes
 	bool Write(const char* s);						// Write a string
 	bool Close();									// Shut the file and tidy up
 	bool Seek(FilePosition pos);					// Jump to pos in the file
@@ -49,6 +49,7 @@ private:
 	Platform* platform;
 
 	FIL file;
+	FileWriteBuffer *writeBuffer;
 	volatile unsigned int openCount;
 	volatile bool closeRequested;
 
@@ -58,4 +59,6 @@ private:
 	static uint32_t longestWriteTime;
 };
 
+inline bool FileStore::Write(const uint8_t *s, size_t len) { return Write(reinterpret_cast<const char *>(s), len); }
+
 #endif

src/Storage/FileWriteBuffer.h

@@ -0,0 +1,57 @@
+/*
+ * FileWriteBuffer.h
+ *
+ *  Created on: 19 May 2017
+ *      Author: Christian
+ */
+
+#ifndef SRC_STORAGE_FILEWRITEBUFFER_H_
+#define SRC_STORAGE_FILEWRITEBUFFER_H_
+
+#include "RepRapFirmware.h"
+
+
+#ifdef DUET_NG
+const size_t NumFileWriteBuffers = 2;					// Number of write buffers
+const size_t FileWriteBufLen = 8192;					// Size of each write buffer
+#else
+const size_t NumFileWriteBuffers = 1;
+const size_t FileWriteBufLen = 4096;
+#endif
+
+
+// Class to cache data that is about to be written to the SD card. This is NOT a ring buffer,
+// instead it just provides simple interfaces to cache a certain amount of data so that fewer
+// f_write() calls are needed. This effectively improves upload speeds.
+class FileWriteBuffer
+{
+public:
+	FileWriteBuffer(FileWriteBuffer *n) : next(n), index(0) { }
+
+	FileWriteBuffer *Next() const { return next; }
+	void SetNext(FileWriteBuffer *n) { next = n; }
+
+	char *Data() { return reinterpret_cast<char *>(data32); }
+	const char *Data() const { return reinterpret_cast<const char *>(data32); }
+	const size_t BytesStored() const { return index; }
+	const size_t BytesLeft() const { return FileWriteBufLen - index; }
+
+	size_t Store(const char *data, size_t length);		// Stores some data and returns how much could be stored
+	void DataTaken() { index = 0; }						// Called to indicate that the buffer has been written
+
+private:
+	FileWriteBuffer *next;
+
+	size_t index;
+	int32_t data32[FileWriteBufLen / sizeof(int32_t)];	// 32-bit aligned buffer for better HSMCI performance
+};
+
+inline size_t FileWriteBuffer::Store(const char *data, size_t length)
+{
+	size_t bytesToStore = min<size_t>(BytesLeft(), length);
+	memcpy(Data() + index, data, bytesToStore);
+	index += bytesToStore;
+	return bytesToStore;
+}
+
+#endif

src/Storage/MassStorage.cpp

@@ -55,6 +55,12 @@ MassStorage::MassStorage(Platform* p) : platform(p)
 
 void MassStorage::Init()
 {
+	freeWriteBuffers = nullptr;
+	for (size_t i = 0; i < NumFileWriteBuffers; ++i)
+	{
+		freeWriteBuffers = new FileWriteBuffer(freeWriteBuffers);
+	}
+
 	for (size_t i = 0; i < NumSdCards; ++i)
 	{
 		isMounted[i] = false;
@@ -73,6 +79,25 @@ void MassStorage::Init()
 	}
 }
 
+FileWriteBuffer *MassStorage::AllocateWriteBuffer()
+{
+	if (freeWriteBuffers == nullptr)
+	{
+		return nullptr;
+	}
+
+	FileWriteBuffer *buffer = freeWriteBuffers;
+	freeWriteBuffers = buffer->Next();
+	buffer->SetNext(nullptr);
+	return buffer;
+}
+
+void MassStorage::ReleaseWriteBuffer(FileWriteBuffer *buffer)
+{
+	buffer->SetNext(freeWriteBuffers);
+	freeWriteBuffers = buffer;
+}
+
 const char* MassStorage::CombineName(const char* directory, const char* fileName)
 {
 	size_t outIndex = 0;

src/Storage/MassStorage.h

@@ -3,6 +3,7 @@
 
 #include "RepRapFirmware.h"
 #include "Pins.h"
+#include "FileWriteBuffer.h"
 #include "Libraries/Fatfs/ff.h"
 #include <ctime>
 
@@ -39,12 +40,16 @@ public:
 	bool CheckDriveMounted(const char* path);
 
 friend class Platform;
+friend class FileStore;
 
 protected:
 
 	MassStorage(Platform* p);
 	void Init();
 
+	FileWriteBuffer *AllocateWriteBuffer();
+	void ReleaseWriteBuffer(FileWriteBuffer *buffer);
+
 private:
 	static time_t ConvertTimeStamp(uint16_t fdate, uint16_t ftime);
 
@@ -53,6 +58,8 @@ private:
 	DIR findDir;
 	bool isMounted[NumSdCards];
 	char combinedName[FILENAME_LENGTH + 1];
+
+	FileWriteBuffer *freeWriteBuffers;
 };
 
 #endif

src/Version.h

@@ -9,11 +9,11 @@
 #define SRC_VERSION_H_
 
 #ifndef VERSION
-# define VERSION "1.19alpha3"
+# define VERSION "1.19beta1"
 #endif
 
 #ifndef DATE
-# define DATE "2017-05-09"
+# define DATE "2017-05-21"
 #endif
 
 #define AUTHORS "reprappro, dc42, chrishamm, t3p3, dnewman"