]> git.r.bdr.sh - rbdr/mobius/blob - hotline/file_transfer.go
Extensive refactor and clean up
[rbdr/mobius] / hotline / file_transfer.go
1 package hotline
2
3 import (
4 "bufio"
5 "crypto/rand"
6 "encoding/binary"
7 "errors"
8 "fmt"
9 "io"
10 "io/fs"
11 "log/slog"
12 "math"
13 "os"
14 "path/filepath"
15 "strings"
16 "sync"
17 )
18
19 // Folder download actions. Send by the client to indicate the next action the server should take
20 // for a folder download.
21 const (
22 DlFldrActionSendFile = 1
23 DlFldrActionResumeFile = 2
24 DlFldrActionNextFile = 3
25 )
26
27 // File transfer types
28 type FileTransferType uint8
29
30 const (
31 FileDownload = FileTransferType(0)
32 FileUpload = FileTransferType(1)
33 FolderDownload = FileTransferType(2)
34 FolderUpload = FileTransferType(3)
35 BannerDownload = FileTransferType(4)
36 )
37
38 type FileTransferID [4]byte
39
40 type FileTransferMgr interface {
41 Add(ft *FileTransfer)
42 Get(id FileTransferID) *FileTransfer
43 Delete(id FileTransferID)
44 }
45
46 type MemFileTransferMgr struct {
47 fileTransfers map[FileTransferID]*FileTransfer
48
49 mu sync.Mutex
50 }
51
52 func NewMemFileTransferMgr() *MemFileTransferMgr {
53 return &MemFileTransferMgr{
54 fileTransfers: make(map[FileTransferID]*FileTransfer),
55 }
56 }
57
58 func (ftm *MemFileTransferMgr) Add(ft *FileTransfer) {
59 ftm.mu.Lock()
60 defer ftm.mu.Unlock()
61
62 _, _ = rand.Read(ft.refNum[:])
63
64 ftm.fileTransfers[ft.refNum] = ft
65
66 ft.ClientConn.ClientFileTransferMgr.Add(ft.Type, ft)
67
68 //ft.ClientConn.transfersMU.Lock()
69 //ft.ClientConn.transfers[ft.Type] = ft
70 //ft.ClientConn.transfersMU.Unlock()
71 }
72
73 func (ftm *MemFileTransferMgr) Get(id FileTransferID) *FileTransfer {
74 ftm.mu.Lock()
75 defer ftm.mu.Unlock()
76
77 return ftm.fileTransfers[id]
78 }
79
80 func (ftm *MemFileTransferMgr) Delete(id FileTransferID) {
81 ftm.mu.Lock()
82 defer ftm.mu.Unlock()
83
84 ft := ftm.fileTransfers[id]
85
86 //ft.ClientConn.transfersMU.Lock()
87 //delete(ft.ClientConn.transfers[ft.Type], ft.refNum)
88 //ft.ClientConn.transfersMU.Unlock()
89 ft.ClientConn.ClientFileTransferMgr.Delete(ft.Type, id)
90
91 delete(ftm.fileTransfers, id)
92
93 }
94
95 type FileTransfer struct {
96 FileName []byte
97 FilePath []byte
98 refNum [4]byte
99 Type FileTransferType
100 TransferSize []byte
101 FolderItemCount []byte
102 fileResumeData *FileResumeData
103 options []byte
104 bytesSentCounter *WriteCounter
105 ClientConn *ClientConn
106 }
107
108 // WriteCounter counts the number of bytes written to it.
109 type WriteCounter struct {
110 mux sync.Mutex
111 Total int64 // Total # of bytes written
112 }
113
114 // Write implements the io.Writer interface.
115 //
116 // Always completes and never returns an error.
117 func (wc *WriteCounter) Write(p []byte) (int, error) {
118 wc.mux.Lock()
119 defer wc.mux.Unlock()
120 n := len(p)
121 wc.Total += int64(n)
122 return n, nil
123 }
124
125 func (cc *ClientConn) newFileTransfer(transferType FileTransferType, fileName, filePath, size []byte) *FileTransfer {
126 ft := &FileTransfer{
127 FileName: fileName,
128 FilePath: filePath,
129 Type: transferType,
130 TransferSize: size,
131 ClientConn: cc,
132 bytesSentCounter: &WriteCounter{},
133 }
134 //
135 //_, _ = rand.Read(ft.refNum[:])
136 //
137 //cc.transfersMU.Lock()
138 //defer cc.transfersMU.Unlock()
139 //cc.transfers[transferType][ft.refNum] = ft
140
141 cc.Server.FileTransferMgr.Add(ft)
142
143 //cc.Server.mux.Lock()
144 //defer cc.Server.mux.Unlock()
145 //cc.Server.fileTransfers[ft.refNum] = ft
146
147 return ft
148 }
149
150 // String returns a string representation of a file transfer and its progress for display in the GetInfo window
151 // Example:
152 // MasterOfOrionII1.4.0. 0% 197.9M
153 func (ft *FileTransfer) String() string {
154 trunc := fmt.Sprintf("%.21s", ft.FileName)
155 return fmt.Sprintf("%-21s %.3s%% %6s\n", trunc, ft.percentComplete(), ft.formattedTransferSize())
156 }
157
158 func (ft *FileTransfer) percentComplete() string {
159 ft.bytesSentCounter.mux.Lock()
160 defer ft.bytesSentCounter.mux.Unlock()
161 return fmt.Sprintf(
162 "%v",
163 math.RoundToEven(float64(ft.bytesSentCounter.Total)/float64(binary.BigEndian.Uint32(ft.TransferSize))*100),
164 )
165 }
166
167 func (ft *FileTransfer) formattedTransferSize() string {
168 sizeInKB := float32(binary.BigEndian.Uint32(ft.TransferSize)) / 1024
169 if sizeInKB >= 1024 {
170 return fmt.Sprintf("%.1fM", sizeInKB/1024)
171 } else {
172 return fmt.Sprintf("%.0fK", sizeInKB)
173 }
174 }
175
176 func (ft *FileTransfer) ItemCount() int {
177 return int(binary.BigEndian.Uint16(ft.FolderItemCount))
178 }
179
180 type folderUpload struct {
181 DataSize [2]byte
182 IsFolder [2]byte
183 PathItemCount [2]byte
184 FileNamePath []byte
185 }
186
187 //func (fu *folderUpload) Write(p []byte) (int, error) {
188 // if len(p) < 7 {
189 // return 0, errors.New("buflen too short")
190 // }
191 // copy(fu.DataSize[:], p[0:2])
192 // copy(fu.IsFolder[:], p[2:4])
193 // copy(fu.PathItemCount[:], p[4:6])
194 //
195 // fu.FileNamePath = make([]byte, binary.BigEndian.Uint16(fu.DataSize[:])-4) // -4 to subtract the path separator bytes TODO: wat
196 // n, err := io.ReadFull(rwc, fu.FileNamePath)
197 // if err != nil {
198 // return 0, err
199 // }
200 //
201 // return n + 6, nil
202 //}
203
204 func (fu *folderUpload) FormattedPath() string {
205 pathItemLen := binary.BigEndian.Uint16(fu.PathItemCount[:])
206
207 var pathSegments []string
208 pathData := fu.FileNamePath
209
210 // TODO: implement scanner interface instead?
211 for i := uint16(0); i < pathItemLen; i++ {
212 segLen := pathData[2]
213 pathSegments = append(pathSegments, string(pathData[3:3+segLen]))
214 pathData = pathData[3+segLen:]
215 }
216
217 return filepath.Join(pathSegments...)
218 }
219
220 func DownloadHandler(w io.Writer, fullPath string, fileTransfer *FileTransfer, fs FileStore, rLogger *slog.Logger, preserveForks bool) error {
221 //s.Stats.DownloadCounter += 1
222 //s.Stats.DownloadsInProgress += 1
223 //defer func() {
224 // s.Stats.DownloadsInProgress -= 1
225 //}()
226
227 var dataOffset int64
228 if fileTransfer.fileResumeData != nil {
229 dataOffset = int64(binary.BigEndian.Uint32(fileTransfer.fileResumeData.ForkInfoList[0].DataSize[:]))
230 }
231
232 fw, err := newFileWrapper(fs, fullPath, 0)
233 if err != nil {
234 return fmt.Errorf("reading file header: %v", err)
235 }
236
237 rLogger.Info("Download file", "filePath", fullPath)
238
239 // If file transfer options are included, that means this is a "quick preview" request. In this case skip sending
240 // the flat file info and proceed directly to sending the file data.
241 if fileTransfer.options == nil {
242 if _, err = io.Copy(w, fw.ffo); err != nil {
243 return fmt.Errorf("send flat file object: %v", err)
244 }
245 }
246
247 file, err := fw.dataForkReader()
248 if err != nil {
249 return fmt.Errorf("open data fork reader: %v", err)
250 }
251
252 br := bufio.NewReader(file)
253 if _, err := br.Discard(int(dataOffset)); err != nil {
254 return fmt.Errorf("seek to resume offsent: %v", err)
255 }
256
257 if _, err = io.Copy(w, io.TeeReader(br, fileTransfer.bytesSentCounter)); err != nil {
258 return fmt.Errorf("send data fork: %v", err)
259 }
260
261 // If the client requested to resume transfer, do not send the resource fork header.
262 if fileTransfer.fileResumeData == nil {
263 err = binary.Write(w, binary.BigEndian, fw.rsrcForkHeader())
264 if err != nil {
265 return fmt.Errorf("send resource fork header: %v", err)
266 }
267 }
268
269 rFile, err := fw.rsrcForkFile()
270 if err != nil {
271 // return fmt.Errorf("open resource fork file: %v", err)
272 }
273
274 if _, err = io.Copy(w, io.TeeReader(rFile, fileTransfer.bytesSentCounter)); err != nil {
275 // return fmt.Errorf("send resource fork data: %v", err)
276 }
277
278 return nil
279 }
280
281 func UploadHandler(rwc io.ReadWriter, fullPath string, fileTransfer *FileTransfer, fileStore FileStore, rLogger *slog.Logger, preserveForks bool) error {
282 var file *os.File
283
284 // A file upload has two possible cases:
285 // 1) Upload a new file
286 // 2) Resume a partially transferred file
287 // We have to infer which case applies by inspecting what is already on the filesystem
288
289 // Check for existing file. If found, do not proceed. This is an invalid scenario, as the file upload transaction
290 // handler should have returned an error to the client indicating there was an existing file present.
291 _, err := os.Stat(fullPath)
292 if err == nil {
293 return fmt.Errorf("existing file found: %s", fullPath)
294 }
295 if errors.Is(err, fs.ErrNotExist) {
296 // If not found, open or create a new .incomplete file
297 file, err = os.OpenFile(fullPath+incompleteFileSuffix, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
298 if err != nil {
299 return err
300 }
301 }
302
303 f, err := newFileWrapper(fileStore, fullPath, 0)
304 if err != nil {
305 return err
306 }
307
308 rLogger.Info("File upload started", "dstFile", fullPath)
309
310 rForkWriter := io.Discard
311 iForkWriter := io.Discard
312 if preserveForks {
313 rForkWriter, err = f.rsrcForkWriter()
314 if err != nil {
315 return err
316 }
317
318 iForkWriter, err = f.infoForkWriter()
319 if err != nil {
320 return err
321 }
322 }
323
324 if err := receiveFile(rwc, file, rForkWriter, iForkWriter, fileTransfer.bytesSentCounter); err != nil {
325 rLogger.Error(err.Error())
326 }
327
328 if err := file.Close(); err != nil {
329 return err
330 }
331
332 if err := fileStore.Rename(fullPath+".incomplete", fullPath); err != nil {
333 return err
334 }
335
336 rLogger.Info("File upload complete", "dstFile", fullPath)
337
338 return nil
339 }
340
341 func DownloadFolderHandler(rwc io.ReadWriter, fullPath string, fileTransfer *FileTransfer, fileStore FileStore, rLogger *slog.Logger, preserveForks bool) error {
342 // Folder Download flow:
343 // 1. Get filePath from the transfer
344 // 2. Iterate over files
345 // 3. For each fileWrapper:
346 // Send fileWrapper header to client
347 // The client can reply in 3 ways:
348 //
349 // 1. If type is an odd number (unknown type?), or fileWrapper download for the current fileWrapper is completed:
350 // client sends []byte{0x00, 0x03} to tell the server to continue to the next fileWrapper
351 //
352 // 2. If download of a fileWrapper is to be resumed:
353 // client sends:
354 // []byte{0x00, 0x02} // download folder action
355 // [2]byte // Resume data size
356 // []byte fileWrapper resume data (see myField_FileResumeData)
357 //
358 // 3. Otherwise, download of the fileWrapper is requested and client sends []byte{0x00, 0x01}
359 //
360 // When download is requested (case 2 or 3), server replies with:
361 // [4]byte - fileWrapper size
362 // []byte - Flattened File Object
363 //
364 // After every fileWrapper download, client could request next fileWrapper with:
365 // []byte{0x00, 0x03}
366 //
367 // This notifies the server to send the next item header
368
369 basePathLen := len(fullPath)
370
371 rLogger.Info("Start folder download", "path", fullPath)
372
373 nextAction := make([]byte, 2)
374 if _, err := io.ReadFull(rwc, nextAction); err != nil {
375 return err
376 }
377
378 i := 0
379 err := filepath.Walk(fullPath+"/", func(path string, info os.FileInfo, err error) error {
380 //s.Stats.DownloadCounter += 1
381 i += 1
382
383 if err != nil {
384 return err
385 }
386
387 // skip dot files
388 if strings.HasPrefix(info.Name(), ".") {
389 return nil
390 }
391
392 hlFile, err := newFileWrapper(fileStore, path, 0)
393 if err != nil {
394 return err
395 }
396
397 subPath := path[basePathLen+1:]
398 rLogger.Debug("Sending fileheader", "i", i, "path", path, "fullFilePath", fullPath, "subPath", subPath, "IsDir", info.IsDir())
399
400 if i == 1 {
401 return nil
402 }
403
404 fileHeader := NewFileHeader(subPath, info.IsDir())
405 if _, err := io.Copy(rwc, &fileHeader); err != nil {
406 return fmt.Errorf("error sending file header: %w", err)
407 }
408
409 // Read the client's Next Action request
410 if _, err := io.ReadFull(rwc, nextAction); err != nil {
411 return err
412 }
413
414 rLogger.Debug("Client folder download action", "action", fmt.Sprintf("%X", nextAction[0:2]))
415
416 var dataOffset int64
417
418 switch nextAction[1] {
419 case DlFldrActionResumeFile:
420 // get size of resumeData
421 resumeDataByteLen := make([]byte, 2)
422 if _, err := io.ReadFull(rwc, resumeDataByteLen); err != nil {
423 return err
424 }
425
426 resumeDataLen := binary.BigEndian.Uint16(resumeDataByteLen)
427 resumeDataBytes := make([]byte, resumeDataLen)
428 if _, err := io.ReadFull(rwc, resumeDataBytes); err != nil {
429 return err
430 }
431
432 var frd FileResumeData
433 if err := frd.UnmarshalBinary(resumeDataBytes); err != nil {
434 return err
435 }
436 dataOffset = int64(binary.BigEndian.Uint32(frd.ForkInfoList[0].DataSize[:]))
437 case DlFldrActionNextFile:
438 // client asked to skip this file
439 return nil
440 }
441
442 if info.IsDir() {
443 return nil
444 }
445
446 rLogger.Info("File download started",
447 "fileName", info.Name(),
448 "TransferSize", fmt.Sprintf("%x", hlFile.ffo.TransferSize(dataOffset)),
449 )
450
451 // Send file size to client
452 if _, err := rwc.Write(hlFile.ffo.TransferSize(dataOffset)); err != nil {
453 rLogger.Error(err.Error())
454 return fmt.Errorf("error sending file size: %w", err)
455 }
456
457 // Send ffo bytes to client
458 _, err = io.Copy(rwc, hlFile.ffo)
459 if err != nil {
460 return fmt.Errorf("error sending flat file object: %w", err)
461 }
462
463 file, err := fileStore.Open(path)
464 if err != nil {
465 return fmt.Errorf("error opening file: %w", err)
466 }
467
468 // wr := bufio.NewWriterSize(rwc, 1460)
469 if _, err = io.Copy(rwc, io.TeeReader(file, fileTransfer.bytesSentCounter)); err != nil {
470 return fmt.Errorf("error sending file: %w", err)
471 }
472
473 if nextAction[1] != 2 && hlFile.ffo.FlatFileHeader.ForkCount[1] == 3 {
474 err = binary.Write(rwc, binary.BigEndian, hlFile.rsrcForkHeader())
475 if err != nil {
476 return fmt.Errorf("error sending resource fork header: %w", err)
477 }
478
479 rFile, err := hlFile.rsrcForkFile()
480 if err != nil {
481 return fmt.Errorf("error opening resource fork: %w", err)
482 }
483
484 if _, err = io.Copy(rwc, io.TeeReader(rFile, fileTransfer.bytesSentCounter)); err != nil {
485 return fmt.Errorf("error sending resource fork: %w", err)
486 }
487 }
488
489 // Read the client's Next Action request. This is always 3, I think?
490 if _, err := io.ReadFull(rwc, nextAction); err != nil && err != io.EOF {
491 return fmt.Errorf("error reading client next action: %w", err)
492 }
493
494 return nil
495 })
496
497 if err != nil {
498 return err
499 }
500
501 return nil
502 }
503
504 func UploadFolderHandler(rwc io.ReadWriter, fullPath string, fileTransfer *FileTransfer, fileStore FileStore, rLogger *slog.Logger, preserveForks bool) error {
505
506 // Check if the target folder exists. If not, create it.
507 if _, err := fileStore.Stat(fullPath); os.IsNotExist(err) {
508 if err := fileStore.Mkdir(fullPath, 0777); err != nil {
509 return err
510 }
511 }
512
513 // Begin the folder upload flow by sending the "next file action" to client
514 if _, err := rwc.Write([]byte{0, DlFldrActionNextFile}); err != nil {
515 return err
516 }
517
518 fileSize := make([]byte, 4)
519
520 for i := 0; i < fileTransfer.ItemCount(); i++ {
521 //s.Stats.UploadCounter += 1
522
523 var fu folderUpload
524 // TODO: implement io.Writer on folderUpload and replace this
525 if _, err := io.ReadFull(rwc, fu.DataSize[:]); err != nil {
526 return err
527 }
528 if _, err := io.ReadFull(rwc, fu.IsFolder[:]); err != nil {
529 return err
530 }
531 if _, err := io.ReadFull(rwc, fu.PathItemCount[:]); err != nil {
532 return err
533 }
534 fu.FileNamePath = make([]byte, binary.BigEndian.Uint16(fu.DataSize[:])-4) // -4 to subtract the path separator bytes TODO: wat
535 if _, err := io.ReadFull(rwc, fu.FileNamePath); err != nil {
536 return err
537 }
538
539 if fu.IsFolder == [2]byte{0, 1} {
540 if _, err := os.Stat(filepath.Join(fullPath, fu.FormattedPath())); os.IsNotExist(err) {
541 if err := os.Mkdir(filepath.Join(fullPath, fu.FormattedPath()), 0777); err != nil {
542 return err
543 }
544 }
545
546 // Tell client to send next file
547 if _, err := rwc.Write([]byte{0, DlFldrActionNextFile}); err != nil {
548 return err
549 }
550 } else {
551 nextAction := DlFldrActionSendFile
552
553 // Check if we have the full file already. If so, send dlFldrAction_NextFile to client to skip.
554 _, err := os.Stat(filepath.Join(fullPath, fu.FormattedPath()))
555 if err != nil && !errors.Is(err, fs.ErrNotExist) {
556 return err
557 }
558 if err == nil {
559 nextAction = DlFldrActionNextFile
560 }
561
562 // Check if we have a partial file already. If so, send dlFldrAction_ResumeFile to client to resume upload.
563 incompleteFile, err := os.Stat(filepath.Join(fullPath, fu.FormattedPath()+incompleteFileSuffix))
564 if err != nil && !errors.Is(err, fs.ErrNotExist) {
565 return err
566 }
567 if err == nil {
568 nextAction = DlFldrActionResumeFile
569 }
570
571 if _, err := rwc.Write([]byte{0, uint8(nextAction)}); err != nil {
572 return err
573 }
574
575 switch nextAction {
576 case DlFldrActionNextFile:
577 continue
578 case DlFldrActionResumeFile:
579 offset := make([]byte, 4)
580 binary.BigEndian.PutUint32(offset, uint32(incompleteFile.Size()))
581
582 file, err := os.OpenFile(fullPath+"/"+fu.FormattedPath()+incompleteFileSuffix, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
583 if err != nil {
584 return err
585 }
586
587 fileResumeData := NewFileResumeData([]ForkInfoList{*NewForkInfoList(offset)})
588
589 b, _ := fileResumeData.BinaryMarshal()
590
591 bs := make([]byte, 2)
592 binary.BigEndian.PutUint16(bs, uint16(len(b)))
593
594 if _, err := rwc.Write(append(bs, b...)); err != nil {
595 return err
596 }
597
598 if _, err := io.ReadFull(rwc, fileSize); err != nil {
599 return err
600 }
601
602 if err := receiveFile(rwc, file, io.Discard, io.Discard, fileTransfer.bytesSentCounter); err != nil {
603 rLogger.Error(err.Error())
604 }
605
606 err = os.Rename(fullPath+"/"+fu.FormattedPath()+".incomplete", fullPath+"/"+fu.FormattedPath())
607 if err != nil {
608 return err
609 }
610
611 case DlFldrActionSendFile:
612 if _, err := io.ReadFull(rwc, fileSize); err != nil {
613 return err
614 }
615
616 filePath := filepath.Join(fullPath, fu.FormattedPath())
617
618 hlFile, err := newFileWrapper(fileStore, filePath, 0)
619 if err != nil {
620 return err
621 }
622
623 rLogger.Info("Starting file transfer", "path", filePath, "fileNum", i+1, "fileSize", binary.BigEndian.Uint32(fileSize))
624
625 incWriter, err := hlFile.incFileWriter()
626 if err != nil {
627 return err
628 }
629
630 rForkWriter := io.Discard
631 iForkWriter := io.Discard
632 if preserveForks {
633 iForkWriter, err = hlFile.infoForkWriter()
634 if err != nil {
635 return err
636 }
637
638 rForkWriter, err = hlFile.rsrcForkWriter()
639 if err != nil {
640 return err
641 }
642 }
643 if err := receiveFile(rwc, incWriter, rForkWriter, iForkWriter, fileTransfer.bytesSentCounter); err != nil {
644 return err
645 }
646
647 if err := os.Rename(filePath+".incomplete", filePath); err != nil {
648 return err
649 }
650 }
651
652 // Tell client to send next fileWrapper
653 if _, err := rwc.Write([]byte{0, DlFldrActionNextFile}); err != nil {
654 return err
655 }
656 }
657 }
658 rLogger.Info("Folder upload complete")
659 return nil
660 }