]> git.r.bdr.sh - rbdr/mobius/blob - hotline/file_transfer.go
Add option for account-specific file root
[rbdr/mobius] / hotline / file_transfer.go
1 package hotline
2
3 import (
4 "bufio"
5 "crypto/rand"
6 "encoding/binary"
7 "errors"
8 "fmt"
9 "io"
10 "io/fs"
11 "log/slog"
12 "math"
13 "os"
14 "path/filepath"
15 "slices"
16 "strings"
17 "sync"
18 )
19
20 // Folder download actions. Send by the client to indicate the next action the server should take
21 // for a folder download.
22 const (
23 DlFldrActionSendFile = 1
24 DlFldrActionResumeFile = 2
25 DlFldrActionNextFile = 3
26 )
27
28 // File transfer types
29 type FileTransferType uint8
30
31 const (
32 FileDownload = FileTransferType(0)
33 FileUpload = FileTransferType(1)
34 FolderDownload = FileTransferType(2)
35 FolderUpload = FileTransferType(3)
36 BannerDownload = FileTransferType(4)
37 )
38
39 type FileTransferID [4]byte
40
41 type FileTransferMgr interface {
42 Add(ft *FileTransfer)
43 Get(id FileTransferID) *FileTransfer
44 Delete(id FileTransferID)
45 }
46
47 type MemFileTransferMgr struct {
48 fileTransfers map[FileTransferID]*FileTransfer
49
50 mu sync.Mutex
51 }
52
53 func NewMemFileTransferMgr() *MemFileTransferMgr {
54 return &MemFileTransferMgr{
55 fileTransfers: make(map[FileTransferID]*FileTransfer),
56 }
57 }
58
59 func (ftm *MemFileTransferMgr) Add(ft *FileTransfer) {
60 ftm.mu.Lock()
61 defer ftm.mu.Unlock()
62
63 _, _ = rand.Read(ft.RefNum[:])
64
65 ftm.fileTransfers[ft.RefNum] = ft
66
67 ft.ClientConn.ClientFileTransferMgr.Add(ft.Type, ft)
68 }
69
70 func (ftm *MemFileTransferMgr) Get(id FileTransferID) *FileTransfer {
71 ftm.mu.Lock()
72 defer ftm.mu.Unlock()
73
74 return ftm.fileTransfers[id]
75 }
76
77 func (ftm *MemFileTransferMgr) Delete(id FileTransferID) {
78 ftm.mu.Lock()
79 defer ftm.mu.Unlock()
80
81 ft := ftm.fileTransfers[id]
82
83 ft.ClientConn.ClientFileTransferMgr.Delete(ft.Type, id)
84
85 delete(ftm.fileTransfers, id)
86
87 }
88
89 type FileTransfer struct {
90 FileRoot string
91 FileName []byte
92 FilePath []byte
93 RefNum [4]byte
94 Type FileTransferType
95 TransferSize []byte
96 FolderItemCount []byte
97 FileResumeData *FileResumeData
98 Options []byte
99 bytesSentCounter *WriteCounter
100 ClientConn *ClientConn
101 }
102
103 // WriteCounter counts the number of bytes written to it.
104 type WriteCounter struct {
105 mux sync.Mutex
106 Total int64 // Total # of bytes written
107 }
108
109 // Write implements the io.Writer interface.
110 //
111 // Always completes and never returns an error.
112 func (wc *WriteCounter) Write(p []byte) (int, error) {
113 wc.mux.Lock()
114 defer wc.mux.Unlock()
115 n := len(p)
116 wc.Total += int64(n)
117 return n, nil
118 }
119
120 func (cc *ClientConn) NewFileTransfer(transferType FileTransferType, fileroot string, fileName, filePath, size []byte) *FileTransfer {
121 ft := &FileTransfer{
122 FileName: fileName,
123 FileRoot: fileroot,
124 FilePath: filePath,
125 Type: transferType,
126 TransferSize: size,
127 ClientConn: cc,
128 bytesSentCounter: &WriteCounter{},
129 }
130
131 cc.Server.FileTransferMgr.Add(ft)
132
133 return ft
134 }
135
136 // String returns a string representation of a file transfer and its progress for display in the GetInfo window
137 // Example:
138 // MasterOfOrionII1.4.0. 0% 197.9M
139 func (ft *FileTransfer) String() string {
140 trunc := fmt.Sprintf("%.21s", ft.FileName)
141 return fmt.Sprintf("%-21s %.3s%% %6s\n", trunc, ft.percentComplete(), ft.formattedTransferSize())
142 }
143
144 func (ft *FileTransfer) percentComplete() string {
145 ft.bytesSentCounter.mux.Lock()
146 defer ft.bytesSentCounter.mux.Unlock()
147 return fmt.Sprintf(
148 "%v",
149 math.RoundToEven(float64(ft.bytesSentCounter.Total)/float64(binary.BigEndian.Uint32(ft.TransferSize))*100),
150 )
151 }
152
153 func (ft *FileTransfer) formattedTransferSize() string {
154 sizeInKB := float32(binary.BigEndian.Uint32(ft.TransferSize)) / 1024
155 if sizeInKB >= 1024 {
156 return fmt.Sprintf("%.1fM", sizeInKB/1024)
157 } else {
158 return fmt.Sprintf("%.0fK", sizeInKB)
159 }
160 }
161
162 func (ft *FileTransfer) ItemCount() int {
163 return int(binary.BigEndian.Uint16(ft.FolderItemCount))
164 }
165
166 type folderUpload struct {
167 DataSize [2]byte
168 IsFolder [2]byte
169 PathItemCount [2]byte
170 FileNamePath []byte
171 }
172
173 //func (fu *folderUpload) Write(p []byte) (int, error) {
174 // if len(p) < 7 {
175 // return 0, errors.New("buflen too short")
176 // }
177 // copy(fu.DataSize[:], p[0:2])
178 // copy(fu.IsFolder[:], p[2:4])
179 // copy(fu.PathItemCount[:], p[4:6])
180 //
181 // fu.FileNamePath = make([]byte, binary.BigEndian.Uint16(fu.DataSize[:])-4) // -4 to subtract the path separator bytes TODO: wat
182 // n, err := io.ReadFull(rwc, fu.FileNamePath)
183 // if err != nil {
184 // return 0, err
185 // }
186 //
187 // return n + 6, nil
188 //}
189
190 func (fu *folderUpload) FormattedPath() string {
191 pathItemLen := binary.BigEndian.Uint16(fu.PathItemCount[:])
192
193 var pathSegments []string
194 pathData := fu.FileNamePath
195
196 // TODO: implement scanner interface instead?
197 for i := uint16(0); i < pathItemLen; i++ {
198 segLen := pathData[2]
199 pathSegments = append(pathSegments, string(pathData[3:3+segLen]))
200 pathData = pathData[3+segLen:]
201 }
202
203 return filepath.Join(pathSegments...)
204 }
205
206 type FileHeader struct {
207 Size [2]byte // Total size of FileHeader payload
208 Type [2]byte // 0 for file, 1 for dir
209 FilePath []byte // encoded file path
210
211 readOffset int // Internal offset to track read progress
212 }
213
214 func NewFileHeader(fileName string, isDir bool) FileHeader {
215 fh := FileHeader{
216 FilePath: EncodeFilePath(fileName),
217 }
218 if isDir {
219 fh.Type = [2]byte{0x00, 0x01}
220 }
221
222 encodedPathLen := uint16(len(fh.FilePath) + len(fh.Type))
223 binary.BigEndian.PutUint16(fh.Size[:], encodedPathLen)
224
225 return fh
226 }
227
228 func (fh *FileHeader) Read(p []byte) (int, error) {
229 buf := slices.Concat(
230 fh.Size[:],
231 fh.Type[:],
232 fh.FilePath,
233 )
234
235 if fh.readOffset >= len(buf) {
236 return 0, io.EOF // All bytes have been read
237 }
238
239 n := copy(p, buf[fh.readOffset:])
240 fh.readOffset += n
241
242 return n, nil
243 }
244
245 func DownloadHandler(w io.Writer, fullPath string, fileTransfer *FileTransfer, fs FileStore, rLogger *slog.Logger, preserveForks bool) error {
246 //s.Stats.DownloadCounter += 1
247 //s.Stats.DownloadsInProgress += 1
248 //defer func() {
249 // s.Stats.DownloadsInProgress -= 1
250 //}()
251
252 var dataOffset int64
253 if fileTransfer.FileResumeData != nil {
254 dataOffset = int64(binary.BigEndian.Uint32(fileTransfer.FileResumeData.ForkInfoList[0].DataSize[:]))
255 }
256
257 fw, err := NewFileWrapper(fs, fullPath, 0)
258 if err != nil {
259 return fmt.Errorf("reading file header: %v", err)
260 }
261
262 rLogger.Info("Download file", "filePath", fullPath)
263
264 // If file transfer options are included, that means this is a "quick preview" request. In this case skip sending
265 // the flat file info and proceed directly to sending the file data.
266 if fileTransfer.Options == nil {
267 if _, err = io.Copy(w, fw.Ffo); err != nil {
268 return fmt.Errorf("send flat file object: %v", err)
269 }
270 }
271
272 file, err := fw.dataForkReader()
273 if err != nil {
274 return fmt.Errorf("open data fork reader: %v", err)
275 }
276
277 br := bufio.NewReader(file)
278 if _, err := br.Discard(int(dataOffset)); err != nil {
279 return fmt.Errorf("seek to resume offsent: %v", err)
280 }
281
282 if _, err = io.Copy(w, io.TeeReader(br, fileTransfer.bytesSentCounter)); err != nil {
283 return fmt.Errorf("send data fork: %v", err)
284 }
285
286 // If the client requested to resume transfer, do not send the resource fork header.
287 if fileTransfer.FileResumeData == nil {
288 err = binary.Write(w, binary.BigEndian, fw.rsrcForkHeader())
289 if err != nil {
290 return fmt.Errorf("send resource fork header: %v", err)
291 }
292 }
293
294 rFile, _ := fw.rsrcForkFile()
295 //if err != nil {
296 // // return fmt.Errorf("open resource fork file: %v", err)
297 //}
298
299 _, _ = io.Copy(w, io.TeeReader(rFile, fileTransfer.bytesSentCounter))
300 //if err != nil {
301 // // return fmt.Errorf("send resource fork data: %v", err)
302 //}
303
304 return nil
305 }
306
307 func UploadHandler(rwc io.ReadWriter, fullPath string, fileTransfer *FileTransfer, fileStore FileStore, rLogger *slog.Logger, preserveForks bool) error {
308 var file *os.File
309
310 // A file upload has two possible cases:
311 // 1) Upload a new file
312 // 2) Resume a partially transferred file
313 // We have to infer which case applies by inspecting what is already on the filesystem
314
315 // Check for existing file. If found, do not proceed. This is an invalid scenario, as the file upload transaction
316 // handler should have returned an error to the client indicating there was an existing file present.
317 _, err := os.Stat(fullPath)
318 if err == nil {
319 return fmt.Errorf("existing file found: %s", fullPath)
320 }
321 if errors.Is(err, fs.ErrNotExist) {
322 // If not found, open or create a new .incomplete file
323 file, err = os.OpenFile(fullPath+IncompleteFileSuffix, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
324 if err != nil {
325 return err
326 }
327 }
328
329 f, err := NewFileWrapper(fileStore, fullPath, 0)
330 if err != nil {
331 return err
332 }
333
334 rLogger.Info("File upload started", "dstFile", fullPath)
335
336 rForkWriter := io.Discard
337 iForkWriter := io.Discard
338 if preserveForks {
339 rForkWriter, err = f.rsrcForkWriter()
340 if err != nil {
341 return err
342 }
343
344 iForkWriter, err = f.InfoForkWriter()
345 if err != nil {
346 return err
347 }
348 }
349
350 if err := receiveFile(rwc, file, rForkWriter, iForkWriter, fileTransfer.bytesSentCounter); err != nil {
351 rLogger.Error(err.Error())
352 }
353
354 if err := file.Close(); err != nil {
355 return err
356 }
357
358 if err := fileStore.Rename(fullPath+".incomplete", fullPath); err != nil {
359 return err
360 }
361
362 rLogger.Info("File upload complete", "dstFile", fullPath)
363
364 return nil
365 }
366
367 func DownloadFolderHandler(rwc io.ReadWriter, fullPath string, fileTransfer *FileTransfer, fileStore FileStore, rLogger *slog.Logger, preserveForks bool) error {
368 // Folder Download flow:
369 // 1. Get filePath from the transfer
370 // 2. Iterate over files
371 // 3. For each fileWrapper:
372 // Send fileWrapper header to client
373 // The client can reply in 3 ways:
374 //
375 // 1. If type is an odd number (unknown type?), or fileWrapper download for the current fileWrapper is completed:
376 // client sends []byte{0x00, 0x03} to tell the server to continue to the next fileWrapper
377 //
378 // 2. If download of a fileWrapper is to be resumed:
379 // client sends:
380 // []byte{0x00, 0x02} // download folder action
381 // [2]byte // Resume data size
382 // []byte fileWrapper resume data (see myField_FileResumeData)
383 //
384 // 3. Otherwise, download of the fileWrapper is requested and client sends []byte{0x00, 0x01}
385 //
386 // When download is requested (case 2 or 3), server replies with:
387 // [4]byte - fileWrapper size
388 // []byte - Flattened File Object
389 //
390 // After every fileWrapper download, client could request next fileWrapper with:
391 // []byte{0x00, 0x03}
392 //
393 // This notifies the server to send the next item header
394
395 basePathLen := len(fullPath)
396
397 rLogger.Info("Start folder download", "path", fullPath)
398
399 nextAction := make([]byte, 2)
400 if _, err := io.ReadFull(rwc, nextAction); err != nil {
401 return err
402 }
403
404 i := 0
405 err := filepath.Walk(fullPath+"/", func(path string, info os.FileInfo, err error) error {
406 //s.Stats.DownloadCounter += 1
407 i += 1
408
409 if err != nil {
410 return err
411 }
412
413 // skip dot files
414 if strings.HasPrefix(info.Name(), ".") {
415 return nil
416 }
417
418 hlFile, err := NewFileWrapper(fileStore, path, 0)
419 if err != nil {
420 return err
421 }
422
423 subPath := path[basePathLen+1:]
424 rLogger.Debug("Sending fileheader", "i", i, "path", path, "fullFilePath", fullPath, "subPath", subPath, "IsDir", info.IsDir())
425
426 if i == 1 {
427 return nil
428 }
429
430 fileHeader := NewFileHeader(subPath, info.IsDir())
431 if _, err := io.Copy(rwc, &fileHeader); err != nil {
432 return fmt.Errorf("error sending file header: %w", err)
433 }
434
435 // Read the client's Next Action request
436 if _, err := io.ReadFull(rwc, nextAction); err != nil {
437 return err
438 }
439
440 rLogger.Debug("Client folder download action", "action", fmt.Sprintf("%X", nextAction[0:2]))
441
442 var dataOffset int64
443
444 switch nextAction[1] {
445 case DlFldrActionResumeFile:
446 // get size of resumeData
447 resumeDataByteLen := make([]byte, 2)
448 if _, err := io.ReadFull(rwc, resumeDataByteLen); err != nil {
449 return err
450 }
451
452 resumeDataLen := binary.BigEndian.Uint16(resumeDataByteLen)
453 resumeDataBytes := make([]byte, resumeDataLen)
454 if _, err := io.ReadFull(rwc, resumeDataBytes); err != nil {
455 return err
456 }
457
458 var frd FileResumeData
459 if err := frd.UnmarshalBinary(resumeDataBytes); err != nil {
460 return err
461 }
462 dataOffset = int64(binary.BigEndian.Uint32(frd.ForkInfoList[0].DataSize[:]))
463 case DlFldrActionNextFile:
464 // client asked to skip this file
465 return nil
466 }
467
468 if info.IsDir() {
469 return nil
470 }
471
472 rLogger.Info("File download started",
473 "fileName", info.Name(),
474 "TransferSize", fmt.Sprintf("%x", hlFile.Ffo.TransferSize(dataOffset)),
475 )
476
477 // Send file size to client
478 if _, err := rwc.Write(hlFile.Ffo.TransferSize(dataOffset)); err != nil {
479 rLogger.Error(err.Error())
480 return fmt.Errorf("error sending file size: %w", err)
481 }
482
483 // Send ffo bytes to client
484 _, err = io.Copy(rwc, hlFile.Ffo)
485 if err != nil {
486 return fmt.Errorf("error sending flat file object: %w", err)
487 }
488
489 file, err := fileStore.Open(path)
490 if err != nil {
491 return fmt.Errorf("error opening file: %w", err)
492 }
493
494 // wr := bufio.NewWriterSize(rwc, 1460)
495 if _, err = io.Copy(rwc, io.TeeReader(file, fileTransfer.bytesSentCounter)); err != nil {
496 return fmt.Errorf("error sending file: %w", err)
497 }
498
499 if nextAction[1] != 2 && hlFile.Ffo.FlatFileHeader.ForkCount[1] == 3 {
500 err = binary.Write(rwc, binary.BigEndian, hlFile.rsrcForkHeader())
501 if err != nil {
502 return fmt.Errorf("error sending resource fork header: %w", err)
503 }
504
505 rFile, err := hlFile.rsrcForkFile()
506 if err != nil {
507 return fmt.Errorf("error opening resource fork: %w", err)
508 }
509
510 if _, err = io.Copy(rwc, io.TeeReader(rFile, fileTransfer.bytesSentCounter)); err != nil {
511 return fmt.Errorf("error sending resource fork: %w", err)
512 }
513 }
514
515 // Read the client's Next Action request. This is always 3, I think?
516 if _, err := io.ReadFull(rwc, nextAction); err != nil && err != io.EOF {
517 return fmt.Errorf("error reading client next action: %w", err)
518 }
519
520 return nil
521 })
522
523 if err != nil {
524 return err
525 }
526
527 return nil
528 }
529
530 func UploadFolderHandler(rwc io.ReadWriter, fullPath string, fileTransfer *FileTransfer, fileStore FileStore, rLogger *slog.Logger, preserveForks bool) error {
531
532 // Check if the target folder exists. If not, create it.
533 if _, err := fileStore.Stat(fullPath); os.IsNotExist(err) {
534 if err := fileStore.Mkdir(fullPath, 0777); err != nil {
535 return err
536 }
537 }
538
539 // Begin the folder upload flow by sending the "next file action" to client
540 if _, err := rwc.Write([]byte{0, DlFldrActionNextFile}); err != nil {
541 return err
542 }
543
544 fileSize := make([]byte, 4)
545
546 for i := 0; i < fileTransfer.ItemCount(); i++ {
547 //s.Stats.UploadCounter += 1
548
549 var fu folderUpload
550 // TODO: implement io.Writer on folderUpload and replace this
551 if _, err := io.ReadFull(rwc, fu.DataSize[:]); err != nil {
552 return err
553 }
554 if _, err := io.ReadFull(rwc, fu.IsFolder[:]); err != nil {
555 return err
556 }
557 if _, err := io.ReadFull(rwc, fu.PathItemCount[:]); err != nil {
558 return err
559 }
560 fu.FileNamePath = make([]byte, binary.BigEndian.Uint16(fu.DataSize[:])-4) // -4 to subtract the path separator bytes TODO: wat
561 if _, err := io.ReadFull(rwc, fu.FileNamePath); err != nil {
562 return err
563 }
564
565 if fu.IsFolder == [2]byte{0, 1} {
566 if _, err := os.Stat(filepath.Join(fullPath, fu.FormattedPath())); os.IsNotExist(err) {
567 if err := os.Mkdir(filepath.Join(fullPath, fu.FormattedPath()), 0777); err != nil {
568 return err
569 }
570 }
571
572 // Tell client to send next file
573 if _, err := rwc.Write([]byte{0, DlFldrActionNextFile}); err != nil {
574 return err
575 }
576 } else {
577 nextAction := DlFldrActionSendFile
578
579 // Check if we have the full file already. If so, send dlFldrAction_NextFile to client to skip.
580 _, err := os.Stat(filepath.Join(fullPath, fu.FormattedPath()))
581 if err != nil && !errors.Is(err, fs.ErrNotExist) {
582 return err
583 }
584 if err == nil {
585 nextAction = DlFldrActionNextFile
586 }
587
588 // Check if we have a partial file already. If so, send dlFldrAction_ResumeFile to client to resume upload.
589 incompleteFile, err := os.Stat(filepath.Join(fullPath, fu.FormattedPath()+IncompleteFileSuffix))
590 if err != nil && !errors.Is(err, fs.ErrNotExist) {
591 return err
592 }
593 if err == nil {
594 nextAction = DlFldrActionResumeFile
595 }
596
597 if _, err := rwc.Write([]byte{0, uint8(nextAction)}); err != nil {
598 return err
599 }
600
601 switch nextAction {
602 case DlFldrActionNextFile:
603 continue
604 case DlFldrActionResumeFile:
605 offset := make([]byte, 4)
606 binary.BigEndian.PutUint32(offset, uint32(incompleteFile.Size()))
607
608 file, err := os.OpenFile(fullPath+"/"+fu.FormattedPath()+IncompleteFileSuffix, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
609 if err != nil {
610 return err
611 }
612
613 fileResumeData := NewFileResumeData([]ForkInfoList{*NewForkInfoList(offset)})
614
615 b, _ := fileResumeData.BinaryMarshal()
616
617 bs := make([]byte, 2)
618 binary.BigEndian.PutUint16(bs, uint16(len(b)))
619
620 if _, err := rwc.Write(append(bs, b...)); err != nil {
621 return err
622 }
623
624 if _, err := io.ReadFull(rwc, fileSize); err != nil {
625 return err
626 }
627
628 if err := receiveFile(rwc, file, io.Discard, io.Discard, fileTransfer.bytesSentCounter); err != nil {
629 rLogger.Error(err.Error())
630 }
631
632 err = os.Rename(fullPath+"/"+fu.FormattedPath()+".incomplete", fullPath+"/"+fu.FormattedPath())
633 if err != nil {
634 return err
635 }
636
637 case DlFldrActionSendFile:
638 if _, err := io.ReadFull(rwc, fileSize); err != nil {
639 return err
640 }
641
642 filePath := filepath.Join(fullPath, fu.FormattedPath())
643
644 hlFile, err := NewFileWrapper(fileStore, filePath, 0)
645 if err != nil {
646 return err
647 }
648
649 rLogger.Info("Starting file transfer", "path", filePath, "fileNum", i+1, "fileSize", binary.BigEndian.Uint32(fileSize))
650
651 incWriter, err := hlFile.incFileWriter()
652 if err != nil {
653 return err
654 }
655
656 rForkWriter := io.Discard
657 iForkWriter := io.Discard
658 if preserveForks {
659 iForkWriter, err = hlFile.InfoForkWriter()
660 if err != nil {
661 return err
662 }
663
664 rForkWriter, err = hlFile.rsrcForkWriter()
665 if err != nil {
666 return err
667 }
668 }
669 if err := receiveFile(rwc, incWriter, rForkWriter, iForkWriter, fileTransfer.bytesSentCounter); err != nil {
670 return err
671 }
672
673 if err := os.Rename(filePath+".incomplete", filePath); err != nil {
674 return err
675 }
676 }
677
678 // Tell client to send next fileWrapper
679 if _, err := rwc.Write([]byte{0, DlFldrActionNextFile}); err != nil {
680 return err
681 }
682 }
683 }
684 rLogger.Info("Folder upload complete")
685 return nil
686 }