File upload and download processing

File Upload Handling

In Express, handling file uploads typically relies on the multer middleware. This middleware is specifically designed to process multipart/form-data type data, which is the standard format for file uploads. Installation method:

npm install multer

Basic configuration example:

const express = require('express');
const multer = require('multer');
const path = require('path');

const app = express();

// Disk storage configuration
const storage = multer.diskStorage({
  destination: (req, file, cb) => {
    cb(null, 'uploads/')
  },
  filename: (req, file, cb) => {
    const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9)
    cb(null, file.fieldname + '-' + uniqueSuffix + path.extname(file.originalname))
  }
})

const upload = multer({ storage: storage })

Single file upload route handling:

app.post('/upload', upload.single('avatar'), (req, res) => {
  console.log(req.file) // Uploaded file information
  res.send('File uploaded successfully')
})

Multiple file upload handling:

app.post('/upload-multiple', upload.array('photos', 12), (req, res) => {
  console.log(req.files) // File array
  res.send(`${req.files.length} files uploaded successfully`)
})

File filtering and validation:

const fileFilter = (req, file, cb) => {
  const allowedTypes = ['image/jpeg', 'image/png']
  if (!allowedTypes.includes(file.mimetype)) {
    const error = new Error('Unsupported file type')
    error.code = 'LIMIT_FILE_TYPE'
    return cb(error, false)
  }
  cb(null, true)
}

const upload = multer({
  storage: storage,
  fileFilter: fileFilter,
  limits: { fileSize: 1024 * 1024 * 5 } // 5MB limit
})

Error handling middleware:

app.use((err, req, res, next) => {
  if (err.code === 'LIMIT_FILE_SIZE') {
    res.status(413).json({ error: 'File size exceeds limit' })
    return
  }
  if (err.code === 'LIMIT_FILE_TYPE') {
    res.status(415).json({ error: 'Unsupported file type' })
    return
  }
  next(err)
})

File Download Handling

Express implements file downloads primarily through the res.download() method, which automatically handles file streams and appropriate HTTP headers.

Basic download example:

app.get('/download/:filename', (req, res) => {
  const file = `./downloads/${req.params.filename}`
  res.download(file) // Automatically sets Content-Disposition header
})

Download with custom filename:

app.get('/download-custom-name', (req, res) => {
  const file = './reports/monthly.pdf'
  res.download(file, '2023-08-report.pdf') // Client will see the custom filename
})

Handling non-existent files:

const fs = require('fs')

app.get('/safe-download/:filename', (req, res) => {
  const filePath = path.join(__dirname, 'downloads', req.params.filename)
  
  fs.access(filePath, fs.constants.F_OK, (err) => {
    if (err) {
      res.status(404).send('File does not exist')
      return
    }
    res.download(filePath)
  })
})

Streaming download for large files:

app.get('/stream-download/:filename', (req, res) => {
  const filePath = path.join(__dirname, 'large-files', req.params.filename)
  const stat = fs.statSync(filePath)
  
  res.writeHead(200, {
    'Content-Type': 'application/octet-stream',
    'Content-Length': stat.size,
    'Content-Disposition': `attachment; filename="${req.params.filename}"`
  })

  const readStream = fs.createReadStream(filePath)
  readStream.pipe(res)
})

Enhanced File Management Features

Implementing a file list API:

app.get('/api/files', (req, res) => {
  fs.readdir('./uploads', (err, files) => {
    if (err) {
      res.status(500).json({ error: 'Unable to read directory' })
      return
    }
    res.json(files.filter(file => !file.startsWith('.')))
  })
})

Paginated file list:

app.get('/api/files/paginated', (req, res) => {
  const page = parseInt(req.query.page) || 1
  const limit = parseInt(req.query.limit) || 10
  const offset = (page - 1) * limit

  fs.readdir('./uploads', (err, files) => {
    if (err) return res.status(500).send(err)
    
    const filteredFiles = files
      .filter(file => !file.startsWith('.'))
      .slice(offset, offset + limit)
    
    res.json({
      total: files.length,
      page,
      limit,
      files: filteredFiles
    })
  })
})

File deletion interface:

app.delete('/api/files/:filename', (req, res) => {
  const filePath = path.join(__dirname, 'uploads', req.params.filename)
  
  fs.unlink(filePath, (err) => {
    if (err) {
      if (err.code === 'ENOENT') {
        return res.status(404).json({ error: 'File does not exist' })
      }
      return res.status(500).json({ error: 'Deletion failed' })
    }
    res.json({ success: true })
  })
})

Security Measures

File type whitelist validation:

const allowedExtensions = ['.jpg', '.jpeg', '.png', '.pdf']

function validateExtension(filename) {
  const ext = path.extname(filename).toLowerCase()
  return allowedExtensions.includes(ext)
}

app.post('/secure-upload', upload.single('file'), (req, res) => {
  if (!req.file) return res.status(400).send('No file uploaded')
  
  if (!validateExtension(req.file.originalname)) {
    fs.unlinkSync(req.file.path) // Delete uploaded invalid file
    return res.status(415).send('Unsupported file type')
  }
  
  res.send('File uploaded securely')
})

Virus scanning integration:

const { exec } = require('child_process')

function scanFile(filePath) {
  return new Promise((resolve, reject) => {
    exec(`clamscan ${filePath}`, (error, stdout, stderr) => {
      if (error && error.code === 1) {
        resolve({ infected: true, output: stdout })
      } else if (error) {
        reject(error)
      } else {
        resolve({ infected: false, output: stdout })
      }
    })
  })
}

app.post('/scan-upload', upload.single('file'), async (req, res) => {
  try {
    const result = await scanFile(req.file.path)
    if (result.infected) {
      fs.unlinkSync(req.file.path)
      return res.status(422).send('File contains virus')
    }
    res.send('File scan passed')
  } catch (err) {
    res.status(500).send('Scan service error')
  }
})

Performance Optimization Techniques

Upload progress feedback:

const progress = require('progress-stream')

app.post('/upload-with-progress', (req, res) => {
  const progressStream = progress({
    length: req.headers['content-length'],
    time: 100 // milliseconds
  })

  progressStream.on('progress', (state) => {
    console.log(`Progress: ${Math.round(state.percentage)}%`)
    // Can push real-time updates to client via WebSocket
  })

  const upload = multer({ storage: storage }).single('file')
  req.pipe(progressStream).pipe(req)
  
  upload(req, res, (err) => {
    if (err) return res.status(500).send(err.message)
    res.send('Upload completed')
  })
})

Chunked upload handling:

const chunkUploadDir = './chunks'

app.post('/upload-chunk', upload.single('chunk'), (req, res) => {
  const { chunkNumber, totalChunks, fileId } = req.body
  const chunkDir = path.join(chunkUploadDir, fileId)
  
  if (!fs.existsSync(chunkDir)) {
    fs.mkdirSync(chunkDir, { recursive: true })
  }
  
  const chunkPath = path.join(chunkDir, `chunk-${chunkNumber}`)
  fs.renameSync(req.file.path, chunkPath)
  
  const chunks = fs.readdirSync(chunkDir)
  if (chunks.length === parseInt(totalChunks)) {
    // All chunks uploaded, trigger merge
    res.json({ status: 'complete' })
  } else {
    res.json({ status: 'partial', received: chunks.length })
  }
})

Frontend Integration

React file upload component example:

function FileUpload() {
  const [progress, setProgress] = useState(0)
  
  const handleUpload = async (e) => {
    const file = e.target.files[0]
    const formData = new FormData()
    formData.append('file', file)
    
    const config = {
      onUploadProgress: progressEvent => {
        const percent = Math.round(
          (progressEvent.loaded * 100) / progressEvent.total
        )
        setProgress(percent)
      }
    }
    
    try {
      await axios.post('/upload', formData, config)
      alert('Upload successful')
    } catch (err) {
      console.error(err)
    }
  }
  
  return (
    <div>
      <input type="file" onChange={handleUpload} />
      {progress > 0 && <progress value={progress} max="100" />}
    </div>
  )
}

HTML implementation with drag-and-drop:

<div id="drop-area">
  <form class="upload-form">
    <input type="file" id="fileElem" multiple accept="image/*" />
    <label for="fileElem">Select files</label>
    <p>or drag and drop files here</p>
  </form>
  <div id="preview"></div>
</div>

<script>
  const dropArea = document.getElementById('drop-area')
  
  ;['dragenter', 'dragover', 'dragleave', 'drop'].forEach(eventName => {
    dropArea.addEventListener(eventName, preventDefaults, false)
  })
  
  function preventDefaults(e) {
    e.preventDefault()
    e.stopPropagation()
  }
  
  dropArea.addEventListener('drop', handleDrop, false)
  
  function handleDrop(e) {
    const dt = e.dataTransfer
    const files = dt.files
    handleFiles(files)
  }
  
  function handleFiles(files) {
    const formData = new FormData()
    Array.from(files).forEach(file => {
      formData.append('files', file)
    })
    
    fetch('/upload-multiple', {
      method: 'POST',
      body: formData
    })
    .then(response => response.json())
    .then(data => console.log(data))
    .catch(error => console.error(error))
  }
</script>

Cloud Storage Integration

AWS S3 upload example:

const AWS = require('aws-sdk')
const s3 = new AWS.S3({
  accessKeyId: process.env.AWS_ACCESS_KEY,
  secretAccessKey: process.env.AWS_SECRET_KEY
})

app.post('/upload-to-s3', upload.single('file'), (req, res) => {
  const fileContent = fs.readFileSync(req.file.path)
  
  const params = {
    Bucket: process.env.S3_BUCKET,
    Key: `uploads/${req.file.filename}`,
    Body: fileContent
  }
  
  s3.upload(params, (err, data) => {
    if (err) {
      return res.status(500).send(err)
    }
    
    // Delete local temp file after successful upload
    fs.unlinkSync(req.file.path)
    
    res.json({
      location: data.Location,
      key: data.Key
    })
  })
})

Qiniu Cloud Storage integration:

const qiniu = require('qiniu')

const mac = new qiniu.auth.digest.Mac(
  process.env.QINIU_ACCESS_KEY,
  process.env.QINIU_SECRET_KEY
)

const config = new qiniu.conf.Config()
const formUploader = new qiniu.form_up.FormUploader(config)

app.post('/upload-to-qiniu', upload.single('file'), (req, res) => {
  const putPolicy = new qiniu.rs.PutPolicy({
    scope: process.env.QINIU_BUCKET
  })
  
  const uploadToken = putPolicy.uploadToken(mac)
  const localFile = req.file.path
  
  const putExtra = new qiniu.form_up.PutExtra()
  const key = `uploads/${req.file.filename}`
  
  formUploader.putFile(uploadToken, key, localFile, putExtra, (err, body, info) => {
    fs.unlinkSync(localFile) // Clean up temp file
    
    if (err) {
      return res.status(500).json(err)
    }
    
    if (info.statusCode === 200) {
      const url = `http://${process.env.QINIU_DOMAIN}/${body.key}`
      res.json({ url })
    } else {
      res.status(info.statusCode).json(body)
    }
  })
})