node.js - receive by HTTP API a partial payload nodejs express service

Question

We have designed a service in nodejs that permit to receive file via API. We use the concept of multipart data to receive the file. For the begining we start with the content-type text/plain. It works for most test case but we found an issue, sometime the multipart/form-data is not complete, but on the client side, we clearly see that the http payload is good with all informations required. Do you have an idea where have forgot something ?

Really thank you for your help

require('dotenv').config()

var cors = require('cors')
const express = require('express');
const { json } = require('body-parser');
const fs = require('fs');

const multer = require('multer');

const { uuid } = require('./common/index');
const checkComposite = require('./middleware/check-composite');
const healthCheckComposite = require('./middleware/health-check-composit');
const errorHandler = require('./middleware/error-handler');
const { NotFound, BadRequest } = require('./common/error');

const Log = require('./common/log');

const PORT = process.env.PORT;


const bodyParser = require('body-parser')
const Busboy = require('busboy')
const getRawBody = require('raw-body')
const contentType = require('content-type')

const app = express();

app.use(errorHandler);
app.use(cors());


const storage = multer.diskStorage({
  destination: (req, file, cb) => {
      console.log("UPLOADING!!",req.folder);
    const dir = req.folder.Path;
    // check if exists create on if not recursive
    fs.mkdirSync(dir, { recursive: true }); 
    cb(null, dir);
  },
  filename: (req, file, cb) => {
    const filename = encodeURIComponent(file.originalname);

    const fileArr = filename.split(".");
    const fileExtension = fileArr[fileArr.length - 1];
    const finalFilename = `${req.body.file_name || uuid()}.${fileExtension}`;
    cb(null,  finalFilename);
  }
});

const upload = multer({ storage: storage });

app.get('/', function (req, res) {
  return res.send(`filetransfer Project listening in port ${PORT}`);
})



app.post( '/:locationCode/:system/job/:fileType',
  [    
    bodyParser.json(),
    bodyParser.urlencoded({
      extended: true,
    }),

    (req, res, next) => {
      if (
        req.rawBody === undefined &&
        req.method === 'POST' &&
        req.headers['content-type'].startsWith('multipart/form-data')
      ) {
        getRawBody(
          req,
          {
            length: req.headers['content-length'],
            limit: '100mb',
            encoding: contentType.parse(req).parameters.charset,
          },
          function(err, string) {
            if (err) return next(err)
            req.rawBody = string
            next()
          }
        )
      } else {
        next()
      }
    },

    (req, res, next) => {
      if (
        req.method === 'POST' &&
        req.headers['content-type'].startsWith('multipart/form-data')
      ) {
        const busboy = new Busboy({
          headers: req.headers,
        })

        var fileBuffer = Buffer.from('');

        busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
          file.on('data', data => {
            fileBuffer = Buffer.concat([fileBuffer, data])          
          })

          file.on('end', () => {
            const file_object = {
              fieldname,
              originalname: filename,
              encoding,
              mimetype,
              buffer: fileBuffer,
            }

            req.file = file_object
          })
          
          file.on('error',(err)=>{
            console.log('fstream error ' + err);
            file.unpipe();
          })

        })
        
        req.data = {}

        busboy.on('field', function(
          fieldname,
          val
         
        ) {
          req.data[fieldname] = val
        })

        busboy.on('finish', function() {
            
            
            console.log('Done parsing form!')
            
            
            next()
        })

        busboy.end(req.rawBody)
        
        
        busboy.on('error',function(err){

            let logpath = "D:\filetransfer\logging";          
            console.log(req);
            
            let logto;
            let fsPromises = require('fs').promises
    
            
            logto = `${logpath}\${uuid()}_headers.log`;
            if(req.headers) fsPromises.writeFile(logto, JSON.stringify(req.headers));

            
            logto = `${logpath}\${uuid()}_rawHeaders.log`;
            if(req.rawHeaders) fsPromises.writeFile(logto, req.rawHeaders);

            logto =`${logpath}\${uuid()}_buffer.log`;          
            if(req.file && req.file.buffer) fsPromises.writeFile(logto, req.file.buffer);
            
            logto = `${logpath}\${uuid()}_rawBody.log`;
            if(req.rawBody) fsPromises.writeFile(logto, req.rawBody);
            var errStr =JSON.stringify(err); 
            console.log(0,err);
            console.log('error', errStr);
            
            next(err);

            
        });


      } else {
        next()
      }
    },
    
  ],

  checkComposite,
  function(req, res, next) {
    // request handler
    console.log("next", next);
    if(!req.file) throw new BadRequest('No File Found!');
    
    const file = req.file;
    const buffer = req.file.buffer;

    const path = req.folder.Path;
    var fileName = req.data.filename ? req.data.filename : req.file.originalname;   
    
    var saveTo = `${path}\${uuid()}_${fileName}`;
    
    const fsPromises = require('fs').promises
    fsPromises.writeFile(saveTo, buffer);
    
    console.log(req.file);
    let logpath =`D:\filetransfer\logging\${uuid()}_success_Headers.txt`;
    fsPromises.writeFile(logpath, JSON.stringify(req.headers));
            
    logpath = `D:\filetransfer\logging\${uuid()}_success_Body.txt`;
    if(req.rawBody) fsPromises.writeFile(logpath, req.rawBody);

    Log.success(req.file);
    res.send(`file published!`);
  }
);


app.get('/:locationCode/:system/healthcheck', healthCheckComposite, (req, res) => {
 
  Log.success(req.folders)
  res.send({ folders: req.folders });
});

app.all('*', () => {
  throw new NotFound('Route not Found');
});


module.exports = {
  app
}

question from:https://stackoverflow.com/questions/66059590/receive-by-http-api-a-partial-payload-nodejs-express-service

Answer 1

Waitting for answers