refactor file upload

* addReadableStream instead of add
* handle multiple files
* ipfs functions splitup

nginx/ipfs.kretschmann.io.conf

@@ -13,7 +13,7 @@ server {
         proxy_cache_bypass $http_upgrade;
     }
 
-    location ~ "^/api/v0/(add|version|id)" {
+    location ~ "^/api/v0/(add|version|id|ls)" {
         proxy_pass http://localhost:5001;
         proxy_set_header Host $host;
         proxy_cache_bypass $http_upgrade;

package.json

@@ -14,23 +14,23 @@
   "dependencies": {
     "@zeit/next-css": "^1.0.1",
     "axios": "^0.19.0",
-    "ipfs-http-client": "^39.0.0",
+    "ipfs-http-client": "^39.0.2",
     "next": "9.1.1",
-    "next-seo": "^2.1.2",
+    "next-seo": "^2.2.1",
     "next-svgr": "0.0.2",
-    "react": "^16.10.2",
+    "react": "^16.11.0",
-    "react-dom": "^16.10.2",
+    "react-dom": "^16.11.0",
     "react-dropzone": "^10.1.10",
     "use-dark-mode": "^2.3.1"
   },
   "devDependencies": {
     "@types/next-seo": "^1.10.0",
-    "@types/node": "^12.11.1",
+    "@types/node": "^12.11.7",
-    "@types/react": "^16.9.9",
+    "@types/react": "^16.9.11",
-    "@typescript-eslint/eslint-plugin": "^2.4.0",
+    "@typescript-eslint/eslint-plugin": "^2.5.0",
-    "@typescript-eslint/parser": "^2.4.0",
+    "@typescript-eslint/parser": "^2.5.0",
     "cssnano": "^4.1.10",
-    "eslint": "^6.5.1",
+    "eslint": "^6.6.0",
     "eslint-config-prettier": "^6.4.0",
     "eslint-plugin-jsx-a11y": "^6.2.3",
     "eslint-plugin-prettier": "^3.1.1",

src/components/Add.tsx

@@ -1,11 +1,11 @@
-import React, { useState, useEffect } from 'react'
+import React, { useState } from 'react'
 import { ipfsNodeUri, ipfsGateway } from '../../site.config'
-import Dropzone from './Dropzone'
+import Dropzone, { FileDropzone } from './Dropzone'
 import styles from './Add.module.css'
 import Loader from './Loader'
 import useIpfsApi from '../hooks/use-ipfs-api'
 import { IpfsConfig } from '../@types/ipfs'
-import { formatBytes, addToIpfs } from '../utils'
+import { addToIpfs, FileIpfs } from '../ipfs'
 
 const { hostname, port, protocol } = new URL(ipfsNodeUri)
 
@@ -17,33 +17,23 @@ const ipfsConfig: IpfsConfig = {
 
 export default function Add() {
   const { ipfs, isIpfsReady, ipfsError } = useIpfsApi(ipfsConfig)
-  const [fileHash, setFileHash] = useState()
+  const [files, setFiles] = useState()
   const [loading, setLoading] = useState(false)
-  const [message, setMessage] = useState()
+  const [message] = useState()
   const [error, setError] = useState()
-  const [fileSize, setFileSize] = useState()
-  const [fileSizeReceived, setFileSizeReceived] = useState('')
 
-  useEffect(() => {
+  async function handleOnDrop(acceptedFiles: FileDropzone[]) {
-    setMessage(
+    if (!acceptedFiles) return
-      `Adding to IPFS<br />
-       <small>${fileSizeReceived || 0}/${fileSize}</small><br />`
-    )
-  }, [fileSize, fileSizeReceived])
-
-  async function handleOnDrop(acceptedFiles: File[]) {
-    if (!acceptedFiles[0]) return
 
     setLoading(true)
     setError(null)
 
-    const totalSize = formatBytes(acceptedFiles[0].size, 0)
-    setFileSize(totalSize)
-
     try {
-      const cid = await addToIpfs(acceptedFiles, setFileSizeReceived, ipfs)
+      const directoryCid = await addToIpfs(ipfs, acceptedFiles)
-      if (!cid) return
+      if (!directoryCid) return
-      setFileHash(cid)
+
+      const fileList = await ipfs.ls(directoryCid)
+      setFiles(fileList)
       setLoading(false)
     } catch (error) {
       setError(`Adding to IPFS failed: ${error.message}`)
@@ -55,17 +45,22 @@ export default function Add() {
     <div className={styles.add}>
       {loading ? (
         <Loader message={message} />
-      ) : fileHash ? (
+      ) : files ? (
-        <a
+        <ul style={{ textAlign: 'left' }}>
-          target="_blank"
+          {files.map((file: FileIpfs) => (
-          rel="noopener noreferrer"
+            <li key={file.path}>
-          href={`${ipfsGateway}/ipfs/${fileHash}`}
+              <a
-        >
+                target="_blank"
-          ipfs://{fileHash}
+                rel="noopener noreferrer"
-        </a>
+                href={`${ipfsGateway}/ipfs/${file.path}`}
+              >
+                ipfs://{file.path}
+              </a>
+            </li>
+          ))}
+        </ul>
       ) : (
         <Dropzone
-          multiple={false}
           handleOnDrop={handleOnDrop}
           disabled={!isIpfsReady}
           error={error || ipfsError}

src/components/Dropzone.tsx

@@ -2,13 +2,17 @@ import React, { useCallback } from 'react'
 import { useDropzone } from 'react-dropzone'
 import styles from './Dropzone.module.css'
 
+export interface FileDropzone extends File {
+  path: string
+}
+
 export default function Dropzone({
   handleOnDrop,
   disabled,
   multiple,
   error
 }: {
-  handleOnDrop(files: File[]): void
+  handleOnDrop(files: FileDropzone[]): void
   disabled?: boolean
   multiple?: boolean
   error?: string
@@ -37,12 +41,12 @@ export default function Dropzone({
       <input {...getInputProps({ multiple })} />
       {isDragActive && !isDragReject ? (
         `Drop it like it's hot!`
-      ) : multiple ? (
+      ) : multiple === false ? (
-        `Drag 'n' drop some files here, or click to select files.`
+        `Drag 'n' drop a file here, or click to select a file.`
       ) : error ? (
         <div className={styles.error}>{error}</div>
       ) : (
-        `Drag 'n' drop a file here, or click to select a file.`
+        `Drag 'n' drop some files here, or click to select files.`
       )}
     </div>
   )

src/ipfs.ts

@@ -0,0 +1,41 @@
+import { FileDropzone } from './components/Dropzone'
+
+export interface FileIpfsAdd {
+  path: string
+  content: File | ReadableStream | Buffer | string
+}
+
+export interface FileIpfs {
+  path: string
+  hash: string
+  size: number
+}
+
+export function streamFiles(ipfs: any, ipfsFiles: FileIpfsAdd[]) {
+  return new Promise((resolve, reject) => {
+    const stream = ipfs.addReadableStream({
+      wrapWithDirectory: true
+      // progress: (length: number) => setFileSizeReceived(formatBytes(length, 0))
+    })
+
+    stream.on('data', (data: FileIpfs) => {
+      console.log(`Added ${data.path} hash: ${data.hash}`)
+      // The last data event will contain the directory hash
+      if (data.path === '') resolve(data.hash)
+    })
+
+    stream.on('error', reject)
+    ipfsFiles.forEach((file: FileIpfsAdd) => stream.write(file))
+    stream.end()
+  })
+}
+
+export async function addToIpfs(ipfs: any, files: FileDropzone[]) {
+  const ipfsFiles = [
+    ...files.map((file: FileDropzone) => {
+      return { path: file.path, content: file }
+    })
+  ]
+  const directoryCid = await streamFiles(ipfs, ipfsFiles)
+  return directoryCid
+}

src/utils.ts

@@ -9,24 +9,6 @@ export function formatBytes(a: number, b: number) {
   return parseFloat((a / Math.pow(c, f)).toFixed(d)) + ' ' + e[f]
 }
 
-export async function addToIpfs(
-  files: File[],
-  setFileSizeReceived: (size: string) => void,
-  ipfs: any
-) {
-  const file = [...files][0]
-  const fileDetails = { path: file.name, content: file }
-
-  const response = await ipfs.add(fileDetails, {
-    wrapWithDirectory: true,
-    progress: (length: number) => setFileSizeReceived(formatBytes(length, 0))
-  })
-
-  // CID of wrapping directory is returned last
-  const cid = `${response[response.length - 1].hash}/${file.name}`
-  return cid
-}
-
 export async function pingUrl(url: string) {
   try {
     const response = await axios(url, { timeout: 5000 })