来自不同线程的顺序文件 IO 因 FileSystemException 而失败

Question

我想看看我能多快地做类似的事情：

写一个小文件
重命名
删除它

这基本上是这样的：

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;


public class QuickIO {

  public static void main(String[] args) throws IOException {
    Path fileToWriteTo = Paths.get("C:\Temp\somefile.txt");
    Path fileToMoveTo = Paths.get("C:\Temp\someotherfile.txt");
    Files.write(fileToWriteTo, "sometext".getBytes());
    Files.move(fileToWriteTo, fileToMoveTo);
    Files.delete(fileToMoveTo);
  }

}

以上绝对没问题。我想对大量文件做同样的事情以获得相关数据；每个操作都应该由不同的线程完成（这类似于我的最终架构）。他们将通过阻塞队列进行通信（因此文件上的每个操作都将在没有其他线程在处理文件时完成）。我在 windows 7 上运行并且失败并显示以下内容：

10:21:39.756 [Thread-1] WARN  FileMover - IOException
java.nio.file.FileSystemException: C:\Temp\file_0.in -> C:\Temp\file_0.in\file_0.out: The process cannot access the file because it is being used by another process.

    at sun.nio.fs.WindowsException.translateToIOException(WindowsException.java:86) ~[na:1.7.0_67]
    at sun.nio.fs.WindowsException.rethrowAsIOException(WindowsException.java:97) ~[na:1.7.0_67]
    at sun.nio.fs.WindowsFileCopy.move(WindowsFileCopy.java:387) ~[na:1.7.0_67]
    at sun.nio.fs.WindowsFileSystemProvider.move(WindowsFileSystemProvider.java:287) ~[na:1.7.0_67]
    at java.nio.file.Files.move(Files.java:1347) ~[na:1.7.0_67]
    at FileMover.run(FileMover.java:40) ~[classes/:na]
    at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]

所以基本上写操作工作正常，将 Path 对象移交给尝试移动文件的 FileMover，但是由于文件是 "used" 另一个进程而失败。没有其他人使用该文件，因为它刚刚创建，并且创建它的进程关闭了文件，如 Files.write() 所确保的那样。

知道为什么文件系统会看到不应该使用的文件仍在使用吗？以及如何解决？ :)

这是完整的代码（使用 "java App C:\Temp 1" 只调用一个文件，但失败了）

App.java

import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;

public class App {

  public static void main(String[] args) {
    if(args.length != 2 ) {
      System.exit(1);
    }
    Path workDir = Paths.get(args[0]);
    int numFilesToWrite = Integer.parseInt(args[1]);

    if(!Files.isDirectory(workDir)) {
      System.exit(1);
    }

    if(numFilesToWrite < 1) {
      System.exit(1);
    }

    BlockingQueue<Path> fromFileWriterToFileMover = new LinkedBlockingQueue<Path>();
    BlockingQueue<Path> fromFileMoverToFileRemover = new LinkedBlockingQueue<Path>();

    Thread writer = new Thread(new FileWriter(workDir, fromFileWriterToFileMover, numFilesToWrite));
    Thread mover = new Thread(new FileMover(fromFileWriterToFileMover, fromFileMoverToFileRemover));
    Thread remover = new Thread(new FileRemover(fromFileMoverToFileRemover));

    remover.start();
    mover.start();
    writer.start();

  }

}

FileWrite.java

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.concurrent.BlockingQueue;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class FileWriter implements Runnable {
  private static final Logger log = LoggerFactory.getLogger(FileWriter.class);

  Path workDirectory;
  BlockingQueue<Path> toFileMover;
  int numFiles;

  public FileWriter(
      Path workDirectory,
      BlockingQueue<Path> toFileMover,
      int numFiles) {
    this.workDirectory = workDirectory;
    this.toFileMover = toFileMover;
    this.numFiles = numFiles;
  }

  public void run() {
    RandomString rs = new RandomString(2345);
    long msSpentWritingFiles = 0L;

    for(int i = 0 ; i < numFiles ; i++) {
      try {
        Path file = workDirectory.resolve(Paths.get("file_" + i + ".in"));

        byte [] bytes = rs.nextString().getBytes();
        long beforeWrite = System.currentTimeMillis();
        Files.write(file, bytes);
        long afterWrite = System.currentTimeMillis();
        msSpentWritingFiles = msSpentWritingFiles + (afterWrite - beforeWrite);

        toFileMover.put(file);
      } catch (IOException e) {
        log.warn("IO Exception", e);
      }
      catch (InterruptedException e) {
        log.warn("InterruptedException", e);
      }
    }

    Path endPath = Paths.get("/THEEND");
    try {
      toFileMover.put(endPath);
    } catch (InterruptedException e) {
      log.warn("InterruptedException", e);
    }

    log.info("Time spent writing files: " + msSpentWritingFiles + "ms");
  }

}

FileMover.java

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.BlockingQueue;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class FileMover implements Runnable {
  private static final Logger log = LoggerFactory.getLogger(FileMover.class);

  BlockingQueue<Path> fromFileWriter;
  BlockingQueue<Path> toFileRemover;

  public FileMover(
      BlockingQueue<Path> fromFileWriter,
      BlockingQueue<Path> toFileRemover) {
    this.fromFileWriter = fromFileWriter;
    this.toFileRemover = toFileRemover;
  }

  @Override
  public void run() {
    boolean carryOn = true;
    long msSpentMovingFiles = 0L;

    while(carryOn) {
      try {
        Path origin = fromFileWriter.take();

        if("THEEND".equals(origin.getFileName().toString())){
          carryOn = false;
          toFileRemover.put(origin);
        } else {

          Path destination = origin.resolve(origin.getFileName().toString().replace(".in", ".out"));
          long beforeMove = System.currentTimeMillis();
          Files.move(origin, destination);
          long afterMove = System.currentTimeMillis();

          msSpentMovingFiles = msSpentMovingFiles + (afterMove - beforeMove);

          toFileRemover.put(destination);
        }

      } catch (InterruptedException e) {
        log.warn("InterruptedException", e);
      } catch (IOException e) {
        log.warn("IOException", e);
      }

    }
    log.info("Time spent moving files: " + msSpentMovingFiles + "ms");
  }
}

FileRemover.java

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.BlockingQueue;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class FileRemover implements Runnable {
private static final Logger log = LoggerFactory.getLogger(FileRemover.class);

  BlockingQueue<Path> fromFileMover;

  public FileRemover(BlockingQueue<Path> fromFileMover) {
    this.fromFileMover = fromFileMover;
  }

  @Override
  public void run() {
    boolean carryOn = true;
    long msSpentDeletingFiles = 0L;

    while(carryOn) {
      try {
        Path fileToDelete = fromFileMover.take();

        if("THEEND".equals(fileToDelete.getFileName().toString())){
          carryOn = false;
        } else {


          long beforeDelete = System.currentTimeMillis();
          Files.delete(fileToDelete);
          long afterDelete = System.currentTimeMillis();

          msSpentDeletingFiles = msSpentDeletingFiles + (afterDelete - beforeDelete);

        }
      } catch (InterruptedException e) {
        log.warn("InterruptedException", e);
      } catch (IOException e) {
        log.warn("IOException", e);
      }

    }
    log.info("Time spent deleting files: " + msSpentDeletingFiles + "ms");
  }

}

RandomString.java

import java.util.Random;

public class RandomString {

  private static final char[] symbols;

  static {
    StringBuilder tmp = new StringBuilder();
    for (char ch = '0'; ch <= '9'; ++ch)
      tmp.append(ch);
    for (char ch = 'a'; ch <= 'z'; ++ch)
      tmp.append(ch);
    symbols = tmp.toString().toCharArray();
  }   

  private final Random random = new Random();

  private final char[] buf;

  public RandomString(int length) {
    if (length < 1)
      throw new IllegalArgumentException("length < 1: " + length);
    buf = new char[length];
  }

  public String nextString() {
    for (int idx = 0; idx < buf.length; ++idx) 
      buf[idx] = symbols[random.nextInt(symbols.length)];
    return new String(buf);
  }
}

pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.company</groupId>
  <artifactId>small-file-writer</artifactId>
  <version>0.0.1-SNAPSHOT</version>

  <properties>
    <java.version>1.7</java.version>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <logback.version>1.1.2</logback.version>
    <slf4j.version>1.7.10</slf4j.version>
  </properties>

  <dependencies>
      <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>jcl-over-slf4j</artifactId>
      <version>${slf4j.version}</version>
    </dependency>
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>jul-to-slf4j</artifactId>
      <version>${slf4j.version}</version>
    </dependency>
    <dependency>
      <groupId>ch.qos.logback</groupId>
      <artifactId>logback-classic</artifactId>
      <version>${logback.version}</version>
    </dependency>
  </dependencies>

  <build>
    <plugins>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-compiler-plugin</artifactId>
        <configuration>
          <source>${java.version}</source>
          <target>${java.version}</target>
          <encoding>UTF-8</encoding>
        </configuration>
      </plugin>
    </plugins>
  </build>
</project>

Answer 1

您只是在 FileMover 运行方法中有一些拼写错误：

Path destination = origin.resolve(origin.getFileName().toString().replace(".in", ".out"));

目标将像 C:\temp\file_0.in\file_0.out 这样是行不通的，因为它是文件而不是目录:-)

将其替换成这样：

String now = origin.toString().replace(".in", ".out");
Path destination = Paths.get(now);

来自不同线程的顺序文件 IO 因 FileSystemException 而失败

Sequential file IO from different threads fail with FileSystemException

java

io

multithreading

windows-7