Java 实现大文件统计字母出现个数

简述:

在大文件中用多线程实现查找里面次数出现最多的字母

文件格式如,

Z
Q
S
D
N
O
E
U
...


所有的类,包图



0.Constant.java

常量类

package com.anialy.test.io;

public class Constants {
	public static String[] words = new String[]{
		"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K","L",
		"M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z"};
	
	// 随机生成的数据文件
	public static String OUTPUT_FILE_NAME = "output.txt";
	
	// 统计结果文件
	public static String RESULTS_FILE = "results.txt";
}



1. DataProductor.java

数据生成代码,可以跑一段时间然后看输出的文件

运行main函数即可

package com.anialy.test.io;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.Random;

public class DataProductor {

	public static void main(String[] args) {
		// 随机生成数据
		new DataProductor().produceData();
	}

	private FileOutputStream outputFileStream = null;
	private static final File outputFile = new File(Constants.OUTPUT_FILE_NAME);

	public void produceData(){
		DataProductor productor = new DataProductor();
		PrepareOutput prepareOutput = productor.new PrepareOutput();
		Thread thread = new Thread(prepareOutput);
		thread.start();
	}


	/**
	 * 子线程在文件中插入英文字符
	 */
	private class PrepareOutput implements Runnable{
		String output = ""	;

		private String genWord(){
			int index = new Random().nextInt(Constants.words.length);
			return Constants.words[index];
		}

		public void run(){
			while(true){
				for(int i=0; i<10000;i++){
					output += genWord() + "\n";
				}
				try{
					byte[] outputBytes = output.getBytes("UTF-8");
					//append text at the end, NO covering the previous file
					outputFileStream = new FileOutputStream(outputFile,true);
					outputFileStream.write(outputBytes);
				} catch (FileNotFoundException e) {
					e.printStackTrace();
				} catch (UnsupportedEncodingException e) {
					e.printStackTrace();
				} catch (IOException e) {
					e.printStackTrace();
				}finally{
					//close file stream
					try {
						outputFileStream.close();
					} catch (IOException e) {
						e.printStackTrace();
					}
				}
			}
		}
	}
}


生成的数据文件



示例如下,左侧为notepad中的行号,右侧是字符每行一个换行符间隔




3. CalcDemo

分成多个子线程统计每个英文字符出现的次数

package com.anialy.test.io;

import java.io.File;

public class CalcDemo {
	// the number of calc threads number
	public static final int CALC_THREADS_NUM = 8;
	// the src file 
	private static final File file = new File(Constants.OUTPUT_FILE_NAME);
	// total length
	private static final Long totalBytes = file.length();
	// bytes per thread
	private static final Long bytesPerThread = totalBytes / CALC_THREADS_NUM;
	// bytes left
	private static final Long bytesLeft = totalBytes % bytesPerThread;
	
	private static void initInfo(){
		System.out.printf("file size: %d bytes\n", totalBytes);
		System.out.printf("per thread: %d bytes\n", bytesPerThread);
		System.out.printf("bytes left: %d bytes\n", bytesLeft);
	}
	
	public static void doCalc() {
		initInfo();
		// calc thread start
		for(int threadId=0; threadId<CALC_THREADS_NUM; threadId++){
			Long start = threadId * bytesPerThread;
			Long end = start + bytesPerThread;
			new CalcThread(file, start, end, threadId).start();
		}
		
		// for the bytes left
		CalcThread calcThread = new CalcThread(file
				, bytesPerThread*CALC_THREADS_NUM, totalBytes, CALC_THREADS_NUM);
		calcThread.start();
		
	}
}


生成结果文件,



4.CalcThread.java

3中使用的统计文件的线程类

package com.anialy.test.io;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;

public class CalcThread extends Thread{
	// Thread NO.
	private int NO = 0; 
	
	private RandomAccessFile raf;
	
	private Long start; // access start index;
	
	private Long end; // access end index;
	
	private Map<String, Integer> map = new TreeMap<String, Integer>();
	
	public CalcThread(File file, Long start, Long end, int NO) {
		try {
			raf = new RandomAccessFile(file, "rw");
			this.start = start;
			this.end = end;
			this.NO = NO;
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		}
		
		int letterSize = Constants.words.length;
		for(int i=0; i<letterSize; i++){
			map.put(Constants.words[i], 0);
		}
	}

	@Override
	public void run() {
		System.out.printf("Thread %d is now processing !!\n", NO);
		try {
			raf.seek(start);
			byte[] buff = new byte[1]; // 带着换行符一起统计,每行两个字符
			while(raf.read(buff) != -1){
				String letter = new String(buff, "UTF-8").substring(0);
				if(!"\n".equals(letter)){
					Integer cnt = map.get(letter);
					map.put(letter, cnt + 1);
				}
				start++;
				if(start.equals(end)){
					break;
				}
			}  
			
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			try {
				raf.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
			outputResult();
		}
	}
	
	// 将结果输出到每个线程各自的文件
	private void outputResult() {
		Iterator<String> iter = map.keySet().iterator();
		FileOutputStream outputFileStream = null;
		try {
			// 将统计的结果导入文件result-{NO}
			File outputFile = new File("result-" + NO);
			StringBuffer sbf = new StringBuffer();
			while(iter.hasNext()){
				String key = iter.next();
				sbf.append(key + ":" + map.get(key) + "\n");
			}
			outputFileStream = new FileOutputStream(outputFile,true);
			outputFileStream.write(sbf.toString().getBytes("UTF-8"));
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally{
			if(outputFileStream != null){
				try {
					outputFileStream.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}

	
	// main 测试
	public static void main(String[] args) {
		File file = new File(Constants.OUTPUT_FILE_NAME);
		new CalcThread(file, 0L, file.length(), 1).start();
	}
}

5. Conclusion.java

对分割统计所得的小文件,最后进行统计

package com.anialy.test.io;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;

public class Conclusion {
	
	static Map<String, Integer> resultMap = new TreeMap<String, Integer>();
	static File file = new File("results.txt");
	
	/**
	 * 分别统计每个子结果文件
	 */
	@SuppressWarnings("resource")
	public static void sumUp() {
		// include the thread to deal with byte-left
		for(int i=0; i<CalcDemo.CALC_THREADS_NUM+1; i++){
			File file = new File("result-" + i);
			FileInputStream fio = null;
			try {
				fio = new FileInputStream(file);
				InputStreamReader inputStreamReader = new InputStreamReader(fio);  
		        BufferedReader bufferedReader = new BufferedReader(inputStreamReader); 
		        String line;
		        while((line = bufferedReader.readLine()) != null){
		        	String letter = line.substring(0,1);
		        	String cntStr = line.substring(2);
		        	if(resultMap.get(letter) == null){
		        		resultMap.put(letter, 0);
		        	}
		        	resultMap.put(letter, resultMap.get(letter) + Integer.parseInt(cntStr));
		        }
			} catch (FileNotFoundException e) {
				e.printStackTrace();
			} catch (IOException e) {
				e.printStackTrace();
			}
			showResults();
		}
	}
	
	
	/**
	 * 输出所有结果
	 */
	private static void showResults(){
		Iterator<String> iter = resultMap.keySet().iterator();
		StringBuffer sbf = new StringBuffer();
		while(iter.hasNext()){
			String letter = iter.next();
			Integer cnt = resultMap.get(letter);
			String line = letter + ": " + cnt + "\n";
			sbf.append(line);
		}
		
		FileOutputStream fos;
		try {
			fos = new FileOutputStream(file);
			fos.write(sbf.toString().getBytes("UTF-8"));
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	public static void main(String[] args) {
		sumUp();
	}
}


得到,



结果如下

A: 47177
B: 45772
C: 46418
D: 46785
E: 47498
F: 46146
G: 45693
H: 46705
I: 47734
J: 46402
K: 46519
L: 46679
M: 46958
N: 46577
O: 47473
P: 45797
Q: 46858
R: 46850
S: 47055
T: 46028
U: 45530
V: 45026
W: 46445
X: 46960
Y: 46718
Z: 46197

6. MainTest.java

主测试文件(20s sleep如果output.txt数据文件更大则需要更长时间)

package com.anialy.test.io;

import java.io.File;

/**
 * Package: com.anialy.test.io
 *
 * File: MainTest.java 
 *
 * Author: anialy   Date: 2014-9-3
 * 
 */
public class MainTest {
	public static void main(String[] args) {
		// clear data
		File file = null;
		for(int i=0; i<=CalcDemo.CALC_THREADS_NUM; i++){
			// delete old data file
			file = new File("result-" + i);
			if(file.isFile())
				file.delete();
		}

		file = new File(Constants.RESULTS_FILE);
		if(file.isFile())
			file.delete();


		// CALC_THREADS_NUM+1(for the bytes-left) Threads to analyze data
		CalcDemo.doCalc();

		// 20 seconds time waiting for all threads' process 
		try {
			Thread.sleep(20000);
		} catch (InterruptedException e) {
			e.printStackTrace();
		}

		// main thread to sum up data
		Conclusion.sumUp();
	}
}














相关推荐
©️2020 CSDN 皮肤主题: 大白 设计师:CSDN官方博客 返回首页