日志提取分析工具(java源码)

最近有个项目,是硬件结合的,硬件上传到服务器的日志,每天数百万条,有时候某个设备出问题了,因为日志的数据很混乱,很难查出具体的原因。

所以写了这个工具,主要是提高日志分析的效率,可以通过关键词提取日志数据。

工具使用了多线程、I/O等技术,本人技术有限,所以只能写到这样子,测试过很多次。

测试出来的数据:400MB的日志,5个线程:96~97秒完成分割,分割出来的日志大小大同小异,为什么不把分割出来的日志合并呢?因为线程的启动时间不是顺序的,加上本人懒,所以没做了。

不建议使用超过20个线程去处理日志。因为如果是2GB的数据,10个线程去处理,每个线程也只需要处理204.8MB。这个已经是非常快的效率了。

 

原理:

a000

java源码:

package test;

import io.netty.util.concurrent.DefaultThreadFactory;

import javax.swing.*;
import javax.swing.filechooser.FileFilter;
import java.awt.*;
import java.awt.event.*;
import java.io.*;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.LinkedList;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

public class WebLogParser {

    public static LinkedBlockingQueue<String> logQueue = new LinkedBlockingQueue<>();//加入队列写出日志

    public static ThreadPoolExecutor executor;
    public static JTextField filePath, threadNum, keywords;
    public static JTextArea logs;
    public static JButton start, stop, choose;
    public static JLabel text;
    public static WorkerListener listener;
    public static Timer timer;

    public static boolean isStop = false;
    public static double processLen = 0.0;
    public static double splitLenSum = 0;
    public static double fileLen = 0.0;

    public static File file;
    public static DefaultThreadFactory factory;

    private static String keywordsTip = "you can use \";\" split this string.";

    public static void main(String[] args) {
        // TODO Auto-generated method stub
        JFrame windows = createWindows("日志分析", windowListener());
        createPanel(windows);

        ActionListener actionListener = createClickListener();
        start.addActionListener(actionListener);
        stop.addActionListener(actionListener);
        choose.addActionListener(actionListener);

        filePath.setText("C:\\Users\\Administrator\\Desktop\\1.log");
//        keywords.setText("867186033556969");
        threadNum.setText("1");

        windows.setBounds(100, 100, 700, 480);
        windows.setVisible(true);

        StringBuffer sb = new StringBuffer();
        timer = new Timer();
        timer.schedule(new TimerTask() {
            @Override
            public void run() {
                StringBuffer sb = new StringBuffer();
                if (executor != null) {
                    sb.append("核心线程:" + executor.getCorePoolSize() + "     ")
                            .append("激活线程:" + executor.getActiveCount() + "     ");
                }

                if (processLen > 0 || splitLenSum > 0) {
                    if (processLen > 0) {
                        text.setText("搜索进度:" + Math.ceil(processLen / fileLen * 100) + "% - " + nowDateTime() + " - " + sb.toString());
                        if (Math.ceil(processLen / fileLen * 100) == 100) {
                            initView(true);
                        }
                    } else {
                        text.setText("分割进度:" + formatNumber(splitLenSum / fileLen * 100, null) + "% - " + nowDateTime() + " - " + sb.toString());
                    }
                } else {
                    text.setText(String.format("init success.  - " + sb.toString()));
                }

            }
        }, 1000, 1000);

        initView(true);

    }

    private static void initView(boolean isEnable) {
        out("init view.");
        if (isEnable) {
            isStop = true;
            start.setEnabled(true);
            stop.setEnabled(false);

            threadNum.setEnabled(true);
            keywords.setEnabled(true);
            filePath.setEnabled(true);

        } else {
            isStop = false;
            start.setEnabled(false);
            stop.setEnabled(true);

            threadNum.setEnabled(false);
            keywords.setEnabled(false);
            filePath.setEnabled(false);
        }

        processLen = 0;
        splitLenSum = 0;

    }

    private static void doWork() {
        factory = new DefaultThreadFactory("logs");
        factory.newThread(new Runnable() {
            @Override
            public void run() {
                while (!isStop){
                    if (logQueue.size()>0){
                        try {
                            logs.setText(logQueue.take()+"\n" + logs.getText());
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                        }
                    }
                }
            }
        }).start();



        int tnum = Integer.valueOf(threadNum.getText());
        if (file == null){
            file = new File(filePath.getText());
        }
        out("read file " + file.getName());

        String key = keywords.getText();
        if (key.contains(keywordsTip)) {
            key = null;
        }

        listener = createWorkerListener();

        out("Create Thread...");
        if (executor == null) {
            executor = new ThreadPoolExecutor(20, 20, 30, TimeUnit.SECONDS, new LinkedBlockingQueue<>(100));
        }

        for (int i = 0; i < tnum; i++) {
            executor.execute(new Worker(file, i, key, null, logQueue, listener, tnum));
        }
        out("Create Thread Success...");
    }

    private static WorkerListener createWorkerListener() {

        return new WorkerListener() {
            @Override
            public void process(long f, long process, long pos) {
                processLen = process;
                fileLen = f;
            }

            @Override
            public void splitFile(long fl, long splitLen, long len) {
                splitLenSum = splitLen;
                fileLen = fl;
            }

            @Override
            public boolean stop() {
                return isStop;
            }
        };
    }

    private static ActionListener createClickListener() {
        return new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {
                if (e.getActionCommand().equals("start")) {
                    if ("".equals(filePath.getText()) || filePath.getText() == null) {
                        JOptionPane.showMessageDialog(null, "请选择日志文件路径", "info", JOptionPane.PLAIN_MESSAGE);
                        return;
                    }
                    if ("".equals(threadNum.getText()) || threadNum.getText() == null) {
                        JOptionPane.showMessageDialog(null, "请输入线程数", "info", JOptionPane.PLAIN_MESSAGE);
                        return;
                    }
                    if (Integer.valueOf(threadNum.getText()) <= 0) {
                        JOptionPane.showMessageDialog(null, "线程数必须>0", "info", JOptionPane.PLAIN_MESSAGE);
                        return;
                    }
                    if (Integer.valueOf(threadNum.getText()) > 20) {
                        JOptionPane.showMessageDialog(null, "线程数必须<=20", "info", JOptionPane.PLAIN_MESSAGE);
                        return;
                    }
                    if ("".equals(keywords.getText()) || keywords.getText() == null) {
                        JOptionPane.showMessageDialog(null, "请输入关键词", "info", JOptionPane.PLAIN_MESSAGE);
                        return;
                    }

                    initView(false);
                    doWork();
                } else if (e.getActionCommand().equals("stop")) {
                    initView(true);
                } else if (e.getActionCommand().equals(">>")) {
                    chooseFile();
                } else {
                    out(e.getActionCommand());

                }
            }
        };
    }

    private static void chooseFile() {
        JFileChooser jfc = new JFileChooser();

        jfc.addChoosableFileFilter(new FileFilter() {
            @Override
            public boolean accept(File f) {
                String name = f.getName();
                return f.isDirectory() || name.toLowerCase().endsWith(".log");
            }

            @Override
            public String getDescription() {
                return "*.log";
            }
        });
        jfc.showDialog(new JLabel(), "选择日志文件");
        jfc.setFileSelectionMode(JFileChooser.FILES_ONLY);
        file = jfc.getSelectedFile();

        if (file != null && file.isFile()) {
            filePath.setText(file.getAbsolutePath() + file.getName());
        }


    }

    private static WindowListener windowListener() {
        return new WindowAdapter() {
            @Override
            public void windowOpened(WindowEvent e) {
                super.windowOpened(e);
            }

            @Override
            public void windowClosing(WindowEvent e) {
                isStop = true;
                timer.cancel();

                super.windowClosing(e);
                if (executor != null) {
                    executor.shutdownNow();
                }
            }

            @Override
            public void windowClosed(WindowEvent e) {
                super.windowClosed(e);
                if (executor != null) {
                    executor.shutdown();
                }
            }

            @Override
            public void windowActivated(WindowEvent e) {
                super.windowActivated(e);
            }

            @Override
            public void windowStateChanged(WindowEvent e) {
                super.windowStateChanged(e);
            }
        };
    }

    private static JFrame createWindows(String title, WindowListener listener) {
        JFrame jFrame = new JFrame(title);
        jFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
        jFrame.addWindowListener(listener);
        return jFrame;
    }

    private static JPanel createPanel(JFrame jFrame) {
        JPanel jPanel = new JPanel();
        jPanel.setLayout(null);
        jFrame.add(jPanel);

        /**
         * lable
         */
        JLabel label1 = new JLabel("Log File Absolute Path:", JLabel.RIGHT);
        label1.setBounds(20, 20, 140, 26);

        JLabel label2 = new JLabel("Thread Number:", JLabel.RIGHT);
        label2.setBounds(20, 56, 140, 26);

        JLabel label4 = new JLabel("keywords:", JLabel.RIGHT);
        label4.setBounds(20, 92, 140, 26);

        JLabel label3 = new JLabel("logs:", JLabel.RIGHT);
        label3.setBounds(4, 132, 48, 26);
        label3.setBackground(Color.BLACK);

        text = new JLabel();
        text.setBounds(20, 400, 500, 26);
        text.setText("init success.");
        text.setForeground(Color.gray);

        /**
         * editor
         */
        filePath = new JTextField();
        filePath.setBounds(165, 20, 400, 26);

        choose = new JButton(">>");
        choose.setBounds(565, 20, 26, 25);

        threadNum = new JTextField();
        threadNum.setBounds(165, 56, 80, 26);
        threadNum.addKeyListener(new KeyListener() {
            @Override
            public void keyTyped(KeyEvent e) {
                int temp = e.getKeyChar();
                if (temp == 10) {//按回车时

                } else if (temp != 46) {
                    if (temp != 8) {
                        if (temp > 57) {
                            e.consume();
                        } else if (temp < 48) {
                            e.consume();
                        }
                    }
                }
            }

            @Override
            public void keyPressed(KeyEvent e) {

            }

            @Override
            public void keyReleased(KeyEvent e) {

            }
        });

        keywords = new JTextField();
        keywords.setBounds(165, 92, 495, 26);
        keywords.setText(keywordsTip);
        keywords.setForeground(Color.gray);
        keywords.addFocusListener(new FocusAdapter() {
            @Override
            public void focusGained(FocusEvent e) {
                super.focusGained(e);
                if (keywords.getText().contains(keywordsTip)) {
                    keywords.setText("");
                }
                keywords.setForeground(Color.black);
            }

            @Override
            public void focusLost(FocusEvent e) {
                super.focusLost(e);
                if ("".equals(keywords.getText())) {
                    keywords.setForeground(Color.gray);
                    keywords.setText(keywordsTip);
                }
            }
        });

        logs = new JTextArea(10, 20);
        logs.setBounds(60, 130, 600, 200);
        logs.setLineWrap(true);

        JScrollPane logSp = new JScrollPane(logs);
        logSp.setBounds(60, 130, 600, 200);


        /**
         * button
         */
        start = new JButton("start");
        start.setBounds(580, 360, 65, 26);

        stop = new JButton("stop");
        stop.setBounds(500, 360, 65, 26);

        jPanel.add(label1);
        jPanel.add(label2);
        jPanel.add(label3);
        jPanel.add(label4);
        jPanel.add(text);
        jPanel.add(filePath);
        jPanel.add(choose);
        jPanel.add(threadNum);
        jPanel.add(keywords);
//        jPanel.add(logs);
        jPanel.add(logSp);
        jPanel.add(start);
        jPanel.add(stop);
        return jPanel;
    }

    private static void out(String s) {
        if (s != null) {
            s = nowDateTime() + " " + s;
            System.out.println(s);
            logs.setText(s + "\n" + logs.getText());
        }
    }

    private static String nowDateTime() {
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
        return sdf.format(new Date());
    }

    private static String formatNumber(Double number, String pattern) {
        if (pattern == null) {
            pattern = "#0.00";
        }
        DecimalFormat df = new DecimalFormat(pattern);
        return df.format(number);
    }
}

class Worker implements Runnable {

    private static final boolean debug = true;

    private File file;

    /**
     * index must be start in 0;
     */
    private long index = 0;

    private int threadNum = 1;

    /**
     * you can use ";" split this string.
     */
    private String keywords = null;

    private File targetFile = null;
    private FileOutputStream fos = null;

    private LinkedBlockingQueue<String> logQueue;
    private long stime = 0L;
    long pos = 0L;
    private WorkerListener listener;

    public Worker() {
        // TODO Auto-generated constructor stub
    }

    public Worker(File f) {
        this(f, 1L);
    }

    public Worker(File f, long num) {
        this(f, num, null);
    }

    public Worker(File file, long index, String keywords) {
        this(file, index, keywords, null);
    }

    public Worker(File file, long index, String keywords, File targetFile) {
        this(file, index, keywords, targetFile, null, null);
    }

    public Worker(File file, long index, String keywords, File targetFile, LinkedBlockingQueue<String> logQueue) {
        this(file, index, keywords, targetFile, logQueue, null);
    }

    public Worker(File file, long index, String keywords, File targetFile, LinkedBlockingQueue<String> logQueue, WorkerListener listener) {
        this(file, index, keywords, targetFile, logQueue, listener, 1);
    }

    public Worker(File file, long index, String keywords, File targetFile, LinkedBlockingQueue<String> logQueue, WorkerListener listener, int threadNum) {
        this.file = file;
        this.index = index;
        this.keywords = keywords;
        this.targetFile = targetFile;
        this.logQueue = logQueue;
        this.listener = listener;
        this.threadNum = threadNum;
    }


    @Override
    public void run() {
        // TODO Auto-generated method stub
        out("file exists: "+file.exists());
        if (file.exists()) {
            stime = System.currentTimeMillis();
            out("Thread start....");
            this.checkTargetFile();
            RandomAccessFile raf = null;
            try {
                long len = 0L;
                if (index > 0) {
                    pos = (long) Math.floor(file.length() / (index + 1));
                }

                file = splitFiles(file);

                raf = new RandomAccessFile(file, "r");
                raf.seek(pos);
                String line;
                out("init point is " + pos);
                out("temp file is " + file.getAbsolutePath() + file.getName());
                while ((line = raf.readLine()) != null) {
                    len += line.length();
                    if (listener != null) {
                        if (listener.stop()) {
                            break;
                        }
                        listener.process(file.length(), len, pos);
                    }

                    if (keywords != null) {
                        if (this.checkKeywords(line)) {
                            this.put(line);
                        }
                    }
                }
                out(String.format("search end and file length is %s,search total is %s.", file.length(), len));
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                try {
                    if (raf != null) {
                        raf.close();
                    }
                    if (fos != null) {
                        fos.close();
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                }
                stime = (System.currentTimeMillis() - stime) / 1000;
                out("Thread end,The work use time is " + stime + "s.");

                delSplitFile(file);
            }
        }

    }

    private void delSplitFile(File file) {
        if (threadNum > 1) {
            file.delete();
        }
    }

    private File splitFiles(File file) throws Exception {
        if (threadNum < 1) {
            throw new Exception("Thread number must be >= 1.");
        }
        if (threadNum == 1) {
            return file;
        }
        long len = (file.length() / threadNum);

        File newFile = new File(getFilePath() + getFileName() + Thread.currentThread().getName());
        FileInputStream _in = new FileInputStream(file);
        FileOutputStream _out = new FileOutputStream(newFile);

        out("split file and skip " + pos + ",split length " + len);
        _in.skip(pos);

        byte[] b = new byte[1024];
        int n = 0;
        long l = 0;
        stime = System.currentTimeMillis();
        out("start split file.");
        while ((n = _in.read(b)) != -1) {
            l += n;
            if (listener != null) {
                if (listener.stop()) {
                    break;
                }
                listener.splitFile(len, l, n);
            }
//            out(String.format(">>test>>fileLen:%s sum:%s now:%s",len,l,n));
            if (l > len) {
                break;
            }
            _out.write(b, 0, n);
        }
        out("end split file use time is " + ((System.currentTimeMillis() - stime) / 1000) + "s");
        _in.close();
        _out.close();

        pos = 0;
        return newFile;
    }

    private void put(String data) {
        try {
            if (fos == null) {
                fos = new FileOutputStream(targetFile);
            }
            fos.write(data.trim().getBytes());
            fos.write("\n".getBytes());
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    private boolean checkKeywords(String line) {
        String[] keys = this.keywords.split(";");
        boolean res = false;
        for (String key : keys) {
            if (line.contains(key)) {
//                out(key + " : " + line);
                res = true;
                break;
            }
        }
        return res;
    }

    private void checkTargetFile() {
        if (this.targetFile == null) {
            this.targetFile = new File(getFilePath() + getFileName() + "_" + index + "." + getFileExt());
        }
        out("out put new file is " + this.targetFile.getAbsolutePath());
    }

    private String getFileExt() {
        String s = file.getName();
        return s.substring(s.indexOf(".") + 1);
    }

    private String getFileName() {
        String s = file.getName();
        return s.substring(0, s.indexOf(".") - 1);
    }

    private String getFilePath() {
        return file.getAbsolutePath();
    }

    private void out(String s) {
        if (debug) {
            if (s != null) {
                if (!Thread.interrupted()) {
                    s = nowDateTime() + " - " + Thread.currentThread().getName() + " " + s;
                    System.out.println(s);
                    try {
                        logQueue.put(s);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
    }

    private String nowDateTime() {
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
        return sdf.format(new Date());
    }

}

interface WorkerListener {
    void process(long flen, long process, long pos);

    void splitFile(long fileLen, long splitLen, long len);

    boolean stop();
}

测试:

日志文件是400MB,创建了5个线程,每个线程处理80MB的数据,耗时大约96~97秒之间,如下图:

a001

 

分割出来的日志文件:

a002

下面是System.out.println打印出来的日志:

a003

 

再测试一次,下面是用单线程去处理400MB的数据:

a004

 

测试完毕的日志数据,需要耗时333秒,如果使用线程,要快上3倍多:

a005

 

 

 

Leave a Comment