как убрать программу из панели задач Java Swing

Можно использовать JDialog вместо JFrame.

import java.awt.Dimension;
import java.awt.Window;
import java.awt.BorderLayout;
import java.awt.event.WindowAdapter;
import java.awt.event.WindowEvent;
import javax.swing.*;
public class Main {
    public static void main(String[] args) {
        JPanel panel = new JPanel(new BorderLayout(5, 5));
        panel.add(Box.createRigidArea(new Dimension(400, 300)));
        JDialog window = new JDialog();
        window.setTitle("Dialog");
        window.setDefaultCloseOperation(JDialog.DISPOSE_ON_CLOSE);
        window.addWindowListener(new WindowAdapter() { 
            @Override
            public void windowClosed(WindowEvent e) { 
                System.exit(0);
            }
        });
        window.setAlwaysOnTop(true);
        window.add(panel);
        window.pack();
        window.setLocationRelativeTo(null);
        window.setVisible(true);
    }
}