Preface: Given your “console application” doesn’t involve forking any processes or performing any “actual” I/O, you may want to redesign your application to use the TextField and TextArea directly, or at least more directly than you’re currently trying to do. Using System.out and System.in adds a layer of unnecessary indirection and makes everything more complex. Also, redirecting standard out and standard in affects the whole process, which may not be desirable.

---

General Idea

Reading characters from standard in and writing characters to standard out will involve decoding and encoding those characters. You’ll want to use classes which already implement this for you, such as BufferedReader and BufferedWriter. And since you essentially want to communicate between threads via streams, you can have the underlying streams be instances of PipedInputStream and PipedOutputStream.

Also, since most console-based applications can print prompting text without a new line at the end, you have to “read from” standard out by reading each individual character rather than line-by-line. This could involve a lot of calls to Platform#runLater(Runnable), potentially overwhelming the FX thread, which means you should probably buffer this output.

The example below shows a proof-of-concept for this. It uses UTF-8 for the character encoding. There’s a GIF of the example code running at the end of this answer.

Here’s a brief overview of the classes:

---

Example

Note that I have the TextArea set up to automatically scroll to the end when the text changes. Unfortunately, this doesn’t seem to work the first time text goes “off the screen”. But after that first time it seems to work.

Main.java

import java.io.IOException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import javafx.application.Application;
import javafx.application.Platform;
import javafx.geometry.Insets;
import javafx.scene.Scene;
import javafx.scene.control.TextArea;
import javafx.scene.control.TextField;
import javafx.scene.layout.Priority;
import javafx.scene.layout.VBox;
import javafx.scene.text.Font;
import javafx.stage.Stage;

public class Main extends Application {

    private final ExecutorService threadPool = Executors.newFixedThreadPool(3, r -> {
        var t = new Thread(r);
        t.setDaemon(true);
        return t;
    });

    @Override
    public void start(Stage primaryStage) throws Exception {
        var textField = new TextField();
        textField.setFont(Font.font("Monospaced", 13));

        var textArea = new TextArea();
        textArea.setFont(textField.getFont());
        textArea.setEditable(false);
        textArea.setFocusTraversable(false);
        textArea.textProperty().addListener(obs -> textArea.end());

        var root = new VBox(10, textArea, textField);
        root.setPadding(new Insets(10));
        VBox.setVgrow(textArea, Priority.ALWAYS);

        primaryStage.setScene(new Scene(root, 600, 400));
        textField.requestFocus();

        primaryStage.setTitle("Console App");
        primaryStage.show();

        wireInputAndOutput(textField, textArea);
        startConsoleTask();
    }

    private void wireInputAndOutput(TextField input, TextArea output) throws IOException {
        var inputConsumer = StreamWriter.redirectStandardIn(threadPool);
        StreamReader.redirectStandardOut(new BufferedTextAreaAppender(output), threadPool);

        input.setOnAction(e -> {
            e.consume();
            var text = input.textProperty().getValueSafe() + "\n";
            output.appendText(text);
            inputConsumer.accept(text);
            input.clear();
        });
    }

    private void startConsoleTask() {
        threadPool.execute(new ConsoleTask(Platform::exit));
    }

    @Override
    public void stop() {
        threadPool.shutdownNow();
    }
}

BufferedTextAreaAppender.java

import java.util.function.IntConsumer;
import javafx.application.Platform;
import javafx.scene.control.TextArea;

class BufferedTextAreaAppender implements IntConsumer {

    private final StringBuilder buffer = new StringBuilder();
    private final TextArea textArea;
    private boolean runLater = true;

    BufferedTextAreaAppender(TextArea textArea) {
        this.textArea = textArea;
    }

    @Override
    public void accept(int characterCodePoint) {
        synchronized (buffer) {
            buffer.append((char) characterCodePoint);
            if (runLater) {
                runLater = false;
                Platform.runLater(this::appendToTextArea);
            }
        }
    }

    private void appendToTextArea() {
        synchronized (buffer) {
            textArea.appendText(buffer.toString());
            buffer.delete(0, buffer.length());
            runLater = true;
        }
    }
}

StreamReader.java (could probably use a better name)

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.InterruptedIOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.io.PrintStream;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.Executor;
import java.util.function.IntConsumer;

class StreamReader implements Runnable {

    static void redirectStandardOut(IntConsumer onNextCharacter, Executor executor) throws IOException {
        var out = new PipedOutputStream();
        System.setOut(new PrintStream(out, true, StandardCharsets.UTF_8));

        var reader = new StreamReader(new PipedInputStream(out), onNextCharacter);
        executor.execute(reader);
    }

    private final BufferedReader reader;
    private final IntConsumer onNextCharacter;

    StreamReader(InputStream stream, IntConsumer onNextCharacter) {
        this.reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8));
        this.onNextCharacter = onNextCharacter;
    }

    @Override
    public void run() {
        try (reader) {
            int charAsInt;
            while (!Thread.interrupted() && (charAsInt = reader.read()) != -1) {
                onNextCharacter.accept(charAsInt);
            }
        } catch (InterruptedIOException ignore) {
        } catch (IOException ioe) {
            throw new UncheckedIOException(ioe);
        }
    }
}

StreamWriter.java (could probably use a better name)

import java.io.BufferedWriter;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Executor;
import java.util.function.Consumer;

class StreamWriter implements Runnable {

    static Consumer<CharSequence> redirectStandardIn(Executor executor) throws IOException {
        var in = new PipedInputStream();
        System.setIn(in);

        var queue = new ArrayBlockingQueue<CharSequence>(500);
        var writer = new StreamWriter(new PipedOutputStream(in), queue);
        executor.execute(writer);

        return queue::add;
    }

    private final BufferedWriter writer;
    private final BlockingQueue<? extends CharSequence> lineQueue;

    StreamWriter(OutputStream out, BlockingQueue<? extends CharSequence> lineQueue) {
        this.writer = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8));
        this.lineQueue = lineQueue;
    }

    @Override
    public void run() {
        try (writer) {
            while (!Thread.interrupted()) {
                writer.append(lineQueue.take());
                writer.flush();
            }
        } catch (InterruptedIOException | InterruptedException ignore) {
        } catch (IOException ioe) {
            throw new UncheckedIOException(ioe);
        }
    }
}

ConsoleTask.java

import java.nio.charset.StandardCharsets;
import java.util.NoSuchElementException;
import java.util.Scanner;

class ConsoleTask implements Runnable {

    private final Scanner scanner = new Scanner(System.in);
    private final Runnable onExit;

    ConsoleTask(Runnable onExit) {
        this.onExit = onExit;
    }

    @Override
    public void run() {
        System.out.println("WELCOME TO CONSOLE APP!");

        boolean running = true;
        while (running && !Thread.interrupted()) {
            printOptions();

            int choice = readInt("Choose option: ", 1, 3);
            switch (choice) {
                case 1 -> doCheckIfLeapYear();
                case 2 -> doCheckIfPrime();
                case 3 -> running = false;
                default -> System.out.println("Unknown option!");
            }
        }

        onExit.run();
    }

    private void printOptions() {
        System.out.println();
        System.out.println("Options");
        System.out.println("-------");
        System.out.println("  1) Test Leap Year");
        System.out.println("  2) Test Prime");
        System.out.println("  3) Exit");
        System.out.println();
    }

    private int readInt(String prompt, int min, int max) {
        while (true) {
            System.out.print(prompt);

            try {
                int i = Integer.parseInt(scanner.nextLine());
                if (i >= min && i <= max) {
                    return i;
                }
            } catch (NumberFormatException | NoSuchElementException ignored) {
            }
            System.out.printf("Please enter an integer between [%,d, %,d]%n", min, max);
        }
    }

    private void doCheckIfLeapYear() {
        System.out.println();
        int year = readInt("Enter year: ", 0, 1_000_000);
        if (year % 4 == 0 || (year % 100 == 0 && year % 400 != 0)) {
            System.out.printf("The year %d is a leap year.%n", year);
        } else {
            System.out.printf("The year %d is NOT a leap year.%n", year);
        }
    }

    private void doCheckIfPrime() {
        System.out.println();
        int number = readInt("Enter an integer: ", 1, Integer.MAX_VALUE);

        boolean isPrime = true;
        for (int i = 2; i <= (int) Math.sqrt(number); i++) {
            if (number % i == 0) {
                isPrime = false;
                break;
            }
        }

        if (isPrime) {
            System.out.printf("The number %,d is prime.%n", number);
        } else {
            System.out.printf("The number %,d is NOT prime.%n", number);
        }
    }
}

GIF of Example