To remove the ugly "$-x^2$".
\begin{align}F(x)&=\int_0^x \frac{\ln(t^2+1)dt}{t}\,dt\\
&=\frac{1}{2}\int_0^x \frac{2t\ln(t^2+1)dt}{t^2}\,dt\\
\end{align}
Perform the change of variable $y=t^2$,
\begin{align}F(x)&=\frac{1}{2}\int_0^{x^2}\dfrac{\ln(1+t)}{t}\,dt\\
&=\frac{1}{2}\Big[\ln t\ln(1+t)\Big]_0^{x^2}-\frac{1}{2}\int_0^{x^2} \frac{\ln t}{1+t}\,dt\\
&=\ln x\ln(1+x^2)-\frac{1}{2}\int_0^{x^2} \frac{\ln t}{1+t}\,dt\\
&=\ln x\ln(1+x^2)-\frac{1}{2}\int_0^{x^2}\dfrac{\ln t}{1-t}\,dt+\int_0^{x^2}\frac{t\ln t}{1-t^2}\,dt\\
\end{align}
In the last integral perform the change of variable $y=t^2$,
\begin{align}F(x)&=\ln x\ln(1+x^2)-\frac{1}{2}\int_0^{x^2}\dfrac{\ln t}{1-t}\,dt+\frac{1}{4}\int_0^{x^4}\dfrac{\ln t}{1-t}\,dt\end{align}
In the first integral perform the change of variable $y=\dfrac{t}{x^2}$,
In the second integral perform the change of variable $y=\dfrac{t}{x^4}$,
\begin{align}F(x)&=\ln x\ln(1+x^2)-\frac{x^2}{2}\int_0^{1}\dfrac{\ln(tx^2)}{1-tx^2}\,dt+\frac{x^4}{4}\int_0^{1}\dfrac{\ln(tx^4)}{1-tx^4}\,dt\\
&=\ln x\ln(1+x^2)-x^2\ln x\int_0^{1}\dfrac{1}{1-tx^2}\,dt+x^4\ln x\int_0^{1}\dfrac{1}{1-tx^4}\,dt-\\
&\frac{x^2}{2}\int_0^{1}\dfrac{\ln t}{1-tx^2}\,dt+\frac{x^4}{4}\int_0^{1}\dfrac{\ln t}{1-tx^4}\,dt\\
&=\boxed{\frac{1}{2}\text{Li}_2(x^2)-\frac{1}{4}\text{Li}_2(x^4)}
\end{align}
NB:
For $0\leq x<1$, $\text{Li}_2$ is defined by,
\begin{align}\text{Li}_2(x)=\sum_{n=1}^{\infty} \dfrac{x^n}{n^2}\end{align}
For $0<x<1$,
\begin{align}\int_0^1 \dfrac{\ln t}{1-tx}\,dt=\dfrac{\text{Li}_2(x)}{x}\end{align}
Proof:
\begin{align}\int_0^1 \dfrac{\ln t}{1-tx}\,dt&=\int_0^1 \left(\ln t\sum_{n=0}^\infty (tx)^n\right)\,dt\\
&=\sum_{n=0}^{\infty} x^n\int_0^1 t^n\ln t\,dt\\
&=-\sum_{n=0}^{\infty} \frac{x^n}{(n+1)^2}\\
&=-\frac{1}{x}\sum_{n=0}^{\infty} \frac{x^{n+1}}{(n+1)^2}\\
&=-\dfrac{\text{Li}_2(x)}{x}
\end{align}
Because, for $n\geq 0$, integer
\begin{align}\int_0^1 t^n\ln t\,dt=-\frac{1}{(n+1)^2}\end{align}