>> syms x n;
>> int(x^3*n-1/(x^2*n+1)^2,x)

ans =

1/4*x^4*n-1/2*x/(x^2*n+1)-1/2/n^(1/2)*atan(n^(1/2)*x)


>>