Solve[{a*(m^2-n^2)==2*b*m*n,a^2*(m^2+n^2)^2+b^2*(m^2+n^2)^2==(2*a*m*n+b*(m^2-n^2))^2},{a,b}]